opik 1.9.5__py3-none-any.whl → 1.9.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (248) hide show
  1. opik/__init__.py +10 -3
  2. opik/anonymizer/__init__.py +5 -0
  3. opik/anonymizer/anonymizer.py +12 -0
  4. opik/anonymizer/factory.py +80 -0
  5. opik/anonymizer/recursive_anonymizer.py +64 -0
  6. opik/anonymizer/rules.py +56 -0
  7. opik/anonymizer/rules_anonymizer.py +35 -0
  8. opik/api_objects/dataset/rest_operations.py +5 -0
  9. opik/api_objects/experiment/experiment.py +46 -49
  10. opik/api_objects/experiment/helpers.py +34 -10
  11. opik/api_objects/local_recording.py +8 -3
  12. opik/api_objects/opik_client.py +230 -48
  13. opik/api_objects/opik_query_language.py +9 -0
  14. opik/api_objects/prompt/__init__.py +11 -3
  15. opik/api_objects/prompt/base_prompt.py +69 -0
  16. opik/api_objects/prompt/base_prompt_template.py +29 -0
  17. opik/api_objects/prompt/chat/__init__.py +1 -0
  18. opik/api_objects/prompt/chat/chat_prompt.py +193 -0
  19. opik/api_objects/prompt/chat/chat_prompt_template.py +350 -0
  20. opik/api_objects/prompt/{chat_content_renderer_registry.py → chat/content_renderer_registry.py} +37 -35
  21. opik/api_objects/prompt/client.py +101 -30
  22. opik/api_objects/prompt/text/__init__.py +1 -0
  23. opik/api_objects/prompt/text/prompt.py +174 -0
  24. opik/api_objects/prompt/{prompt_template.py → text/prompt_template.py} +10 -6
  25. opik/api_objects/prompt/types.py +1 -1
  26. opik/cli/export.py +6 -2
  27. opik/cli/usage_report/charts.py +39 -10
  28. opik/cli/usage_report/cli.py +164 -45
  29. opik/cli/usage_report/pdf.py +14 -1
  30. opik/config.py +0 -5
  31. opik/decorator/base_track_decorator.py +37 -40
  32. opik/decorator/context_manager/span_context_manager.py +9 -0
  33. opik/decorator/context_manager/trace_context_manager.py +5 -0
  34. opik/dict_utils.py +3 -3
  35. opik/evaluation/__init__.py +13 -2
  36. opik/evaluation/engine/engine.py +195 -223
  37. opik/evaluation/engine/helpers.py +8 -7
  38. opik/evaluation/engine/metrics_evaluator.py +237 -0
  39. opik/evaluation/evaluation_result.py +35 -1
  40. opik/evaluation/evaluator.py +318 -30
  41. opik/evaluation/models/litellm/util.py +78 -6
  42. opik/evaluation/models/model_capabilities.py +33 -0
  43. opik/evaluation/report.py +14 -2
  44. opik/evaluation/rest_operations.py +36 -33
  45. opik/evaluation/test_case.py +2 -2
  46. opik/evaluation/types.py +9 -1
  47. opik/exceptions.py +17 -0
  48. opik/hooks/__init__.py +17 -1
  49. opik/hooks/anonymizer_hook.py +36 -0
  50. opik/id_helpers.py +18 -0
  51. opik/integrations/adk/helpers.py +16 -7
  52. opik/integrations/adk/legacy_opik_tracer.py +7 -4
  53. opik/integrations/adk/opik_tracer.py +3 -1
  54. opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +7 -3
  55. opik/integrations/adk/recursive_callback_injector.py +1 -6
  56. opik/integrations/dspy/callback.py +1 -4
  57. opik/integrations/haystack/opik_connector.py +2 -2
  58. opik/integrations/haystack/opik_tracer.py +2 -4
  59. opik/integrations/langchain/opik_tracer.py +273 -82
  60. opik/integrations/llama_index/callback.py +110 -108
  61. opik/integrations/openai/agents/opik_tracing_processor.py +1 -2
  62. opik/integrations/openai/opik_tracker.py +1 -1
  63. opik/message_processing/batching/batchers.py +11 -7
  64. opik/message_processing/encoder_helpers.py +79 -0
  65. opik/message_processing/messages.py +25 -1
  66. opik/message_processing/online_message_processor.py +23 -8
  67. opik/opik_context.py +7 -7
  68. opik/rest_api/__init__.py +188 -12
  69. opik/rest_api/client.py +3 -0
  70. opik/rest_api/dashboards/__init__.py +4 -0
  71. opik/rest_api/dashboards/client.py +462 -0
  72. opik/rest_api/dashboards/raw_client.py +648 -0
  73. opik/rest_api/datasets/client.py +893 -89
  74. opik/rest_api/datasets/raw_client.py +1328 -87
  75. opik/rest_api/experiments/client.py +30 -2
  76. opik/rest_api/experiments/raw_client.py +26 -0
  77. opik/rest_api/feedback_definitions/types/find_feedback_definitions_request_type.py +1 -1
  78. opik/rest_api/optimizations/client.py +302 -0
  79. opik/rest_api/optimizations/raw_client.py +463 -0
  80. opik/rest_api/optimizations/types/optimization_update_status.py +3 -1
  81. opik/rest_api/prompts/__init__.py +2 -2
  82. opik/rest_api/prompts/client.py +34 -4
  83. opik/rest_api/prompts/raw_client.py +32 -2
  84. opik/rest_api/prompts/types/__init__.py +3 -1
  85. opik/rest_api/prompts/types/create_prompt_version_detail_template_structure.py +5 -0
  86. opik/rest_api/prompts/types/prompt_write_template_structure.py +5 -0
  87. opik/rest_api/spans/__init__.py +0 -2
  88. opik/rest_api/spans/client.py +148 -64
  89. opik/rest_api/spans/raw_client.py +210 -83
  90. opik/rest_api/spans/types/__init__.py +0 -2
  91. opik/rest_api/traces/client.py +241 -73
  92. opik/rest_api/traces/raw_client.py +344 -90
  93. opik/rest_api/types/__init__.py +200 -15
  94. opik/rest_api/types/aggregation_data.py +1 -0
  95. opik/rest_api/types/alert_trigger_config_public_type.py +6 -1
  96. opik/rest_api/types/alert_trigger_config_type.py +6 -1
  97. opik/rest_api/types/alert_trigger_config_write_type.py +6 -1
  98. opik/rest_api/types/automation_rule_evaluator.py +23 -1
  99. opik/rest_api/types/automation_rule_evaluator_llm_as_judge.py +2 -0
  100. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_public.py +2 -0
  101. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_write.py +2 -0
  102. opik/rest_api/types/{automation_rule_evaluator_object_public.py → automation_rule_evaluator_object_object_public.py} +32 -10
  103. opik/rest_api/types/automation_rule_evaluator_page_public.py +2 -2
  104. opik/rest_api/types/automation_rule_evaluator_public.py +23 -1
  105. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge.py +22 -0
  106. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_public.py +22 -0
  107. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_write.py +22 -0
  108. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge.py +2 -0
  109. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_public.py +2 -0
  110. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_write.py +2 -0
  111. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python.py +2 -0
  112. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_public.py +2 -0
  113. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_write.py +2 -0
  114. opik/rest_api/types/automation_rule_evaluator_update.py +23 -1
  115. opik/rest_api/types/automation_rule_evaluator_update_llm_as_judge.py +2 -0
  116. opik/rest_api/types/automation_rule_evaluator_update_span_llm_as_judge.py +22 -0
  117. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_llm_as_judge.py +2 -0
  118. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_user_defined_metric_python.py +2 -0
  119. opik/rest_api/types/automation_rule_evaluator_update_user_defined_metric_python.py +2 -0
  120. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python.py +2 -0
  121. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_public.py +2 -0
  122. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_write.py +2 -0
  123. opik/rest_api/types/automation_rule_evaluator_write.py +23 -1
  124. opik/rest_api/types/boolean_feedback_definition.py +25 -0
  125. opik/rest_api/types/boolean_feedback_definition_create.py +20 -0
  126. opik/rest_api/types/boolean_feedback_definition_public.py +25 -0
  127. opik/rest_api/types/boolean_feedback_definition_update.py +20 -0
  128. opik/rest_api/types/boolean_feedback_detail.py +29 -0
  129. opik/rest_api/types/boolean_feedback_detail_create.py +29 -0
  130. opik/rest_api/types/boolean_feedback_detail_public.py +29 -0
  131. opik/rest_api/types/boolean_feedback_detail_update.py +29 -0
  132. opik/rest_api/types/dashboard_page_public.py +24 -0
  133. opik/rest_api/types/dashboard_public.py +30 -0
  134. opik/rest_api/types/dataset.py +2 -0
  135. opik/rest_api/types/dataset_item.py +2 -0
  136. opik/rest_api/types/dataset_item_compare.py +2 -0
  137. opik/rest_api/types/dataset_item_filter.py +23 -0
  138. opik/rest_api/types/dataset_item_filter_operator.py +21 -0
  139. opik/rest_api/types/dataset_item_page_compare.py +1 -0
  140. opik/rest_api/types/dataset_item_page_public.py +1 -0
  141. opik/rest_api/types/dataset_item_public.py +2 -0
  142. opik/rest_api/types/dataset_item_update.py +39 -0
  143. opik/rest_api/types/dataset_item_write.py +1 -0
  144. opik/rest_api/types/dataset_public.py +2 -0
  145. opik/rest_api/types/dataset_public_status.py +5 -0
  146. opik/rest_api/types/dataset_status.py +5 -0
  147. opik/rest_api/types/dataset_version_diff.py +22 -0
  148. opik/rest_api/types/dataset_version_diff_stats.py +24 -0
  149. opik/rest_api/types/dataset_version_page_public.py +23 -0
  150. opik/rest_api/types/dataset_version_public.py +49 -0
  151. opik/rest_api/types/experiment.py +2 -0
  152. opik/rest_api/types/experiment_public.py +2 -0
  153. opik/rest_api/types/experiment_score.py +20 -0
  154. opik/rest_api/types/experiment_score_public.py +20 -0
  155. opik/rest_api/types/experiment_score_write.py +20 -0
  156. opik/rest_api/types/feedback.py +20 -1
  157. opik/rest_api/types/feedback_create.py +16 -1
  158. opik/rest_api/types/feedback_object_public.py +22 -1
  159. opik/rest_api/types/feedback_public.py +20 -1
  160. opik/rest_api/types/feedback_score_public.py +4 -0
  161. opik/rest_api/types/feedback_update.py +16 -1
  162. opik/rest_api/types/image_url.py +20 -0
  163. opik/rest_api/types/image_url_public.py +20 -0
  164. opik/rest_api/types/image_url_write.py +20 -0
  165. opik/rest_api/types/llm_as_judge_message.py +5 -1
  166. opik/rest_api/types/llm_as_judge_message_content.py +24 -0
  167. opik/rest_api/types/llm_as_judge_message_content_public.py +24 -0
  168. opik/rest_api/types/llm_as_judge_message_content_write.py +24 -0
  169. opik/rest_api/types/llm_as_judge_message_public.py +5 -1
  170. opik/rest_api/types/llm_as_judge_message_write.py +5 -1
  171. opik/rest_api/types/llm_as_judge_model_parameters.py +2 -0
  172. opik/rest_api/types/llm_as_judge_model_parameters_public.py +2 -0
  173. opik/rest_api/types/llm_as_judge_model_parameters_write.py +2 -0
  174. opik/rest_api/types/optimization.py +2 -0
  175. opik/rest_api/types/optimization_public.py +2 -0
  176. opik/rest_api/types/optimization_public_status.py +3 -1
  177. opik/rest_api/types/optimization_status.py +3 -1
  178. opik/rest_api/types/optimization_studio_config.py +27 -0
  179. opik/rest_api/types/optimization_studio_config_public.py +27 -0
  180. opik/rest_api/types/optimization_studio_config_write.py +27 -0
  181. opik/rest_api/types/optimization_studio_log.py +22 -0
  182. opik/rest_api/types/optimization_write.py +2 -0
  183. opik/rest_api/types/optimization_write_status.py +3 -1
  184. opik/rest_api/types/prompt.py +6 -0
  185. opik/rest_api/types/prompt_detail.py +6 -0
  186. opik/rest_api/types/prompt_detail_template_structure.py +5 -0
  187. opik/rest_api/types/prompt_public.py +6 -0
  188. opik/rest_api/types/prompt_public_template_structure.py +5 -0
  189. opik/rest_api/types/prompt_template_structure.py +5 -0
  190. opik/rest_api/types/prompt_version.py +2 -0
  191. opik/rest_api/types/prompt_version_detail.py +2 -0
  192. opik/rest_api/types/prompt_version_detail_template_structure.py +5 -0
  193. opik/rest_api/types/prompt_version_public.py +2 -0
  194. opik/rest_api/types/prompt_version_public_template_structure.py +5 -0
  195. opik/rest_api/types/prompt_version_template_structure.py +5 -0
  196. opik/rest_api/types/score_name.py +1 -0
  197. opik/rest_api/types/service_toggles_config.py +6 -0
  198. opik/rest_api/types/span_enrichment_options.py +31 -0
  199. opik/rest_api/types/span_filter.py +23 -0
  200. opik/rest_api/types/span_filter_operator.py +21 -0
  201. opik/rest_api/types/span_filter_write.py +23 -0
  202. opik/rest_api/types/span_filter_write_operator.py +21 -0
  203. opik/rest_api/types/span_llm_as_judge_code.py +27 -0
  204. opik/rest_api/types/span_llm_as_judge_code_public.py +27 -0
  205. opik/rest_api/types/span_llm_as_judge_code_write.py +27 -0
  206. opik/rest_api/types/span_update.py +46 -0
  207. opik/rest_api/types/studio_evaluation.py +20 -0
  208. opik/rest_api/types/studio_evaluation_public.py +20 -0
  209. opik/rest_api/types/studio_evaluation_write.py +20 -0
  210. opik/rest_api/types/studio_llm_model.py +21 -0
  211. opik/rest_api/types/studio_llm_model_public.py +21 -0
  212. opik/rest_api/types/studio_llm_model_write.py +21 -0
  213. opik/rest_api/types/studio_message.py +20 -0
  214. opik/rest_api/types/studio_message_public.py +20 -0
  215. opik/rest_api/types/studio_message_write.py +20 -0
  216. opik/rest_api/types/studio_metric.py +21 -0
  217. opik/rest_api/types/studio_metric_public.py +21 -0
  218. opik/rest_api/types/studio_metric_write.py +21 -0
  219. opik/rest_api/types/studio_optimizer.py +21 -0
  220. opik/rest_api/types/studio_optimizer_public.py +21 -0
  221. opik/rest_api/types/studio_optimizer_write.py +21 -0
  222. opik/rest_api/types/studio_prompt.py +20 -0
  223. opik/rest_api/types/studio_prompt_public.py +20 -0
  224. opik/rest_api/types/studio_prompt_write.py +20 -0
  225. opik/rest_api/types/trace.py +6 -0
  226. opik/rest_api/types/trace_public.py +6 -0
  227. opik/rest_api/types/trace_thread_filter_write.py +23 -0
  228. opik/rest_api/types/trace_thread_filter_write_operator.py +21 -0
  229. opik/rest_api/types/trace_thread_update.py +19 -0
  230. opik/rest_api/types/trace_update.py +39 -0
  231. opik/rest_api/types/value_entry.py +2 -0
  232. opik/rest_api/types/value_entry_compare.py +2 -0
  233. opik/rest_api/types/value_entry_experiment_item_bulk_write_view.py +2 -0
  234. opik/rest_api/types/value_entry_public.py +2 -0
  235. opik/rest_api/types/video_url.py +19 -0
  236. opik/rest_api/types/video_url_public.py +19 -0
  237. opik/rest_api/types/video_url_write.py +19 -0
  238. opik/synchronization.py +5 -6
  239. opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py} +6 -7
  240. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/METADATA +5 -4
  241. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/RECORD +246 -151
  242. opik/api_objects/prompt/chat_prompt_template.py +0 -164
  243. opik/api_objects/prompt/prompt.py +0 -131
  244. /opik/rest_api/{spans/types → types}/span_update_type.py +0 -0
  245. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/WHEEL +0 -0
  246. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/entry_points.txt +0 -0
  247. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/licenses/LICENSE +0 -0
  248. {opik-1.9.5.dist-info → opik-1.9.39.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,49 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import datetime as dt
4
+ import typing
5
+
6
+ import pydantic
7
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
+
9
+
10
+ class DatasetVersionPublic(UniversalBaseModel):
11
+ id: typing.Optional[str] = None
12
+ dataset_id: typing.Optional[str] = None
13
+ version_hash: typing.Optional[str] = None
14
+ tags: typing.Optional[typing.List[str]] = None
15
+ items_total: typing.Optional[int] = pydantic.Field(default=None)
16
+ """
17
+ Total number of items in this version
18
+ """
19
+
20
+ items_added: typing.Optional[int] = pydantic.Field(default=None)
21
+ """
22
+ Number of items added since last version
23
+ """
24
+
25
+ items_modified: typing.Optional[int] = pydantic.Field(default=None)
26
+ """
27
+ Number of items modified since last version
28
+ """
29
+
30
+ items_deleted: typing.Optional[int] = pydantic.Field(default=None)
31
+ """
32
+ Number of items deleted since last version
33
+ """
34
+
35
+ change_description: typing.Optional[str] = None
36
+ metadata: typing.Optional[typing.Dict[str, str]] = None
37
+ created_at: typing.Optional[dt.datetime] = None
38
+ created_by: typing.Optional[str] = None
39
+ last_updated_at: typing.Optional[dt.datetime] = None
40
+ last_updated_by: typing.Optional[str] = None
41
+
42
+ if IS_PYDANTIC_V2:
43
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
44
+ else:
45
+
46
+ class Config:
47
+ frozen = True
48
+ smart_union = True
49
+ extra = pydantic.Extra.allow
@@ -6,6 +6,7 @@ import typing
6
6
  import pydantic
7
7
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
8
  from .comment import Comment
9
+ from .experiment_score import ExperimentScore
9
10
  from .experiment_status import ExperimentStatus
10
11
  from .experiment_type import ExperimentType
11
12
  from .feedback_score_average import FeedbackScoreAverage
@@ -34,6 +35,7 @@ class Experiment(UniversalBaseModel):
34
35
  created_by: typing.Optional[str] = None
35
36
  last_updated_by: typing.Optional[str] = None
36
37
  status: typing.Optional[ExperimentStatus] = None
38
+ experiment_scores: typing.Optional[typing.List[ExperimentScore]] = None
37
39
  prompt_version: typing.Optional[PromptVersionLink] = None
38
40
  prompt_versions: typing.Optional[typing.List[PromptVersionLink]] = None
39
41
 
@@ -8,6 +8,7 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
8
  from .comment_public import CommentPublic
9
9
  from .experiment_public_status import ExperimentPublicStatus
10
10
  from .experiment_public_type import ExperimentPublicType
11
+ from .experiment_score_public import ExperimentScorePublic
11
12
  from .feedback_score_average_public import FeedbackScoreAveragePublic
12
13
  from .json_list_string_public import JsonListStringPublic
13
14
  from .percentage_values_public import PercentageValuesPublic
@@ -34,6 +35,7 @@ class ExperimentPublic(UniversalBaseModel):
34
35
  created_by: typing.Optional[str] = None
35
36
  last_updated_by: typing.Optional[str] = None
36
37
  status: typing.Optional[ExperimentPublicStatus] = None
38
+ experiment_scores: typing.Optional[typing.List[ExperimentScorePublic]] = None
37
39
  prompt_version: typing.Optional[PromptVersionLinkPublic] = None
38
40
  prompt_versions: typing.Optional[typing.List[PromptVersionLinkPublic]] = None
39
41
 
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ExperimentScore(UniversalBaseModel):
10
+ name: str
11
+ value: float
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ExperimentScorePublic(UniversalBaseModel):
10
+ name: str
11
+ value: float
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ExperimentScoreWrite(UniversalBaseModel):
10
+ name: str
11
+ value: float
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -9,6 +9,7 @@ import pydantic
9
9
  import typing_extensions
10
10
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
11
11
  from ..core.serialization import FieldMetadata
12
+ from .boolean_feedback_detail import BooleanFeedbackDetail
12
13
  from .categorical_feedback_detail import CategoricalFeedbackDetail
13
14
  from .numerical_feedback_detail import NumericalFeedbackDetail
14
15
 
@@ -74,4 +75,22 @@ class Feedback_Categorical(Base):
74
75
  extra = pydantic.Extra.allow
75
76
 
76
77
 
77
- Feedback = typing.Union[Feedback_Numerical, Feedback_Categorical]
78
+ class Feedback_Boolean(Base):
79
+ type: typing.Literal["boolean"] = "boolean"
80
+ details: typing.Optional[BooleanFeedbackDetail] = None
81
+ created_at: typing.Optional[dt.datetime] = None
82
+ created_by: typing.Optional[str] = None
83
+ last_updated_at: typing.Optional[dt.datetime] = None
84
+ last_updated_by: typing.Optional[str] = None
85
+
86
+ if IS_PYDANTIC_V2:
87
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
88
+ else:
89
+
90
+ class Config:
91
+ frozen = True
92
+ smart_union = True
93
+ extra = pydantic.Extra.allow
94
+
95
+
96
+ Feedback = typing.Union[Feedback_Numerical, Feedback_Categorical, Feedback_Boolean]
@@ -6,6 +6,7 @@ import typing
6
6
 
7
7
  import pydantic
8
8
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
9
+ from .boolean_feedback_detail_create import BooleanFeedbackDetailCreate
9
10
  from .categorical_feedback_detail_create import CategoricalFeedbackDetailCreate
10
11
  from .numerical_feedback_detail_create import NumericalFeedbackDetailCreate
11
12
 
@@ -56,4 +57,18 @@ class FeedbackCreate_Categorical(Base):
56
57
  extra = pydantic.Extra.allow
57
58
 
58
59
 
59
- FeedbackCreate = typing.Union[FeedbackCreate_Numerical, FeedbackCreate_Categorical]
60
+ class FeedbackCreate_Boolean(Base):
61
+ type: typing.Literal["boolean"] = "boolean"
62
+ details: typing.Optional[BooleanFeedbackDetailCreate] = None
63
+
64
+ if IS_PYDANTIC_V2:
65
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
66
+ else:
67
+
68
+ class Config:
69
+ frozen = True
70
+ smart_union = True
71
+ extra = pydantic.Extra.allow
72
+
73
+
74
+ FeedbackCreate = typing.Union[FeedbackCreate_Numerical, FeedbackCreate_Categorical, FeedbackCreate_Boolean]
@@ -9,6 +9,7 @@ import pydantic
9
9
  import typing_extensions
10
10
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
11
11
  from ..core.serialization import FieldMetadata
12
+ from .boolean_feedback_detail_public import BooleanFeedbackDetailPublic
12
13
  from .categorical_feedback_detail_public import CategoricalFeedbackDetailPublic
13
14
  from .numerical_feedback_detail_public import NumericalFeedbackDetailPublic
14
15
 
@@ -74,4 +75,24 @@ class FeedbackObjectPublic_Categorical(Base):
74
75
  extra = pydantic.Extra.allow
75
76
 
76
77
 
77
- FeedbackObjectPublic = typing.Union[FeedbackObjectPublic_Numerical, FeedbackObjectPublic_Categorical]
78
+ class FeedbackObjectPublic_Boolean(Base):
79
+ type: typing.Literal["boolean"] = "boolean"
80
+ details: typing.Optional[BooleanFeedbackDetailPublic] = None
81
+ created_at: typing.Optional[dt.datetime] = None
82
+ created_by: typing.Optional[str] = None
83
+ last_updated_at: typing.Optional[dt.datetime] = None
84
+ last_updated_by: typing.Optional[str] = None
85
+
86
+ if IS_PYDANTIC_V2:
87
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
88
+ else:
89
+
90
+ class Config:
91
+ frozen = True
92
+ smart_union = True
93
+ extra = pydantic.Extra.allow
94
+
95
+
96
+ FeedbackObjectPublic = typing.Union[
97
+ FeedbackObjectPublic_Numerical, FeedbackObjectPublic_Categorical, FeedbackObjectPublic_Boolean
98
+ ]
@@ -9,6 +9,7 @@ import pydantic
9
9
  import typing_extensions
10
10
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
11
11
  from ..core.serialization import FieldMetadata
12
+ from .boolean_feedback_detail_public import BooleanFeedbackDetailPublic
12
13
  from .categorical_feedback_detail_public import CategoricalFeedbackDetailPublic
13
14
  from .numerical_feedback_detail_public import NumericalFeedbackDetailPublic
14
15
 
@@ -74,4 +75,22 @@ class FeedbackPublic_Categorical(Base):
74
75
  extra = pydantic.Extra.allow
75
76
 
76
77
 
77
- FeedbackPublic = typing.Union[FeedbackPublic_Numerical, FeedbackPublic_Categorical]
78
+ class FeedbackPublic_Boolean(Base):
79
+ type: typing.Literal["boolean"] = "boolean"
80
+ details: typing.Optional[BooleanFeedbackDetailPublic] = None
81
+ created_at: typing.Optional[dt.datetime] = None
82
+ created_by: typing.Optional[str] = None
83
+ last_updated_at: typing.Optional[dt.datetime] = None
84
+ last_updated_by: typing.Optional[str] = None
85
+
86
+ if IS_PYDANTIC_V2:
87
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
88
+ else:
89
+
90
+ class Config:
91
+ frozen = True
92
+ smart_union = True
93
+ extra = pydantic.Extra.allow
94
+
95
+
96
+ FeedbackPublic = typing.Union[FeedbackPublic_Numerical, FeedbackPublic_Categorical, FeedbackPublic_Boolean]
@@ -10,6 +10,10 @@ from .value_entry_public import ValueEntryPublic
10
10
 
11
11
 
12
12
  class FeedbackScorePublic(UniversalBaseModel):
13
+ """
14
+ Aggregated feedback scores from all spans in this trace, averaged by score name
15
+ """
16
+
13
17
  name: str
14
18
  category_name: typing.Optional[str] = None
15
19
  value: float
@@ -6,6 +6,7 @@ import typing
6
6
 
7
7
  import pydantic
8
8
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
9
+ from .boolean_feedback_detail_update import BooleanFeedbackDetailUpdate
9
10
  from .categorical_feedback_detail_update import CategoricalFeedbackDetailUpdate
10
11
  from .numerical_feedback_detail_update import NumericalFeedbackDetailUpdate
11
12
 
@@ -56,4 +57,18 @@ class FeedbackUpdate_Categorical(Base):
56
57
  extra = pydantic.Extra.allow
57
58
 
58
59
 
59
- FeedbackUpdate = typing.Union[FeedbackUpdate_Numerical, FeedbackUpdate_Categorical]
60
+ class FeedbackUpdate_Boolean(Base):
61
+ type: typing.Literal["boolean"] = "boolean"
62
+ details: typing.Optional[BooleanFeedbackDetailUpdate] = None
63
+
64
+ if IS_PYDANTIC_V2:
65
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
66
+ else:
67
+
68
+ class Config:
69
+ frozen = True
70
+ smart_union = True
71
+ extra = pydantic.Extra.allow
72
+
73
+
74
+ FeedbackUpdate = typing.Union[FeedbackUpdate_Numerical, FeedbackUpdate_Categorical, FeedbackUpdate_Boolean]
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ImageUrl(UniversalBaseModel):
10
+ url: str
11
+ detail: typing.Optional[str] = None
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ImageUrlPublic(UniversalBaseModel):
10
+ url: str
11
+ detail: typing.Optional[str] = None
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -0,0 +1,20 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+
8
+
9
+ class ImageUrlWrite(UniversalBaseModel):
10
+ url: str
11
+ detail: typing.Optional[str] = None
12
+
13
+ if IS_PYDANTIC_V2:
14
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
15
+ else:
16
+
17
+ class Config:
18
+ frozen = True
19
+ smart_union = True
20
+ extra = pydantic.Extra.allow
@@ -4,12 +4,16 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .llm_as_judge_message_content import LlmAsJudgeMessageContent
7
8
  from .llm_as_judge_message_role import LlmAsJudgeMessageRole
8
9
 
9
10
 
10
11
  class LlmAsJudgeMessage(UniversalBaseModel):
11
12
  role: LlmAsJudgeMessageRole
12
- content: str
13
+ content: typing.Optional[str] = None
14
+ content_array: typing.Optional[typing.List[LlmAsJudgeMessageContent]] = None
15
+ string_content: typing.Optional[bool] = None
16
+ structured_content: typing.Optional[bool] = None
13
17
 
14
18
  if IS_PYDANTIC_V2:
15
19
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -0,0 +1,24 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .image_url import ImageUrl
8
+ from .video_url import VideoUrl
9
+
10
+
11
+ class LlmAsJudgeMessageContent(UniversalBaseModel):
12
+ type: str
13
+ text: typing.Optional[str] = None
14
+ image_url: typing.Optional[ImageUrl] = None
15
+ video_url: typing.Optional[VideoUrl] = None
16
+
17
+ if IS_PYDANTIC_V2:
18
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
19
+ else:
20
+
21
+ class Config:
22
+ frozen = True
23
+ smart_union = True
24
+ extra = pydantic.Extra.allow
@@ -0,0 +1,24 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .image_url_public import ImageUrlPublic
8
+ from .video_url_public import VideoUrlPublic
9
+
10
+
11
+ class LlmAsJudgeMessageContentPublic(UniversalBaseModel):
12
+ type: str
13
+ text: typing.Optional[str] = None
14
+ image_url: typing.Optional[ImageUrlPublic] = None
15
+ video_url: typing.Optional[VideoUrlPublic] = None
16
+
17
+ if IS_PYDANTIC_V2:
18
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
19
+ else:
20
+
21
+ class Config:
22
+ frozen = True
23
+ smart_union = True
24
+ extra = pydantic.Extra.allow
@@ -0,0 +1,24 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .image_url_write import ImageUrlWrite
8
+ from .video_url_write import VideoUrlWrite
9
+
10
+
11
+ class LlmAsJudgeMessageContentWrite(UniversalBaseModel):
12
+ type: str
13
+ text: typing.Optional[str] = None
14
+ image_url: typing.Optional[ImageUrlWrite] = None
15
+ video_url: typing.Optional[VideoUrlWrite] = None
16
+
17
+ if IS_PYDANTIC_V2:
18
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
19
+ else:
20
+
21
+ class Config:
22
+ frozen = True
23
+ smart_union = True
24
+ extra = pydantic.Extra.allow
@@ -4,12 +4,16 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .llm_as_judge_message_content_public import LlmAsJudgeMessageContentPublic
7
8
  from .llm_as_judge_message_public_role import LlmAsJudgeMessagePublicRole
8
9
 
9
10
 
10
11
  class LlmAsJudgeMessagePublic(UniversalBaseModel):
11
12
  role: LlmAsJudgeMessagePublicRole
12
- content: str
13
+ content: typing.Optional[str] = None
14
+ content_array: typing.Optional[typing.List[LlmAsJudgeMessageContentPublic]] = None
15
+ string_content: typing.Optional[bool] = None
16
+ structured_content: typing.Optional[bool] = None
13
17
 
14
18
  if IS_PYDANTIC_V2:
15
19
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -4,12 +4,16 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .llm_as_judge_message_content_write import LlmAsJudgeMessageContentWrite
7
8
  from .llm_as_judge_message_write_role import LlmAsJudgeMessageWriteRole
8
9
 
9
10
 
10
11
  class LlmAsJudgeMessageWrite(UniversalBaseModel):
11
12
  role: LlmAsJudgeMessageWriteRole
12
- content: str
13
+ content: typing.Optional[str] = None
14
+ content_array: typing.Optional[typing.List[LlmAsJudgeMessageContentWrite]] = None
15
+ string_content: typing.Optional[bool] = None
16
+ structured_content: typing.Optional[bool] = None
13
17
 
14
18
  if IS_PYDANTIC_V2:
15
19
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -4,12 +4,14 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .json_node import JsonNode
7
8
 
8
9
 
9
10
  class LlmAsJudgeModelParameters(UniversalBaseModel):
10
11
  name: str
11
12
  temperature: float
12
13
  seed: typing.Optional[int] = None
14
+ custom_parameters: typing.Optional[JsonNode] = None
13
15
 
14
16
  if IS_PYDANTIC_V2:
15
17
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -4,12 +4,14 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .json_node_public import JsonNodePublic
7
8
 
8
9
 
9
10
  class LlmAsJudgeModelParametersPublic(UniversalBaseModel):
10
11
  name: str
11
12
  temperature: float
12
13
  seed: typing.Optional[int] = None
14
+ custom_parameters: typing.Optional[JsonNodePublic] = None
13
15
 
14
16
  if IS_PYDANTIC_V2:
15
17
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -4,12 +4,14 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .json_node_write import JsonNodeWrite
7
8
 
8
9
 
9
10
  class LlmAsJudgeModelParametersWrite(UniversalBaseModel):
10
11
  name: str
11
12
  temperature: float
12
13
  seed: typing.Optional[int] = None
14
+ custom_parameters: typing.Optional[JsonNodeWrite] = None
13
15
 
14
16
  if IS_PYDANTIC_V2:
15
17
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -8,6 +8,7 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
8
  from .feedback_score_average import FeedbackScoreAverage
9
9
  from .json_list_string import JsonListString
10
10
  from .optimization_status import OptimizationStatus
11
+ from .optimization_studio_config import OptimizationStudioConfig
11
12
 
12
13
 
13
14
  class Optimization(UniversalBaseModel):
@@ -17,6 +18,7 @@ class Optimization(UniversalBaseModel):
17
18
  objective_name: str
18
19
  status: OptimizationStatus
19
20
  metadata: typing.Optional[JsonListString] = None
21
+ studio_config: typing.Optional[OptimizationStudioConfig] = None
20
22
  dataset_id: typing.Optional[str] = None
21
23
  num_trials: typing.Optional[int] = None
22
24
  feedback_scores: typing.Optional[typing.List[FeedbackScoreAverage]] = None
@@ -8,6 +8,7 @@ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
8
  from .feedback_score_average_public import FeedbackScoreAveragePublic
9
9
  from .json_list_string_public import JsonListStringPublic
10
10
  from .optimization_public_status import OptimizationPublicStatus
11
+ from .optimization_studio_config_public import OptimizationStudioConfigPublic
11
12
 
12
13
 
13
14
  class OptimizationPublic(UniversalBaseModel):
@@ -17,6 +18,7 @@ class OptimizationPublic(UniversalBaseModel):
17
18
  objective_name: str
18
19
  status: OptimizationPublicStatus
19
20
  metadata: typing.Optional[JsonListStringPublic] = None
21
+ studio_config: typing.Optional[OptimizationStudioConfigPublic] = None
20
22
  dataset_id: typing.Optional[str] = None
21
23
  num_trials: typing.Optional[int] = None
22
24
  feedback_scores: typing.Optional[typing.List[FeedbackScoreAveragePublic]] = None
@@ -2,4 +2,6 @@
2
2
 
3
3
  import typing
4
4
 
5
- OptimizationPublicStatus = typing.Union[typing.Literal["running", "completed", "cancelled"], typing.Any]
5
+ OptimizationPublicStatus = typing.Union[
6
+ typing.Literal["running", "completed", "cancelled", "initialized", "error"], typing.Any
7
+ ]
@@ -2,4 +2,6 @@
2
2
 
3
3
  import typing
4
4
 
5
- OptimizationStatus = typing.Union[typing.Literal["running", "completed", "cancelled"], typing.Any]
5
+ OptimizationStatus = typing.Union[
6
+ typing.Literal["running", "completed", "cancelled", "initialized", "error"], typing.Any
7
+ ]
@@ -0,0 +1,27 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .studio_evaluation import StudioEvaluation
8
+ from .studio_llm_model import StudioLlmModel
9
+ from .studio_optimizer import StudioOptimizer
10
+ from .studio_prompt import StudioPrompt
11
+
12
+
13
+ class OptimizationStudioConfig(UniversalBaseModel):
14
+ dataset_name: str
15
+ prompt: StudioPrompt
16
+ llm_model: StudioLlmModel
17
+ evaluation: StudioEvaluation
18
+ optimizer: StudioOptimizer
19
+
20
+ if IS_PYDANTIC_V2:
21
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
22
+ else:
23
+
24
+ class Config:
25
+ frozen = True
26
+ smart_union = True
27
+ extra = pydantic.Extra.allow
@@ -0,0 +1,27 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .studio_evaluation_public import StudioEvaluationPublic
8
+ from .studio_llm_model_public import StudioLlmModelPublic
9
+ from .studio_optimizer_public import StudioOptimizerPublic
10
+ from .studio_prompt_public import StudioPromptPublic
11
+
12
+
13
+ class OptimizationStudioConfigPublic(UniversalBaseModel):
14
+ dataset_name: str
15
+ prompt: StudioPromptPublic
16
+ llm_model: StudioLlmModelPublic
17
+ evaluation: StudioEvaluationPublic
18
+ optimizer: StudioOptimizerPublic
19
+
20
+ if IS_PYDANTIC_V2:
21
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
22
+ else:
23
+
24
+ class Config:
25
+ frozen = True
26
+ smart_union = True
27
+ extra = pydantic.Extra.allow