opik 1.9.26__py3-none-any.whl → 1.9.39__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (178) hide show
  1. opik/__init__.py +10 -3
  2. opik/api_objects/dataset/rest_operations.py +2 -0
  3. opik/api_objects/experiment/experiment.py +31 -5
  4. opik/api_objects/experiment/helpers.py +34 -10
  5. opik/api_objects/local_recording.py +8 -3
  6. opik/api_objects/opik_client.py +218 -46
  7. opik/api_objects/opik_query_language.py +9 -0
  8. opik/api_objects/prompt/__init__.py +11 -3
  9. opik/api_objects/prompt/base_prompt.py +69 -0
  10. opik/api_objects/prompt/base_prompt_template.py +29 -0
  11. opik/api_objects/prompt/chat/__init__.py +1 -0
  12. opik/api_objects/prompt/chat/chat_prompt.py +193 -0
  13. opik/api_objects/prompt/chat/chat_prompt_template.py +350 -0
  14. opik/api_objects/prompt/{chat_content_renderer_registry.py → chat/content_renderer_registry.py} +31 -34
  15. opik/api_objects/prompt/client.py +101 -30
  16. opik/api_objects/prompt/text/__init__.py +1 -0
  17. opik/api_objects/prompt/{prompt.py → text/prompt.py} +55 -32
  18. opik/api_objects/prompt/{prompt_template.py → text/prompt_template.py} +8 -5
  19. opik/cli/export.py +6 -2
  20. opik/config.py +0 -5
  21. opik/decorator/base_track_decorator.py +37 -40
  22. opik/evaluation/__init__.py +13 -2
  23. opik/evaluation/engine/engine.py +195 -223
  24. opik/evaluation/engine/helpers.py +8 -7
  25. opik/evaluation/engine/metrics_evaluator.py +237 -0
  26. opik/evaluation/evaluation_result.py +35 -1
  27. opik/evaluation/evaluator.py +309 -23
  28. opik/evaluation/models/litellm/util.py +78 -6
  29. opik/evaluation/report.py +14 -2
  30. opik/evaluation/rest_operations.py +6 -9
  31. opik/evaluation/test_case.py +2 -2
  32. opik/evaluation/types.py +9 -1
  33. opik/exceptions.py +17 -0
  34. opik/id_helpers.py +18 -0
  35. opik/integrations/adk/helpers.py +16 -7
  36. opik/integrations/adk/legacy_opik_tracer.py +7 -4
  37. opik/integrations/adk/opik_tracer.py +3 -1
  38. opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +7 -3
  39. opik/integrations/dspy/callback.py +1 -4
  40. opik/integrations/haystack/opik_connector.py +2 -2
  41. opik/integrations/haystack/opik_tracer.py +2 -4
  42. opik/integrations/langchain/opik_tracer.py +1 -4
  43. opik/integrations/llama_index/callback.py +2 -4
  44. opik/integrations/openai/agents/opik_tracing_processor.py +1 -2
  45. opik/integrations/openai/opik_tracker.py +1 -1
  46. opik/opik_context.py +7 -7
  47. opik/rest_api/__init__.py +123 -11
  48. opik/rest_api/dashboards/client.py +65 -2
  49. opik/rest_api/dashboards/raw_client.py +82 -0
  50. opik/rest_api/datasets/client.py +441 -2
  51. opik/rest_api/datasets/raw_client.py +1225 -505
  52. opik/rest_api/experiments/client.py +30 -2
  53. opik/rest_api/experiments/raw_client.py +26 -0
  54. opik/rest_api/optimizations/client.py +302 -0
  55. opik/rest_api/optimizations/raw_client.py +463 -0
  56. opik/rest_api/optimizations/types/optimization_update_status.py +3 -1
  57. opik/rest_api/prompts/__init__.py +2 -2
  58. opik/rest_api/prompts/client.py +34 -4
  59. opik/rest_api/prompts/raw_client.py +32 -2
  60. opik/rest_api/prompts/types/__init__.py +3 -1
  61. opik/rest_api/prompts/types/create_prompt_version_detail_template_structure.py +5 -0
  62. opik/rest_api/prompts/types/prompt_write_template_structure.py +5 -0
  63. opik/rest_api/traces/client.py +6 -6
  64. opik/rest_api/traces/raw_client.py +4 -4
  65. opik/rest_api/types/__init__.py +121 -11
  66. opik/rest_api/types/aggregation_data.py +1 -0
  67. opik/rest_api/types/automation_rule_evaluator.py +23 -1
  68. opik/rest_api/types/automation_rule_evaluator_llm_as_judge.py +2 -0
  69. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_public.py +2 -0
  70. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_write.py +2 -0
  71. opik/rest_api/types/{automation_rule_evaluator_object_public.py → automation_rule_evaluator_object_object_public.py} +32 -10
  72. opik/rest_api/types/automation_rule_evaluator_page_public.py +2 -2
  73. opik/rest_api/types/automation_rule_evaluator_public.py +23 -1
  74. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge.py +22 -0
  75. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_public.py +22 -0
  76. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_write.py +22 -0
  77. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge.py +2 -0
  78. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_public.py +2 -0
  79. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_write.py +2 -0
  80. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python.py +2 -0
  81. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_public.py +2 -0
  82. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_write.py +2 -0
  83. opik/rest_api/types/automation_rule_evaluator_update.py +23 -1
  84. opik/rest_api/types/automation_rule_evaluator_update_llm_as_judge.py +2 -0
  85. opik/rest_api/types/automation_rule_evaluator_update_span_llm_as_judge.py +22 -0
  86. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_llm_as_judge.py +2 -0
  87. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_user_defined_metric_python.py +2 -0
  88. opik/rest_api/types/automation_rule_evaluator_update_user_defined_metric_python.py +2 -0
  89. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python.py +2 -0
  90. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_public.py +2 -0
  91. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_write.py +2 -0
  92. opik/rest_api/types/automation_rule_evaluator_write.py +23 -1
  93. opik/rest_api/types/dashboard_page_public.py +1 -0
  94. opik/rest_api/types/dataset.py +2 -0
  95. opik/rest_api/types/dataset_item.py +1 -0
  96. opik/rest_api/types/dataset_item_compare.py +1 -0
  97. opik/rest_api/types/dataset_item_page_compare.py +1 -0
  98. opik/rest_api/types/dataset_item_page_public.py +1 -0
  99. opik/rest_api/types/dataset_item_public.py +1 -0
  100. opik/rest_api/types/dataset_public.py +2 -0
  101. opik/rest_api/types/dataset_public_status.py +5 -0
  102. opik/rest_api/types/dataset_status.py +5 -0
  103. opik/rest_api/types/dataset_version_diff.py +22 -0
  104. opik/rest_api/types/dataset_version_diff_stats.py +24 -0
  105. opik/rest_api/types/dataset_version_page_public.py +23 -0
  106. opik/rest_api/types/dataset_version_public.py +49 -0
  107. opik/rest_api/types/experiment.py +2 -0
  108. opik/rest_api/types/experiment_public.py +2 -0
  109. opik/rest_api/types/experiment_score.py +20 -0
  110. opik/rest_api/types/experiment_score_public.py +20 -0
  111. opik/rest_api/types/experiment_score_write.py +20 -0
  112. opik/rest_api/types/feedback_score_public.py +4 -0
  113. opik/rest_api/types/optimization.py +2 -0
  114. opik/rest_api/types/optimization_public.py +2 -0
  115. opik/rest_api/types/optimization_public_status.py +3 -1
  116. opik/rest_api/types/optimization_status.py +3 -1
  117. opik/rest_api/types/optimization_studio_config.py +27 -0
  118. opik/rest_api/types/optimization_studio_config_public.py +27 -0
  119. opik/rest_api/types/optimization_studio_config_write.py +27 -0
  120. opik/rest_api/types/optimization_studio_log.py +22 -0
  121. opik/rest_api/types/optimization_write.py +2 -0
  122. opik/rest_api/types/optimization_write_status.py +3 -1
  123. opik/rest_api/types/prompt.py +6 -0
  124. opik/rest_api/types/prompt_detail.py +6 -0
  125. opik/rest_api/types/prompt_detail_template_structure.py +5 -0
  126. opik/rest_api/types/prompt_public.py +6 -0
  127. opik/rest_api/types/prompt_public_template_structure.py +5 -0
  128. opik/rest_api/types/prompt_template_structure.py +5 -0
  129. opik/rest_api/types/prompt_version.py +2 -0
  130. opik/rest_api/types/prompt_version_detail.py +2 -0
  131. opik/rest_api/types/prompt_version_detail_template_structure.py +5 -0
  132. opik/rest_api/types/prompt_version_public.py +2 -0
  133. opik/rest_api/types/prompt_version_public_template_structure.py +5 -0
  134. opik/rest_api/types/prompt_version_template_structure.py +5 -0
  135. opik/rest_api/types/score_name.py +1 -0
  136. opik/rest_api/types/service_toggles_config.py +5 -0
  137. opik/rest_api/types/span_filter.py +23 -0
  138. opik/rest_api/types/span_filter_operator.py +21 -0
  139. opik/rest_api/types/span_filter_write.py +23 -0
  140. opik/rest_api/types/span_filter_write_operator.py +21 -0
  141. opik/rest_api/types/span_llm_as_judge_code.py +27 -0
  142. opik/rest_api/types/span_llm_as_judge_code_public.py +27 -0
  143. opik/rest_api/types/span_llm_as_judge_code_write.py +27 -0
  144. opik/rest_api/types/studio_evaluation.py +20 -0
  145. opik/rest_api/types/studio_evaluation_public.py +20 -0
  146. opik/rest_api/types/studio_evaluation_write.py +20 -0
  147. opik/rest_api/types/studio_llm_model.py +21 -0
  148. opik/rest_api/types/studio_llm_model_public.py +21 -0
  149. opik/rest_api/types/studio_llm_model_write.py +21 -0
  150. opik/rest_api/types/studio_message.py +20 -0
  151. opik/rest_api/types/studio_message_public.py +20 -0
  152. opik/rest_api/types/studio_message_write.py +20 -0
  153. opik/rest_api/types/studio_metric.py +21 -0
  154. opik/rest_api/types/studio_metric_public.py +21 -0
  155. opik/rest_api/types/studio_metric_write.py +21 -0
  156. opik/rest_api/types/studio_optimizer.py +21 -0
  157. opik/rest_api/types/studio_optimizer_public.py +21 -0
  158. opik/rest_api/types/studio_optimizer_write.py +21 -0
  159. opik/rest_api/types/studio_prompt.py +20 -0
  160. opik/rest_api/types/studio_prompt_public.py +20 -0
  161. opik/rest_api/types/studio_prompt_write.py +20 -0
  162. opik/rest_api/types/trace.py +6 -0
  163. opik/rest_api/types/trace_public.py +6 -0
  164. opik/rest_api/types/trace_thread_filter_write.py +23 -0
  165. opik/rest_api/types/trace_thread_filter_write_operator.py +21 -0
  166. opik/rest_api/types/value_entry.py +2 -0
  167. opik/rest_api/types/value_entry_compare.py +2 -0
  168. opik/rest_api/types/value_entry_experiment_item_bulk_write_view.py +2 -0
  169. opik/rest_api/types/value_entry_public.py +2 -0
  170. opik/synchronization.py +5 -6
  171. opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py} +6 -7
  172. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/METADATA +2 -1
  173. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/RECORD +177 -119
  174. opik/api_objects/prompt/chat_prompt_template.py +0 -200
  175. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/WHEEL +0 -0
  176. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/entry_points.txt +0 -0
  177. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/licenses/LICENSE +0 -0
  178. {opik-1.9.26.dist-info → opik-1.9.39.dist-info}/top_level.txt +0 -0
@@ -0,0 +1,22 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .span_filter_public import SpanFilterPublic
8
+ from .span_llm_as_judge_code_public import SpanLlmAsJudgeCodePublic
9
+
10
+
11
+ class AutomationRuleEvaluatorSpanLlmAsJudgePublic(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[SpanFilterPublic]] = None
13
+ code: typing.Optional[SpanLlmAsJudgeCodePublic] = None
14
+
15
+ if IS_PYDANTIC_V2:
16
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
17
+ else:
18
+
19
+ class Config:
20
+ frozen = True
21
+ smart_union = True
22
+ extra = pydantic.Extra.allow
@@ -0,0 +1,22 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .span_filter_write import SpanFilterWrite
8
+ from .span_llm_as_judge_code_write import SpanLlmAsJudgeCodeWrite
9
+
10
+
11
+ class AutomationRuleEvaluatorSpanLlmAsJudgeWrite(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[SpanFilterWrite]] = None
13
+ code: typing.Optional[SpanLlmAsJudgeCodeWrite] = None
14
+
15
+ if IS_PYDANTIC_V2:
16
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
17
+ else:
18
+
19
+ class Config:
20
+ frozen = True
21
+ smart_union = True
22
+ extra = pydantic.Extra.allow
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter import TraceThreadFilter
7
8
  from .trace_thread_llm_as_judge_code import TraceThreadLlmAsJudgeCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadLlmAsJudge(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
11
13
  code: typing.Optional[TraceThreadLlmAsJudgeCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter_public import TraceThreadFilterPublic
7
8
  from .trace_thread_llm_as_judge_code_public import TraceThreadLlmAsJudgeCodePublic
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadLlmAsJudgePublic(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilterPublic]] = None
11
13
  code: typing.Optional[TraceThreadLlmAsJudgeCodePublic] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter_write import TraceThreadFilterWrite
7
8
  from .trace_thread_llm_as_judge_code_write import TraceThreadLlmAsJudgeCodeWrite
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadLlmAsJudgeWrite(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilterWrite]] = None
11
13
  code: typing.Optional[TraceThreadLlmAsJudgeCodeWrite] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter import TraceThreadFilter
7
8
  from .trace_thread_user_defined_metric_python_code import TraceThreadUserDefinedMetricPythonCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadUserDefinedMetricPython(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
11
13
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter_public import TraceThreadFilterPublic
7
8
  from .trace_thread_user_defined_metric_python_code_public import TraceThreadUserDefinedMetricPythonCodePublic
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadUserDefinedMetricPythonPublic(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilterPublic]] = None
11
13
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCodePublic] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter_write import TraceThreadFilterWrite
7
8
  from .trace_thread_user_defined_metric_python_code_write import TraceThreadUserDefinedMetricPythonCodeWrite
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorTraceThreadUserDefinedMetricPythonWrite(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilterWrite]] = None
11
13
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCodeWrite] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -7,7 +7,10 @@ import typing
7
7
  import pydantic
8
8
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
9
9
  from .llm_as_judge_code import LlmAsJudgeCode
10
+ from .span_filter import SpanFilter
11
+ from .span_llm_as_judge_code import SpanLlmAsJudgeCode
10
12
  from .trace_filter import TraceFilter
13
+ from .trace_thread_filter import TraceThreadFilter
11
14
  from .trace_thread_llm_as_judge_code import TraceThreadLlmAsJudgeCode
12
15
  from .trace_thread_user_defined_metric_python_code import TraceThreadUserDefinedMetricPythonCode
13
16
  from .user_defined_metric_python_code import UserDefinedMetricPythonCode
@@ -17,7 +20,6 @@ class Base(UniversalBaseModel):
17
20
  name: str
18
21
  sampling_rate: typing.Optional[float] = None
19
22
  enabled: typing.Optional[bool] = None
20
- filters: typing.Optional[typing.List[TraceFilter]] = None
21
23
  project_id: str
22
24
  action: typing.Literal["evaluator"] = "evaluator"
23
25
 
@@ -33,6 +35,7 @@ class Base(UniversalBaseModel):
33
35
 
34
36
  class AutomationRuleEvaluatorUpdate_LlmAsJudge(Base):
35
37
  type: typing.Literal["llm_as_judge"] = "llm_as_judge"
38
+ filters: typing.Optional[typing.List[TraceFilter]] = None
36
39
  code: typing.Optional[LlmAsJudgeCode] = None
37
40
 
38
41
  if IS_PYDANTIC_V2:
@@ -47,6 +50,7 @@ class AutomationRuleEvaluatorUpdate_LlmAsJudge(Base):
47
50
 
48
51
  class AutomationRuleEvaluatorUpdate_UserDefinedMetricPython(Base):
49
52
  type: typing.Literal["user_defined_metric_python"] = "user_defined_metric_python"
53
+ filters: typing.Optional[typing.List[TraceFilter]] = None
50
54
  code: typing.Optional[UserDefinedMetricPythonCode] = None
51
55
 
52
56
  if IS_PYDANTIC_V2:
@@ -61,6 +65,7 @@ class AutomationRuleEvaluatorUpdate_UserDefinedMetricPython(Base):
61
65
 
62
66
  class AutomationRuleEvaluatorUpdate_TraceThreadLlmAsJudge(Base):
63
67
  type: typing.Literal["trace_thread_llm_as_judge"] = "trace_thread_llm_as_judge"
68
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
64
69
  code: typing.Optional[TraceThreadLlmAsJudgeCode] = None
65
70
 
66
71
  if IS_PYDANTIC_V2:
@@ -75,6 +80,7 @@ class AutomationRuleEvaluatorUpdate_TraceThreadLlmAsJudge(Base):
75
80
 
76
81
  class AutomationRuleEvaluatorUpdate_TraceThreadUserDefinedMetricPython(Base):
77
82
  type: typing.Literal["trace_thread_user_defined_metric_python"] = "trace_thread_user_defined_metric_python"
83
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
78
84
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCode] = None
79
85
 
80
86
  if IS_PYDANTIC_V2:
@@ -87,9 +93,25 @@ class AutomationRuleEvaluatorUpdate_TraceThreadUserDefinedMetricPython(Base):
87
93
  extra = pydantic.Extra.allow
88
94
 
89
95
 
96
+ class AutomationRuleEvaluatorUpdate_SpanLlmAsJudge(Base):
97
+ type: typing.Literal["span_llm_as_judge"] = "span_llm_as_judge"
98
+ filters: typing.Optional[typing.List[SpanFilter]] = None
99
+ code: typing.Optional[SpanLlmAsJudgeCode] = None
100
+
101
+ if IS_PYDANTIC_V2:
102
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
103
+ else:
104
+
105
+ class Config:
106
+ frozen = True
107
+ smart_union = True
108
+ extra = pydantic.Extra.allow
109
+
110
+
90
111
  AutomationRuleEvaluatorUpdate = typing.Union[
91
112
  AutomationRuleEvaluatorUpdate_LlmAsJudge,
92
113
  AutomationRuleEvaluatorUpdate_UserDefinedMetricPython,
93
114
  AutomationRuleEvaluatorUpdate_TraceThreadLlmAsJudge,
94
115
  AutomationRuleEvaluatorUpdate_TraceThreadUserDefinedMetricPython,
116
+ AutomationRuleEvaluatorUpdate_SpanLlmAsJudge,
95
117
  ]
@@ -5,9 +5,11 @@ import typing
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
7
  from .llm_as_judge_code import LlmAsJudgeCode
8
+ from .trace_filter import TraceFilter
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUpdateLlmAsJudge(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceFilter]] = None
11
13
  code: typing.Optional[LlmAsJudgeCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -0,0 +1,22 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .span_filter import SpanFilter
8
+ from .span_llm_as_judge_code import SpanLlmAsJudgeCode
9
+
10
+
11
+ class AutomationRuleEvaluatorUpdateSpanLlmAsJudge(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[SpanFilter]] = None
13
+ code: typing.Optional[SpanLlmAsJudgeCode] = None
14
+
15
+ if IS_PYDANTIC_V2:
16
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
17
+ else:
18
+
19
+ class Config:
20
+ frozen = True
21
+ smart_union = True
22
+ extra = pydantic.Extra.allow
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter import TraceThreadFilter
7
8
  from .trace_thread_llm_as_judge_code import TraceThreadLlmAsJudgeCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUpdateTraceThreadLlmAsJudge(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
11
13
  code: typing.Optional[TraceThreadLlmAsJudgeCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_thread_filter import TraceThreadFilter
7
8
  from .trace_thread_user_defined_metric_python_code import TraceThreadUserDefinedMetricPythonCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUpdateTraceThreadUserDefinedMetricPython(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceThreadFilter]] = None
11
13
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_filter import TraceFilter
7
8
  from .user_defined_metric_python_code import UserDefinedMetricPythonCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUpdateUserDefinedMetricPython(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceFilter]] = None
11
13
  code: typing.Optional[UserDefinedMetricPythonCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_filter import TraceFilter
7
8
  from .user_defined_metric_python_code import UserDefinedMetricPythonCode
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUserDefinedMetricPython(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceFilter]] = None
11
13
  code: typing.Optional[UserDefinedMetricPythonCode] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_filter_public import TraceFilterPublic
7
8
  from .user_defined_metric_python_code_public import UserDefinedMetricPythonCodePublic
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUserDefinedMetricPythonPublic(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceFilterPublic]] = None
11
13
  code: typing.Optional[UserDefinedMetricPythonCodePublic] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -4,10 +4,12 @@ import typing
4
4
 
5
5
  import pydantic
6
6
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .trace_filter_write import TraceFilterWrite
7
8
  from .user_defined_metric_python_code_write import UserDefinedMetricPythonCodeWrite
8
9
 
9
10
 
10
11
  class AutomationRuleEvaluatorUserDefinedMetricPythonWrite(UniversalBaseModel):
12
+ filters: typing.Optional[typing.List[TraceFilterWrite]] = None
11
13
  code: typing.Optional[UserDefinedMetricPythonCodeWrite] = None
12
14
 
13
15
  if IS_PYDANTIC_V2:
@@ -7,7 +7,10 @@ import typing
7
7
  import pydantic
8
8
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
9
9
  from .llm_as_judge_code_write import LlmAsJudgeCodeWrite
10
+ from .span_filter_write import SpanFilterWrite
11
+ from .span_llm_as_judge_code_write import SpanLlmAsJudgeCodeWrite
10
12
  from .trace_filter_write import TraceFilterWrite
13
+ from .trace_thread_filter_write import TraceThreadFilterWrite
11
14
  from .trace_thread_llm_as_judge_code_write import TraceThreadLlmAsJudgeCodeWrite
12
15
  from .trace_thread_user_defined_metric_python_code_write import TraceThreadUserDefinedMetricPythonCodeWrite
13
16
  from .user_defined_metric_python_code_write import UserDefinedMetricPythonCodeWrite
@@ -18,7 +21,6 @@ class Base(UniversalBaseModel):
18
21
  name: str
19
22
  sampling_rate: typing.Optional[float] = None
20
23
  enabled: typing.Optional[bool] = None
21
- filters: typing.Optional[typing.List[TraceFilterWrite]] = None
22
24
  action: typing.Literal["evaluator"] = "evaluator"
23
25
 
24
26
  if IS_PYDANTIC_V2:
@@ -33,6 +35,7 @@ class Base(UniversalBaseModel):
33
35
 
34
36
  class AutomationRuleEvaluatorWrite_LlmAsJudge(Base):
35
37
  type: typing.Literal["llm_as_judge"] = "llm_as_judge"
38
+ filters: typing.Optional[typing.List[TraceFilterWrite]] = None
36
39
  code: typing.Optional[LlmAsJudgeCodeWrite] = None
37
40
 
38
41
  if IS_PYDANTIC_V2:
@@ -47,6 +50,7 @@ class AutomationRuleEvaluatorWrite_LlmAsJudge(Base):
47
50
 
48
51
  class AutomationRuleEvaluatorWrite_UserDefinedMetricPython(Base):
49
52
  type: typing.Literal["user_defined_metric_python"] = "user_defined_metric_python"
53
+ filters: typing.Optional[typing.List[TraceFilterWrite]] = None
50
54
  code: typing.Optional[UserDefinedMetricPythonCodeWrite] = None
51
55
 
52
56
  if IS_PYDANTIC_V2:
@@ -61,6 +65,7 @@ class AutomationRuleEvaluatorWrite_UserDefinedMetricPython(Base):
61
65
 
62
66
  class AutomationRuleEvaluatorWrite_TraceThreadLlmAsJudge(Base):
63
67
  type: typing.Literal["trace_thread_llm_as_judge"] = "trace_thread_llm_as_judge"
68
+ filters: typing.Optional[typing.List[TraceThreadFilterWrite]] = None
64
69
  code: typing.Optional[TraceThreadLlmAsJudgeCodeWrite] = None
65
70
 
66
71
  if IS_PYDANTIC_V2:
@@ -75,6 +80,7 @@ class AutomationRuleEvaluatorWrite_TraceThreadLlmAsJudge(Base):
75
80
 
76
81
  class AutomationRuleEvaluatorWrite_TraceThreadUserDefinedMetricPython(Base):
77
82
  type: typing.Literal["trace_thread_user_defined_metric_python"] = "trace_thread_user_defined_metric_python"
83
+ filters: typing.Optional[typing.List[TraceThreadFilterWrite]] = None
78
84
  code: typing.Optional[TraceThreadUserDefinedMetricPythonCodeWrite] = None
79
85
 
80
86
  if IS_PYDANTIC_V2:
@@ -87,9 +93,25 @@ class AutomationRuleEvaluatorWrite_TraceThreadUserDefinedMetricPython(Base):
87
93
  extra = pydantic.Extra.allow
88
94
 
89
95
 
96
+ class AutomationRuleEvaluatorWrite_SpanLlmAsJudge(Base):
97
+ type: typing.Literal["span_llm_as_judge"] = "span_llm_as_judge"
98
+ filters: typing.Optional[typing.List[SpanFilterWrite]] = None
99
+ code: typing.Optional[SpanLlmAsJudgeCodeWrite] = None
100
+
101
+ if IS_PYDANTIC_V2:
102
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
103
+ else:
104
+
105
+ class Config:
106
+ frozen = True
107
+ smart_union = True
108
+ extra = pydantic.Extra.allow
109
+
110
+
90
111
  AutomationRuleEvaluatorWrite = typing.Union[
91
112
  AutomationRuleEvaluatorWrite_LlmAsJudge,
92
113
  AutomationRuleEvaluatorWrite_UserDefinedMetricPython,
93
114
  AutomationRuleEvaluatorWrite_TraceThreadLlmAsJudge,
94
115
  AutomationRuleEvaluatorWrite_TraceThreadUserDefinedMetricPython,
116
+ AutomationRuleEvaluatorWrite_SpanLlmAsJudge,
95
117
  ]
@@ -12,6 +12,7 @@ class DashboardPagePublic(UniversalBaseModel):
12
12
  page: typing.Optional[int] = None
13
13
  size: typing.Optional[int] = None
14
14
  total: typing.Optional[int] = None
15
+ sortable_by: typing.Optional[typing.List[str]] = None
15
16
 
16
17
  if IS_PYDANTIC_V2:
17
18
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -5,6 +5,7 @@ import typing
5
5
 
6
6
  import pydantic
7
7
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
+ from .dataset_status import DatasetStatus
8
9
  from .dataset_visibility import DatasetVisibility
9
10
 
10
11
 
@@ -25,6 +26,7 @@ class Dataset(UniversalBaseModel):
25
26
  last_created_experiment_at: typing.Optional[dt.datetime] = None
26
27
  most_recent_optimization_at: typing.Optional[dt.datetime] = None
27
28
  last_created_optimization_at: typing.Optional[dt.datetime] = None
29
+ status: typing.Optional[DatasetStatus] = None
28
30
 
29
31
  if IS_PYDANTIC_V2:
30
32
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -12,6 +12,7 @@ from .json_node import JsonNode
12
12
 
13
13
  class DatasetItem(UniversalBaseModel):
14
14
  id: typing.Optional[str] = None
15
+ draft_item_id: typing.Optional[str] = None
15
16
  trace_id: typing.Optional[str] = None
16
17
  span_id: typing.Optional[str] = None
17
18
  source: DatasetItemSource
@@ -12,6 +12,7 @@ from .json_node import JsonNode
12
12
 
13
13
  class DatasetItemCompare(UniversalBaseModel):
14
14
  id: typing.Optional[str] = None
15
+ draft_item_id: typing.Optional[str] = None
15
16
  trace_id: typing.Optional[str] = None
16
17
  span_id: typing.Optional[str] = None
17
18
  source: DatasetItemCompareSource
@@ -19,6 +19,7 @@ class DatasetItemPageCompare(UniversalBaseModel):
19
19
  sortable_by: typing_extensions.Annotated[typing.Optional[typing.List[str]], FieldMetadata(alias="sortableBy")] = (
20
20
  None
21
21
  )
22
+ has_draft: typing_extensions.Annotated[typing.Optional[bool], FieldMetadata(alias="hasDraft")] = None
22
23
 
23
24
  if IS_PYDANTIC_V2:
24
25
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -19,6 +19,7 @@ class DatasetItemPagePublic(UniversalBaseModel):
19
19
  sortable_by: typing_extensions.Annotated[typing.Optional[typing.List[str]], FieldMetadata(alias="sortableBy")] = (
20
20
  None
21
21
  )
22
+ has_draft: typing_extensions.Annotated[typing.Optional[bool], FieldMetadata(alias="hasDraft")] = None
22
23
 
23
24
  if IS_PYDANTIC_V2:
24
25
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -12,6 +12,7 @@ from .json_node import JsonNode
12
12
 
13
13
  class DatasetItemPublic(UniversalBaseModel):
14
14
  id: typing.Optional[str] = None
15
+ draft_item_id: typing.Optional[str] = None
15
16
  trace_id: typing.Optional[str] = None
16
17
  span_id: typing.Optional[str] = None
17
18
  source: DatasetItemPublicSource
@@ -5,6 +5,7 @@ import typing
5
5
 
6
6
  import pydantic
7
7
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
+ from .dataset_public_status import DatasetPublicStatus
8
9
  from .dataset_public_visibility import DatasetPublicVisibility
9
10
 
10
11
 
@@ -25,6 +26,7 @@ class DatasetPublic(UniversalBaseModel):
25
26
  last_created_experiment_at: typing.Optional[dt.datetime] = None
26
27
  most_recent_optimization_at: typing.Optional[dt.datetime] = None
27
28
  last_created_optimization_at: typing.Optional[dt.datetime] = None
29
+ status: typing.Optional[DatasetPublicStatus] = None
28
30
 
29
31
  if IS_PYDANTIC_V2:
30
32
  model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
@@ -0,0 +1,5 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ DatasetPublicStatus = typing.Union[typing.Literal["unknown", "processing", "completed", "failed"], typing.Any]
@@ -0,0 +1,5 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ DatasetStatus = typing.Union[typing.Literal["unknown", "processing", "completed", "failed"], typing.Any]
@@ -0,0 +1,22 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .dataset_version_diff_stats import DatasetVersionDiffStats
8
+
9
+
10
+ class DatasetVersionDiff(UniversalBaseModel):
11
+ from_version: typing.Optional[str] = None
12
+ to_version: typing.Optional[str] = None
13
+ statistics: typing.Optional[DatasetVersionDiffStats] = None
14
+
15
+ if IS_PYDANTIC_V2:
16
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
17
+ else:
18
+
19
+ class Config:
20
+ frozen = True
21
+ smart_union = True
22
+ extra = pydantic.Extra.allow
@@ -0,0 +1,24 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ import typing_extensions
7
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
+ from ..core.serialization import FieldMetadata
9
+
10
+
11
+ class DatasetVersionDiffStats(UniversalBaseModel):
12
+ items_added: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="itemsAdded")] = None
13
+ items_modified: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="itemsModified")] = None
14
+ items_deleted: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="itemsDeleted")] = None
15
+ items_unchanged: typing_extensions.Annotated[typing.Optional[int], FieldMetadata(alias="itemsUnchanged")] = None
16
+
17
+ if IS_PYDANTIC_V2:
18
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
19
+ else:
20
+
21
+ class Config:
22
+ frozen = True
23
+ smart_union = True
24
+ extra = pydantic.Extra.allow
@@ -0,0 +1,23 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import typing
4
+
5
+ import pydantic
6
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
7
+ from .dataset_version_public import DatasetVersionPublic
8
+
9
+
10
+ class DatasetVersionPagePublic(UniversalBaseModel):
11
+ content: typing.Optional[typing.List[DatasetVersionPublic]] = None
12
+ page: typing.Optional[int] = None
13
+ size: typing.Optional[int] = None
14
+ total: typing.Optional[int] = None
15
+
16
+ if IS_PYDANTIC_V2:
17
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
18
+ else:
19
+
20
+ class Config:
21
+ frozen = True
22
+ smart_union = True
23
+ extra = pydantic.Extra.allow
@@ -0,0 +1,49 @@
1
+ # This file was auto-generated by Fern from our API Definition.
2
+
3
+ import datetime as dt
4
+ import typing
5
+
6
+ import pydantic
7
+ from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
+
9
+
10
+ class DatasetVersionPublic(UniversalBaseModel):
11
+ id: typing.Optional[str] = None
12
+ dataset_id: typing.Optional[str] = None
13
+ version_hash: typing.Optional[str] = None
14
+ tags: typing.Optional[typing.List[str]] = None
15
+ items_total: typing.Optional[int] = pydantic.Field(default=None)
16
+ """
17
+ Total number of items in this version
18
+ """
19
+
20
+ items_added: typing.Optional[int] = pydantic.Field(default=None)
21
+ """
22
+ Number of items added since last version
23
+ """
24
+
25
+ items_modified: typing.Optional[int] = pydantic.Field(default=None)
26
+ """
27
+ Number of items modified since last version
28
+ """
29
+
30
+ items_deleted: typing.Optional[int] = pydantic.Field(default=None)
31
+ """
32
+ Number of items deleted since last version
33
+ """
34
+
35
+ change_description: typing.Optional[str] = None
36
+ metadata: typing.Optional[typing.Dict[str, str]] = None
37
+ created_at: typing.Optional[dt.datetime] = None
38
+ created_by: typing.Optional[str] = None
39
+ last_updated_at: typing.Optional[dt.datetime] = None
40
+ last_updated_by: typing.Optional[str] = None
41
+
42
+ if IS_PYDANTIC_V2:
43
+ model_config: typing.ClassVar[pydantic.ConfigDict] = pydantic.ConfigDict(extra="allow", frozen=True) # type: ignore # Pydantic v2
44
+ else:
45
+
46
+ class Config:
47
+ frozen = True
48
+ smart_union = True
49
+ extra = pydantic.Extra.allow
@@ -6,6 +6,7 @@ import typing
6
6
  import pydantic
7
7
  from ..core.pydantic_utilities import IS_PYDANTIC_V2, UniversalBaseModel
8
8
  from .comment import Comment
9
+ from .experiment_score import ExperimentScore
9
10
  from .experiment_status import ExperimentStatus
10
11
  from .experiment_type import ExperimentType
11
12
  from .feedback_score_average import FeedbackScoreAverage
@@ -34,6 +35,7 @@ class Experiment(UniversalBaseModel):
34
35
  created_by: typing.Optional[str] = None
35
36
  last_updated_by: typing.Optional[str] = None
36
37
  status: typing.Optional[ExperimentStatus] = None
38
+ experiment_scores: typing.Optional[typing.List[ExperimentScore]] = None
37
39
  prompt_version: typing.Optional[PromptVersionLink] = None
38
40
  prompt_versions: typing.Optional[typing.List[PromptVersionLink]] = None
39
41