opik 1.9.26__py3-none-any.whl → 1.9.41__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (181) hide show
  1. opik/__init__.py +10 -3
  2. opik/api_objects/dataset/rest_operations.py +2 -0
  3. opik/api_objects/experiment/experiment.py +31 -5
  4. opik/api_objects/experiment/helpers.py +34 -10
  5. opik/api_objects/local_recording.py +8 -3
  6. opik/api_objects/opik_client.py +218 -46
  7. opik/api_objects/opik_query_language.py +9 -0
  8. opik/api_objects/prompt/__init__.py +11 -3
  9. opik/api_objects/prompt/base_prompt.py +69 -0
  10. opik/api_objects/prompt/base_prompt_template.py +29 -0
  11. opik/api_objects/prompt/chat/__init__.py +1 -0
  12. opik/api_objects/prompt/chat/chat_prompt.py +193 -0
  13. opik/api_objects/prompt/chat/chat_prompt_template.py +350 -0
  14. opik/api_objects/prompt/{chat_content_renderer_registry.py → chat/content_renderer_registry.py} +31 -34
  15. opik/api_objects/prompt/client.py +101 -30
  16. opik/api_objects/prompt/text/__init__.py +1 -0
  17. opik/api_objects/prompt/{prompt.py → text/prompt.py} +55 -32
  18. opik/api_objects/prompt/{prompt_template.py → text/prompt_template.py} +8 -5
  19. opik/cli/export.py +6 -2
  20. opik/config.py +0 -5
  21. opik/decorator/base_track_decorator.py +37 -40
  22. opik/evaluation/__init__.py +13 -2
  23. opik/evaluation/engine/engine.py +195 -223
  24. opik/evaluation/engine/helpers.py +8 -7
  25. opik/evaluation/engine/metrics_evaluator.py +237 -0
  26. opik/evaluation/evaluation_result.py +35 -1
  27. opik/evaluation/evaluator.py +309 -23
  28. opik/evaluation/models/litellm/util.py +78 -6
  29. opik/evaluation/report.py +14 -2
  30. opik/evaluation/rest_operations.py +6 -9
  31. opik/evaluation/test_case.py +2 -2
  32. opik/evaluation/types.py +9 -1
  33. opik/exceptions.py +17 -0
  34. opik/id_helpers.py +18 -0
  35. opik/integrations/adk/helpers.py +16 -7
  36. opik/integrations/adk/legacy_opik_tracer.py +7 -4
  37. opik/integrations/adk/opik_tracer.py +3 -1
  38. opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +7 -3
  39. opik/integrations/dspy/callback.py +1 -4
  40. opik/integrations/haystack/opik_connector.py +2 -2
  41. opik/integrations/haystack/opik_tracer.py +2 -4
  42. opik/integrations/langchain/opik_tracer.py +1 -4
  43. opik/integrations/llama_index/callback.py +2 -4
  44. opik/integrations/openai/agents/opik_tracing_processor.py +1 -2
  45. opik/integrations/openai/opik_tracker.py +1 -1
  46. opik/opik_context.py +7 -7
  47. opik/rest_api/__init__.py +127 -11
  48. opik/rest_api/dashboards/client.py +65 -2
  49. opik/rest_api/dashboards/raw_client.py +82 -0
  50. opik/rest_api/datasets/client.py +538 -2
  51. opik/rest_api/datasets/raw_client.py +1347 -441
  52. opik/rest_api/experiments/client.py +30 -2
  53. opik/rest_api/experiments/raw_client.py +26 -0
  54. opik/rest_api/optimizations/client.py +302 -0
  55. opik/rest_api/optimizations/raw_client.py +463 -0
  56. opik/rest_api/optimizations/types/optimization_update_status.py +3 -1
  57. opik/rest_api/prompts/__init__.py +2 -2
  58. opik/rest_api/prompts/client.py +34 -4
  59. opik/rest_api/prompts/raw_client.py +32 -2
  60. opik/rest_api/prompts/types/__init__.py +3 -1
  61. opik/rest_api/prompts/types/create_prompt_version_detail_template_structure.py +5 -0
  62. opik/rest_api/prompts/types/prompt_write_template_structure.py +5 -0
  63. opik/rest_api/traces/client.py +6 -6
  64. opik/rest_api/traces/raw_client.py +4 -4
  65. opik/rest_api/types/__init__.py +125 -11
  66. opik/rest_api/types/aggregation_data.py +1 -0
  67. opik/rest_api/types/automation_rule_evaluator.py +23 -1
  68. opik/rest_api/types/automation_rule_evaluator_llm_as_judge.py +2 -0
  69. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_public.py +2 -0
  70. opik/rest_api/types/automation_rule_evaluator_llm_as_judge_write.py +2 -0
  71. opik/rest_api/types/{automation_rule_evaluator_object_public.py → automation_rule_evaluator_object_object_public.py} +32 -10
  72. opik/rest_api/types/automation_rule_evaluator_page_public.py +2 -2
  73. opik/rest_api/types/automation_rule_evaluator_public.py +23 -1
  74. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge.py +22 -0
  75. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_public.py +22 -0
  76. opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_write.py +22 -0
  77. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge.py +2 -0
  78. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_public.py +2 -0
  79. opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_write.py +2 -0
  80. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python.py +2 -0
  81. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_public.py +2 -0
  82. opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_write.py +2 -0
  83. opik/rest_api/types/automation_rule_evaluator_update.py +23 -1
  84. opik/rest_api/types/automation_rule_evaluator_update_llm_as_judge.py +2 -0
  85. opik/rest_api/types/automation_rule_evaluator_update_span_llm_as_judge.py +22 -0
  86. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_llm_as_judge.py +2 -0
  87. opik/rest_api/types/automation_rule_evaluator_update_trace_thread_user_defined_metric_python.py +2 -0
  88. opik/rest_api/types/automation_rule_evaluator_update_user_defined_metric_python.py +2 -0
  89. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python.py +2 -0
  90. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_public.py +2 -0
  91. opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_write.py +2 -0
  92. opik/rest_api/types/automation_rule_evaluator_write.py +23 -1
  93. opik/rest_api/types/dashboard_page_public.py +1 -0
  94. opik/rest_api/types/dataset.py +4 -0
  95. opik/rest_api/types/dataset_item.py +1 -0
  96. opik/rest_api/types/dataset_item_compare.py +1 -0
  97. opik/rest_api/types/dataset_item_page_compare.py +1 -0
  98. opik/rest_api/types/dataset_item_page_public.py +1 -0
  99. opik/rest_api/types/dataset_item_public.py +1 -0
  100. opik/rest_api/types/dataset_public.py +4 -0
  101. opik/rest_api/types/dataset_public_status.py +5 -0
  102. opik/rest_api/types/dataset_status.py +5 -0
  103. opik/rest_api/types/dataset_version_diff.py +22 -0
  104. opik/rest_api/types/dataset_version_diff_stats.py +24 -0
  105. opik/rest_api/types/dataset_version_page_public.py +23 -0
  106. opik/rest_api/types/dataset_version_public.py +54 -0
  107. opik/rest_api/types/dataset_version_summary.py +41 -0
  108. opik/rest_api/types/dataset_version_summary_public.py +41 -0
  109. opik/rest_api/types/experiment.py +2 -0
  110. opik/rest_api/types/experiment_public.py +2 -0
  111. opik/rest_api/types/experiment_score.py +20 -0
  112. opik/rest_api/types/experiment_score_public.py +20 -0
  113. opik/rest_api/types/experiment_score_write.py +20 -0
  114. opik/rest_api/types/feedback_score_public.py +4 -0
  115. opik/rest_api/types/group_content_with_aggregations.py +1 -0
  116. opik/rest_api/types/optimization.py +2 -0
  117. opik/rest_api/types/optimization_public.py +2 -0
  118. opik/rest_api/types/optimization_public_status.py +3 -1
  119. opik/rest_api/types/optimization_status.py +3 -1
  120. opik/rest_api/types/optimization_studio_config.py +27 -0
  121. opik/rest_api/types/optimization_studio_config_public.py +27 -0
  122. opik/rest_api/types/optimization_studio_config_write.py +27 -0
  123. opik/rest_api/types/optimization_studio_log.py +22 -0
  124. opik/rest_api/types/optimization_write.py +2 -0
  125. opik/rest_api/types/optimization_write_status.py +3 -1
  126. opik/rest_api/types/prompt.py +6 -0
  127. opik/rest_api/types/prompt_detail.py +6 -0
  128. opik/rest_api/types/prompt_detail_template_structure.py +5 -0
  129. opik/rest_api/types/prompt_public.py +6 -0
  130. opik/rest_api/types/prompt_public_template_structure.py +5 -0
  131. opik/rest_api/types/prompt_template_structure.py +5 -0
  132. opik/rest_api/types/prompt_version.py +2 -0
  133. opik/rest_api/types/prompt_version_detail.py +2 -0
  134. opik/rest_api/types/prompt_version_detail_template_structure.py +5 -0
  135. opik/rest_api/types/prompt_version_public.py +2 -0
  136. opik/rest_api/types/prompt_version_public_template_structure.py +5 -0
  137. opik/rest_api/types/prompt_version_template_structure.py +5 -0
  138. opik/rest_api/types/score_name.py +1 -0
  139. opik/rest_api/types/service_toggles_config.py +5 -0
  140. opik/rest_api/types/span_filter.py +23 -0
  141. opik/rest_api/types/span_filter_operator.py +21 -0
  142. opik/rest_api/types/span_filter_write.py +23 -0
  143. opik/rest_api/types/span_filter_write_operator.py +21 -0
  144. opik/rest_api/types/span_llm_as_judge_code.py +27 -0
  145. opik/rest_api/types/span_llm_as_judge_code_public.py +27 -0
  146. opik/rest_api/types/span_llm_as_judge_code_write.py +27 -0
  147. opik/rest_api/types/studio_evaluation.py +20 -0
  148. opik/rest_api/types/studio_evaluation_public.py +20 -0
  149. opik/rest_api/types/studio_evaluation_write.py +20 -0
  150. opik/rest_api/types/studio_llm_model.py +21 -0
  151. opik/rest_api/types/studio_llm_model_public.py +21 -0
  152. opik/rest_api/types/studio_llm_model_write.py +21 -0
  153. opik/rest_api/types/studio_message.py +20 -0
  154. opik/rest_api/types/studio_message_public.py +20 -0
  155. opik/rest_api/types/studio_message_write.py +20 -0
  156. opik/rest_api/types/studio_metric.py +21 -0
  157. opik/rest_api/types/studio_metric_public.py +21 -0
  158. opik/rest_api/types/studio_metric_write.py +21 -0
  159. opik/rest_api/types/studio_optimizer.py +21 -0
  160. opik/rest_api/types/studio_optimizer_public.py +21 -0
  161. opik/rest_api/types/studio_optimizer_write.py +21 -0
  162. opik/rest_api/types/studio_prompt.py +20 -0
  163. opik/rest_api/types/studio_prompt_public.py +20 -0
  164. opik/rest_api/types/studio_prompt_write.py +20 -0
  165. opik/rest_api/types/trace.py +6 -0
  166. opik/rest_api/types/trace_public.py +6 -0
  167. opik/rest_api/types/trace_thread_filter_write.py +23 -0
  168. opik/rest_api/types/trace_thread_filter_write_operator.py +21 -0
  169. opik/rest_api/types/value_entry.py +2 -0
  170. opik/rest_api/types/value_entry_compare.py +2 -0
  171. opik/rest_api/types/value_entry_experiment_item_bulk_write_view.py +2 -0
  172. opik/rest_api/types/value_entry_public.py +2 -0
  173. opik/synchronization.py +5 -6
  174. opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py} +6 -7
  175. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/METADATA +4 -3
  176. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/RECORD +180 -120
  177. opik/api_objects/prompt/chat_prompt_template.py +0 -200
  178. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/WHEEL +0 -0
  179. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/entry_points.txt +0 -0
  180. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/licenses/LICENSE +0 -0
  181. {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/top_level.txt +0 -0
@@ -1,200 +0,0 @@
1
- """
2
- Tools for rendering chat-style prompts with multimodal content.
3
-
4
- The template mirrors :class:`PromptTemplate` but works on a list of OpenAI-like
5
- messages. Rendering is handled by a registry of part renderers so additional
6
- modalities can be plugged in without changing the core implementation.
7
- """
8
-
9
- from __future__ import annotations
10
-
11
- from typing import Any, Dict, List, Optional, Union, cast
12
-
13
- from .prompt_template import PromptTemplate
14
- from .types import (
15
- PromptType,
16
- MessageContent,
17
- ContentPart,
18
- SupportedModalities,
19
- ModalitySet,
20
- )
21
- from .chat_content_renderer_registry import (
22
- ChatContentRendererRegistry,
23
- DEFAULT_CHAT_RENDERER_REGISTRY,
24
- register_default_chat_part_renderer,
25
- )
26
-
27
-
28
- class ChatPromptTemplate:
29
- """
30
- Prompt template for chat-style prompts with multimodal content.
31
- """
32
-
33
- def __init__(
34
- self,
35
- messages: List[Dict[str, MessageContent]],
36
- template_type: PromptType = PromptType.MUSTACHE,
37
- *,
38
- registry: Optional[ChatContentRendererRegistry] = None,
39
- ) -> None:
40
- self._messages = messages
41
- self._template_type = template_type
42
- self._registry = registry or DEFAULT_CHAT_RENDERER_REGISTRY
43
-
44
- @property
45
- def messages(self) -> List[Dict[str, MessageContent]]:
46
- return self._messages
47
-
48
- def required_modalities(self) -> ModalitySet:
49
- """
50
- Return the union of modalities referenced across all template messages.
51
- """
52
- required: ModalitySet = set()
53
- for message in self._messages:
54
- content = cast(MessageContent, message.get("content", ""))
55
- required.update(self._registry.infer_modalities(content))
56
- return required
57
-
58
- def format(
59
- self,
60
- variables: Dict[str, Any],
61
- supported_modalities: Optional[SupportedModalities] = None,
62
- *,
63
- template_type: Optional[Union[str, PromptType]] = None,
64
- ) -> List[Dict[str, MessageContent]]:
65
- """
66
- Render the template messages with the provided variables.
67
-
68
- When a part declares a modality that is not supported, the registry replaces
69
- it with the configured placeholder pair (for example ``<<<image>>>``) so
70
- downstream consumers receive a textual anchor while unsupported structured
71
- content is gracefully elided.
72
- """
73
- resolved_template_type = self._registry.normalize_template_type(
74
- template_type or self._template_type
75
- )
76
- rendered_messages: List[Dict[str, MessageContent]] = []
77
-
78
- for message in self._messages:
79
- role = message.get("role")
80
- if role is None:
81
- continue
82
-
83
- content = cast(MessageContent, message.get("content", ""))
84
- rendered_content: MessageContent
85
- if isinstance(content, str):
86
- rendered_content = _render_template_string(
87
- content, variables, resolved_template_type
88
- )
89
- else:
90
- rendered_content = self._registry.render_content(
91
- content=cast(MessageContent, content),
92
- variables=variables,
93
- template_type=resolved_template_type,
94
- supported_modalities=supported_modalities,
95
- )
96
- rendered_messages.append(
97
- {
98
- "role": role,
99
- "content": rendered_content,
100
- }
101
- )
102
-
103
- return rendered_messages
104
-
105
-
106
- def _render_template_string(
107
- template: str,
108
- variables: Dict[str, Any],
109
- template_type: PromptType,
110
- ) -> str:
111
- if not template:
112
- return ""
113
-
114
- try:
115
- return PromptTemplate(
116
- template,
117
- validate_placeholders=False,
118
- type=template_type,
119
- ).format(**variables)
120
- except Exception:
121
- # Fall back to the raw template if formatting fails so evaluation keeps running.
122
- return template
123
-
124
-
125
- def render_text_part(
126
- part: ContentPart, variables: Dict[str, Any], template_type: PromptType
127
- ) -> Optional[ContentPart]:
128
- text_template = part.get("text", "")
129
- rendered_text = _render_template_string(text_template, variables, template_type)
130
- return {"type": "text", "text": rendered_text}
131
-
132
-
133
- def render_image_url_part(
134
- part: ContentPart, variables: Dict[str, Any], template_type: PromptType
135
- ) -> Optional[ContentPart]:
136
- image_dict = part.get("image_url", {})
137
- if not isinstance(image_dict, dict):
138
- return None
139
-
140
- url_template = image_dict.get("url", "")
141
- rendered_url = _render_template_string(url_template, variables, template_type)
142
- if not rendered_url:
143
- return None
144
-
145
- rendered_image: Dict[str, Any] = {"url": rendered_url}
146
- if "detail" in image_dict:
147
- rendered_image["detail"] = image_dict["detail"]
148
-
149
- return {"type": "image_url", "image_url": rendered_image}
150
-
151
-
152
- def render_video_url_part(
153
- part: ContentPart, variables: Dict[str, Any], template_type: PromptType
154
- ) -> Optional[ContentPart]:
155
- """
156
- Render a ``video_url`` part and preserve optional metadata.
157
-
158
- In addition to the rendered ``url`` we keep:
159
-
160
- - ``detail``: free-form provider hints (mirrors the image renderer semantics).
161
- - ``mime_type``: the content type callers expect the downstream model to load.
162
- - ``duration``: client-supplied duration in seconds to give hosts extra context.
163
- - ``format``: a short format label (``mp4``, ``webm``, etc.) when known.
164
- """
165
- video_dict = part.get("video_url", {})
166
- if not isinstance(video_dict, dict):
167
- return None
168
-
169
- url_template = video_dict.get("url", "")
170
- rendered_url = _render_template_string(url_template, variables, template_type)
171
- if not rendered_url:
172
- return None
173
-
174
- rendered_video: Dict[str, Any] = {"url": rendered_url}
175
- for key in ("detail", "mime_type", "duration", "format"):
176
- if key in video_dict:
177
- rendered_video[key] = video_dict[key]
178
-
179
- return {"type": "video_url", "video_url": rendered_video}
180
-
181
-
182
- register_default_chat_part_renderer("text", render_text_part)
183
- register_default_chat_part_renderer(
184
- "image_url",
185
- render_image_url_part,
186
- modality="vision",
187
- placeholder=("<<<image>>>", "<<</image>>>"),
188
- )
189
- register_default_chat_part_renderer(
190
- "video_url",
191
- render_video_url_part,
192
- modality="video",
193
- placeholder=("<<<video>>>", "<<</video>>>"),
194
- )
195
-
196
-
197
- __all__ = [
198
- "ChatPromptTemplate",
199
- "register_default_chat_part_renderer",
200
- ]
File without changes