opik 1.9.26__py3-none-any.whl → 1.9.41__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opik/__init__.py +10 -3
- opik/api_objects/dataset/rest_operations.py +2 -0
- opik/api_objects/experiment/experiment.py +31 -5
- opik/api_objects/experiment/helpers.py +34 -10
- opik/api_objects/local_recording.py +8 -3
- opik/api_objects/opik_client.py +218 -46
- opik/api_objects/opik_query_language.py +9 -0
- opik/api_objects/prompt/__init__.py +11 -3
- opik/api_objects/prompt/base_prompt.py +69 -0
- opik/api_objects/prompt/base_prompt_template.py +29 -0
- opik/api_objects/prompt/chat/__init__.py +1 -0
- opik/api_objects/prompt/chat/chat_prompt.py +193 -0
- opik/api_objects/prompt/chat/chat_prompt_template.py +350 -0
- opik/api_objects/prompt/{chat_content_renderer_registry.py → chat/content_renderer_registry.py} +31 -34
- opik/api_objects/prompt/client.py +101 -30
- opik/api_objects/prompt/text/__init__.py +1 -0
- opik/api_objects/prompt/{prompt.py → text/prompt.py} +55 -32
- opik/api_objects/prompt/{prompt_template.py → text/prompt_template.py} +8 -5
- opik/cli/export.py +6 -2
- opik/config.py +0 -5
- opik/decorator/base_track_decorator.py +37 -40
- opik/evaluation/__init__.py +13 -2
- opik/evaluation/engine/engine.py +195 -223
- opik/evaluation/engine/helpers.py +8 -7
- opik/evaluation/engine/metrics_evaluator.py +237 -0
- opik/evaluation/evaluation_result.py +35 -1
- opik/evaluation/evaluator.py +309 -23
- opik/evaluation/models/litellm/util.py +78 -6
- opik/evaluation/report.py +14 -2
- opik/evaluation/rest_operations.py +6 -9
- opik/evaluation/test_case.py +2 -2
- opik/evaluation/types.py +9 -1
- opik/exceptions.py +17 -0
- opik/id_helpers.py +18 -0
- opik/integrations/adk/helpers.py +16 -7
- opik/integrations/adk/legacy_opik_tracer.py +7 -4
- opik/integrations/adk/opik_tracer.py +3 -1
- opik/integrations/adk/patchers/adk_otel_tracer/opik_adk_otel_tracer.py +7 -3
- opik/integrations/dspy/callback.py +1 -4
- opik/integrations/haystack/opik_connector.py +2 -2
- opik/integrations/haystack/opik_tracer.py +2 -4
- opik/integrations/langchain/opik_tracer.py +1 -4
- opik/integrations/llama_index/callback.py +2 -4
- opik/integrations/openai/agents/opik_tracing_processor.py +1 -2
- opik/integrations/openai/opik_tracker.py +1 -1
- opik/opik_context.py +7 -7
- opik/rest_api/__init__.py +127 -11
- opik/rest_api/dashboards/client.py +65 -2
- opik/rest_api/dashboards/raw_client.py +82 -0
- opik/rest_api/datasets/client.py +538 -2
- opik/rest_api/datasets/raw_client.py +1347 -441
- opik/rest_api/experiments/client.py +30 -2
- opik/rest_api/experiments/raw_client.py +26 -0
- opik/rest_api/optimizations/client.py +302 -0
- opik/rest_api/optimizations/raw_client.py +463 -0
- opik/rest_api/optimizations/types/optimization_update_status.py +3 -1
- opik/rest_api/prompts/__init__.py +2 -2
- opik/rest_api/prompts/client.py +34 -4
- opik/rest_api/prompts/raw_client.py +32 -2
- opik/rest_api/prompts/types/__init__.py +3 -1
- opik/rest_api/prompts/types/create_prompt_version_detail_template_structure.py +5 -0
- opik/rest_api/prompts/types/prompt_write_template_structure.py +5 -0
- opik/rest_api/traces/client.py +6 -6
- opik/rest_api/traces/raw_client.py +4 -4
- opik/rest_api/types/__init__.py +125 -11
- opik/rest_api/types/aggregation_data.py +1 -0
- opik/rest_api/types/automation_rule_evaluator.py +23 -1
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_llm_as_judge_write.py +2 -0
- opik/rest_api/types/{automation_rule_evaluator_object_public.py → automation_rule_evaluator_object_object_public.py} +32 -10
- opik/rest_api/types/automation_rule_evaluator_page_public.py +2 -2
- opik/rest_api/types/automation_rule_evaluator_public.py +23 -1
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_public.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_span_llm_as_judge_write.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_llm_as_judge_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_trace_thread_user_defined_metric_python_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update.py +23 -1
- opik/rest_api/types/automation_rule_evaluator_update_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_span_llm_as_judge.py +22 -0
- opik/rest_api/types/automation_rule_evaluator_update_trace_thread_llm_as_judge.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_trace_thread_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_update_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_public.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_user_defined_metric_python_write.py +2 -0
- opik/rest_api/types/automation_rule_evaluator_write.py +23 -1
- opik/rest_api/types/dashboard_page_public.py +1 -0
- opik/rest_api/types/dataset.py +4 -0
- opik/rest_api/types/dataset_item.py +1 -0
- opik/rest_api/types/dataset_item_compare.py +1 -0
- opik/rest_api/types/dataset_item_page_compare.py +1 -0
- opik/rest_api/types/dataset_item_page_public.py +1 -0
- opik/rest_api/types/dataset_item_public.py +1 -0
- opik/rest_api/types/dataset_public.py +4 -0
- opik/rest_api/types/dataset_public_status.py +5 -0
- opik/rest_api/types/dataset_status.py +5 -0
- opik/rest_api/types/dataset_version_diff.py +22 -0
- opik/rest_api/types/dataset_version_diff_stats.py +24 -0
- opik/rest_api/types/dataset_version_page_public.py +23 -0
- opik/rest_api/types/dataset_version_public.py +54 -0
- opik/rest_api/types/dataset_version_summary.py +41 -0
- opik/rest_api/types/dataset_version_summary_public.py +41 -0
- opik/rest_api/types/experiment.py +2 -0
- opik/rest_api/types/experiment_public.py +2 -0
- opik/rest_api/types/experiment_score.py +20 -0
- opik/rest_api/types/experiment_score_public.py +20 -0
- opik/rest_api/types/experiment_score_write.py +20 -0
- opik/rest_api/types/feedback_score_public.py +4 -0
- opik/rest_api/types/group_content_with_aggregations.py +1 -0
- opik/rest_api/types/optimization.py +2 -0
- opik/rest_api/types/optimization_public.py +2 -0
- opik/rest_api/types/optimization_public_status.py +3 -1
- opik/rest_api/types/optimization_status.py +3 -1
- opik/rest_api/types/optimization_studio_config.py +27 -0
- opik/rest_api/types/optimization_studio_config_public.py +27 -0
- opik/rest_api/types/optimization_studio_config_write.py +27 -0
- opik/rest_api/types/optimization_studio_log.py +22 -0
- opik/rest_api/types/optimization_write.py +2 -0
- opik/rest_api/types/optimization_write_status.py +3 -1
- opik/rest_api/types/prompt.py +6 -0
- opik/rest_api/types/prompt_detail.py +6 -0
- opik/rest_api/types/prompt_detail_template_structure.py +5 -0
- opik/rest_api/types/prompt_public.py +6 -0
- opik/rest_api/types/prompt_public_template_structure.py +5 -0
- opik/rest_api/types/prompt_template_structure.py +5 -0
- opik/rest_api/types/prompt_version.py +2 -0
- opik/rest_api/types/prompt_version_detail.py +2 -0
- opik/rest_api/types/prompt_version_detail_template_structure.py +5 -0
- opik/rest_api/types/prompt_version_public.py +2 -0
- opik/rest_api/types/prompt_version_public_template_structure.py +5 -0
- opik/rest_api/types/prompt_version_template_structure.py +5 -0
- opik/rest_api/types/score_name.py +1 -0
- opik/rest_api/types/service_toggles_config.py +5 -0
- opik/rest_api/types/span_filter.py +23 -0
- opik/rest_api/types/span_filter_operator.py +21 -0
- opik/rest_api/types/span_filter_write.py +23 -0
- opik/rest_api/types/span_filter_write_operator.py +21 -0
- opik/rest_api/types/span_llm_as_judge_code.py +27 -0
- opik/rest_api/types/span_llm_as_judge_code_public.py +27 -0
- opik/rest_api/types/span_llm_as_judge_code_write.py +27 -0
- opik/rest_api/types/studio_evaluation.py +20 -0
- opik/rest_api/types/studio_evaluation_public.py +20 -0
- opik/rest_api/types/studio_evaluation_write.py +20 -0
- opik/rest_api/types/studio_llm_model.py +21 -0
- opik/rest_api/types/studio_llm_model_public.py +21 -0
- opik/rest_api/types/studio_llm_model_write.py +21 -0
- opik/rest_api/types/studio_message.py +20 -0
- opik/rest_api/types/studio_message_public.py +20 -0
- opik/rest_api/types/studio_message_write.py +20 -0
- opik/rest_api/types/studio_metric.py +21 -0
- opik/rest_api/types/studio_metric_public.py +21 -0
- opik/rest_api/types/studio_metric_write.py +21 -0
- opik/rest_api/types/studio_optimizer.py +21 -0
- opik/rest_api/types/studio_optimizer_public.py +21 -0
- opik/rest_api/types/studio_optimizer_write.py +21 -0
- opik/rest_api/types/studio_prompt.py +20 -0
- opik/rest_api/types/studio_prompt_public.py +20 -0
- opik/rest_api/types/studio_prompt_write.py +20 -0
- opik/rest_api/types/trace.py +6 -0
- opik/rest_api/types/trace_public.py +6 -0
- opik/rest_api/types/trace_thread_filter_write.py +23 -0
- opik/rest_api/types/trace_thread_filter_write_operator.py +21 -0
- opik/rest_api/types/value_entry.py +2 -0
- opik/rest_api/types/value_entry_compare.py +2 -0
- opik/rest_api/types/value_entry_experiment_item_bulk_write_view.py +2 -0
- opik/rest_api/types/value_entry_public.py +2 -0
- opik/synchronization.py +5 -6
- opik/{decorator/tracing_runtime_config.py → tracing_runtime_config.py} +6 -7
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/METADATA +4 -3
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/RECORD +180 -120
- opik/api_objects/prompt/chat_prompt_template.py +0 -200
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/WHEEL +0 -0
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/entry_points.txt +0 -0
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/licenses/LICENSE +0 -0
- {opik-1.9.26.dist-info → opik-1.9.41.dist-info}/top_level.txt +0 -0
|
@@ -1,200 +0,0 @@
|
|
|
1
|
-
"""
|
|
2
|
-
Tools for rendering chat-style prompts with multimodal content.
|
|
3
|
-
|
|
4
|
-
The template mirrors :class:`PromptTemplate` but works on a list of OpenAI-like
|
|
5
|
-
messages. Rendering is handled by a registry of part renderers so additional
|
|
6
|
-
modalities can be plugged in without changing the core implementation.
|
|
7
|
-
"""
|
|
8
|
-
|
|
9
|
-
from __future__ import annotations
|
|
10
|
-
|
|
11
|
-
from typing import Any, Dict, List, Optional, Union, cast
|
|
12
|
-
|
|
13
|
-
from .prompt_template import PromptTemplate
|
|
14
|
-
from .types import (
|
|
15
|
-
PromptType,
|
|
16
|
-
MessageContent,
|
|
17
|
-
ContentPart,
|
|
18
|
-
SupportedModalities,
|
|
19
|
-
ModalitySet,
|
|
20
|
-
)
|
|
21
|
-
from .chat_content_renderer_registry import (
|
|
22
|
-
ChatContentRendererRegistry,
|
|
23
|
-
DEFAULT_CHAT_RENDERER_REGISTRY,
|
|
24
|
-
register_default_chat_part_renderer,
|
|
25
|
-
)
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
class ChatPromptTemplate:
|
|
29
|
-
"""
|
|
30
|
-
Prompt template for chat-style prompts with multimodal content.
|
|
31
|
-
"""
|
|
32
|
-
|
|
33
|
-
def __init__(
|
|
34
|
-
self,
|
|
35
|
-
messages: List[Dict[str, MessageContent]],
|
|
36
|
-
template_type: PromptType = PromptType.MUSTACHE,
|
|
37
|
-
*,
|
|
38
|
-
registry: Optional[ChatContentRendererRegistry] = None,
|
|
39
|
-
) -> None:
|
|
40
|
-
self._messages = messages
|
|
41
|
-
self._template_type = template_type
|
|
42
|
-
self._registry = registry or DEFAULT_CHAT_RENDERER_REGISTRY
|
|
43
|
-
|
|
44
|
-
@property
|
|
45
|
-
def messages(self) -> List[Dict[str, MessageContent]]:
|
|
46
|
-
return self._messages
|
|
47
|
-
|
|
48
|
-
def required_modalities(self) -> ModalitySet:
|
|
49
|
-
"""
|
|
50
|
-
Return the union of modalities referenced across all template messages.
|
|
51
|
-
"""
|
|
52
|
-
required: ModalitySet = set()
|
|
53
|
-
for message in self._messages:
|
|
54
|
-
content = cast(MessageContent, message.get("content", ""))
|
|
55
|
-
required.update(self._registry.infer_modalities(content))
|
|
56
|
-
return required
|
|
57
|
-
|
|
58
|
-
def format(
|
|
59
|
-
self,
|
|
60
|
-
variables: Dict[str, Any],
|
|
61
|
-
supported_modalities: Optional[SupportedModalities] = None,
|
|
62
|
-
*,
|
|
63
|
-
template_type: Optional[Union[str, PromptType]] = None,
|
|
64
|
-
) -> List[Dict[str, MessageContent]]:
|
|
65
|
-
"""
|
|
66
|
-
Render the template messages with the provided variables.
|
|
67
|
-
|
|
68
|
-
When a part declares a modality that is not supported, the registry replaces
|
|
69
|
-
it with the configured placeholder pair (for example ``<<<image>>>``) so
|
|
70
|
-
downstream consumers receive a textual anchor while unsupported structured
|
|
71
|
-
content is gracefully elided.
|
|
72
|
-
"""
|
|
73
|
-
resolved_template_type = self._registry.normalize_template_type(
|
|
74
|
-
template_type or self._template_type
|
|
75
|
-
)
|
|
76
|
-
rendered_messages: List[Dict[str, MessageContent]] = []
|
|
77
|
-
|
|
78
|
-
for message in self._messages:
|
|
79
|
-
role = message.get("role")
|
|
80
|
-
if role is None:
|
|
81
|
-
continue
|
|
82
|
-
|
|
83
|
-
content = cast(MessageContent, message.get("content", ""))
|
|
84
|
-
rendered_content: MessageContent
|
|
85
|
-
if isinstance(content, str):
|
|
86
|
-
rendered_content = _render_template_string(
|
|
87
|
-
content, variables, resolved_template_type
|
|
88
|
-
)
|
|
89
|
-
else:
|
|
90
|
-
rendered_content = self._registry.render_content(
|
|
91
|
-
content=cast(MessageContent, content),
|
|
92
|
-
variables=variables,
|
|
93
|
-
template_type=resolved_template_type,
|
|
94
|
-
supported_modalities=supported_modalities,
|
|
95
|
-
)
|
|
96
|
-
rendered_messages.append(
|
|
97
|
-
{
|
|
98
|
-
"role": role,
|
|
99
|
-
"content": rendered_content,
|
|
100
|
-
}
|
|
101
|
-
)
|
|
102
|
-
|
|
103
|
-
return rendered_messages
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
def _render_template_string(
|
|
107
|
-
template: str,
|
|
108
|
-
variables: Dict[str, Any],
|
|
109
|
-
template_type: PromptType,
|
|
110
|
-
) -> str:
|
|
111
|
-
if not template:
|
|
112
|
-
return ""
|
|
113
|
-
|
|
114
|
-
try:
|
|
115
|
-
return PromptTemplate(
|
|
116
|
-
template,
|
|
117
|
-
validate_placeholders=False,
|
|
118
|
-
type=template_type,
|
|
119
|
-
).format(**variables)
|
|
120
|
-
except Exception:
|
|
121
|
-
# Fall back to the raw template if formatting fails so evaluation keeps running.
|
|
122
|
-
return template
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def render_text_part(
|
|
126
|
-
part: ContentPart, variables: Dict[str, Any], template_type: PromptType
|
|
127
|
-
) -> Optional[ContentPart]:
|
|
128
|
-
text_template = part.get("text", "")
|
|
129
|
-
rendered_text = _render_template_string(text_template, variables, template_type)
|
|
130
|
-
return {"type": "text", "text": rendered_text}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
def render_image_url_part(
|
|
134
|
-
part: ContentPart, variables: Dict[str, Any], template_type: PromptType
|
|
135
|
-
) -> Optional[ContentPart]:
|
|
136
|
-
image_dict = part.get("image_url", {})
|
|
137
|
-
if not isinstance(image_dict, dict):
|
|
138
|
-
return None
|
|
139
|
-
|
|
140
|
-
url_template = image_dict.get("url", "")
|
|
141
|
-
rendered_url = _render_template_string(url_template, variables, template_type)
|
|
142
|
-
if not rendered_url:
|
|
143
|
-
return None
|
|
144
|
-
|
|
145
|
-
rendered_image: Dict[str, Any] = {"url": rendered_url}
|
|
146
|
-
if "detail" in image_dict:
|
|
147
|
-
rendered_image["detail"] = image_dict["detail"]
|
|
148
|
-
|
|
149
|
-
return {"type": "image_url", "image_url": rendered_image}
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
def render_video_url_part(
|
|
153
|
-
part: ContentPart, variables: Dict[str, Any], template_type: PromptType
|
|
154
|
-
) -> Optional[ContentPart]:
|
|
155
|
-
"""
|
|
156
|
-
Render a ``video_url`` part and preserve optional metadata.
|
|
157
|
-
|
|
158
|
-
In addition to the rendered ``url`` we keep:
|
|
159
|
-
|
|
160
|
-
- ``detail``: free-form provider hints (mirrors the image renderer semantics).
|
|
161
|
-
- ``mime_type``: the content type callers expect the downstream model to load.
|
|
162
|
-
- ``duration``: client-supplied duration in seconds to give hosts extra context.
|
|
163
|
-
- ``format``: a short format label (``mp4``, ``webm``, etc.) when known.
|
|
164
|
-
"""
|
|
165
|
-
video_dict = part.get("video_url", {})
|
|
166
|
-
if not isinstance(video_dict, dict):
|
|
167
|
-
return None
|
|
168
|
-
|
|
169
|
-
url_template = video_dict.get("url", "")
|
|
170
|
-
rendered_url = _render_template_string(url_template, variables, template_type)
|
|
171
|
-
if not rendered_url:
|
|
172
|
-
return None
|
|
173
|
-
|
|
174
|
-
rendered_video: Dict[str, Any] = {"url": rendered_url}
|
|
175
|
-
for key in ("detail", "mime_type", "duration", "format"):
|
|
176
|
-
if key in video_dict:
|
|
177
|
-
rendered_video[key] = video_dict[key]
|
|
178
|
-
|
|
179
|
-
return {"type": "video_url", "video_url": rendered_video}
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
register_default_chat_part_renderer("text", render_text_part)
|
|
183
|
-
register_default_chat_part_renderer(
|
|
184
|
-
"image_url",
|
|
185
|
-
render_image_url_part,
|
|
186
|
-
modality="vision",
|
|
187
|
-
placeholder=("<<<image>>>", "<<</image>>>"),
|
|
188
|
-
)
|
|
189
|
-
register_default_chat_part_renderer(
|
|
190
|
-
"video_url",
|
|
191
|
-
render_video_url_part,
|
|
192
|
-
modality="video",
|
|
193
|
-
placeholder=("<<<video>>>", "<<</video>>>"),
|
|
194
|
-
)
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
__all__ = [
|
|
198
|
-
"ChatPromptTemplate",
|
|
199
|
-
"register_default_chat_part_renderer",
|
|
200
|
-
]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|