opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/api_objects/prompt/chat/chat_prompt_template.py

@@ -0,0 +1,350 @@
+"""
+Tools for rendering chat-style prompts with multimodal content.
+
+The template mirrors :class:`PromptTemplate` but works on a list of OpenAI-like
+messages. Rendering is handled by a registry of part renderers so additional
+modalities can be plugged in without changing the core implementation.
+"""
+
+import re
+from typing import Any, Dict, List, Optional, Set, Union, cast
+from typing_extensions import override
+
+import opik.exceptions as exceptions
+
+from ..text import prompt_template
+from .. import types as prompt_types
+from .. import base_prompt_template
+from . import content_renderer_registry
+
+
+class ChatPromptTemplate(base_prompt_template.BasePromptTemplate):
+    """
+    Prompt template for chat-style prompts with multimodal content.
+
+    This class handles OpenAI-like message formats with support for text, images,
+    and video content. Templates use Mustache syntax (``{{variable}}``) by default
+    for variable substitution.
+
+    Args:
+        messages: List of message dictionaries with "role" and "content" keys.
+            Content can be a string or a list of content parts (text, image_url, video_url).
+        template_type: Template syntax to use for variable substitution.
+            Defaults to :class:`~opik.api_objects.prompt.types.PromptType.MUSTACHE`.
+        registry: Custom content renderer registry. If None, uses the default registry
+            with built-in support for text, image, and video rendering.
+        validate_placeholders: If True, raises an exception when template placeholders
+            don't match the provided variables during formatting.
+
+    Example:
+        Simple text-based chat template with variables::
+
+            template = ChatPromptTemplate([
+                {
+                    "role": "system",
+                    "content": "You are a helpful assistant specializing in {{domain}}."
+                },
+                {
+                    "role": "user",
+                    "content": "Explain {{topic}} in simple terms."
+                }
+            ])
+
+            messages = template.format({
+                "domain": "physics",
+                "topic": "quantum entanglement"
+            })
+
+        Multimodal template with image content::
+
+            template = ChatPromptTemplate([
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "What is in this image from {{location}}?"
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": "{{image_path}}",
+                                "detail": "high"
+                            }
+                        }
+                    ]
+                }
+            ])
+
+            messages = template.format({
+                "location": "Paris",
+                "image_path": "https://example.com/photo.jpg"
+            })
+
+        Template with video content::
+
+            template = ChatPromptTemplate([
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": "Analyze this video: {{description}}"
+                        },
+                        {
+                            "type": "video_url",
+                            "video_url": {
+                                "url": "{{video_url}}",
+                                "mime_type": "video/mp4"
+                            }
+                        }
+                    ]
+                }
+            ])
+
+            messages = template.format({
+                "description": "traffic analysis",
+                "video_url": "https://example.com/traffic.mp4"
+            })
+
+        When formatting with unsupported modalities, the content is replaced with
+        placeholders::
+
+            messages = template.format(
+                {"video_url": "https://example.com/video.mp4"},
+                supported_modalities={"text"}  # video not supported
+            )
+            # Returns: [{"role": "user", "content": "Analyze this video\\n<<<video>>><<</video>>>"}]
+    """
+
+    def __init__(
+        self,
+        messages: List[Dict[str, prompt_types.MessageContent]],
+        template_type: prompt_types.PromptType = prompt_types.PromptType.MUSTACHE,
+        *,
+        registry: Optional[
+            content_renderer_registry.ChatContentRendererRegistry
+        ] = None,
+        validate_placeholders: bool = False,
+    ) -> None:
+        self._messages = messages
+        self._template_type = template_type
+        self._registry = (
+            registry or content_renderer_registry.DEFAULT_CHAT_RENDERER_REGISTRY
+        )
+        self._validate_placeholders = validate_placeholders
+
+    @property
+    def messages(self) -> List[Dict[str, prompt_types.MessageContent]]:
+        return self._messages
+
+    def required_modalities(self) -> prompt_types.ModalitySet:
+        """
+        Return the union of modalities referenced across all template messages.
+        """
+        required: prompt_types.ModalitySet = set()
+        for message in self._messages:
+            content = cast(prompt_types.MessageContent, message.get("content", ""))
+            required.update(self._registry.infer_modalities(content))
+        return required
+
+    def _extract_placeholders(self, template_type: prompt_types.PromptType) -> Set[str]:
+        """
+        Extract all placeholders from all messages.
+        """
+        placeholders: Set[str] = set()
+        for message in self._messages:
+            content = cast(prompt_types.MessageContent, message.get("content", ""))
+            if isinstance(content, str):
+                placeholders.update(
+                    _extract_placeholders_from_string(content, template_type)
+                )
+            elif isinstance(content, list):
+                for part in content:
+                    if not isinstance(part, dict):
+                        continue
+                    # Extract from text parts
+                    if "text" in part:
+                        text = str(part["text"])
+                        placeholders.update(
+                            _extract_placeholders_from_string(text, template_type)
+                        )
+                    # Extract from image_url parts
+                    if "image_url" in part and isinstance(part["image_url"], dict):
+                        url = str(part["image_url"].get("url", ""))
+                        placeholders.update(
+                            _extract_placeholders_from_string(url, template_type)
+                        )
+        return placeholders
+
+    @override
+    def format(
+        self,
+        variables: Dict[str, Any],
+        supported_modalities: Optional[prompt_types.SupportedModalities] = None,
+        *,
+        template_type: Optional[Union[str, prompt_types.PromptType]] = None,
+    ) -> List[Dict[str, prompt_types.MessageContent]]:
+        """
+        Render the template messages with the provided variables.
+
+        When a part declares a modality that is not supported, the registry replaces
+        it with the configured placeholder pair (for example ``<<<image>>>``) so
+        downstream consumers receive a textual anchor while unsupported structured
+        content is gracefully elided.
+        """
+        resolved_template_type = self._registry.normalize_template_type(
+            template_type or self._template_type
+        )
+
+        # Validate placeholders if enabled and using Mustache templates
+        if (
+            self._validate_placeholders
+            and resolved_template_type == prompt_types.PromptType.MUSTACHE
+        ):
+            placeholders = self._extract_placeholders(resolved_template_type)
+            variables_keys: Set[str] = set(variables.keys())
+
+            if variables_keys != placeholders:
+                raise exceptions.PromptPlaceholdersDontMatchFormatArguments(
+                    prompt_placeholders=placeholders, format_arguments=variables_keys
+                )
+
+        rendered_messages: List[Dict[str, prompt_types.MessageContent]] = []
+
+        for message in self._messages:
+            role = message.get("role")
+            if role is None:
+                continue
+
+            content = cast(prompt_types.MessageContent, message.get("content", ""))
+            rendered_content: prompt_types.MessageContent
+            if isinstance(content, str):
+                rendered_content = _render_template_string(
+                    content, variables, resolved_template_type
+                )
+            else:
+                rendered_content = self._registry.render_content(
+                    content=cast(prompt_types.MessageContent, content),
+                    variables=variables,
+                    template_type=resolved_template_type,
+                    supported_modalities=supported_modalities,
+                )
+            rendered_messages.append(
+                {
+                    "role": role,
+                    "content": rendered_content,
+                }
+            )
+
+        return rendered_messages
+
+
+def _render_template_string(
+    template: str,
+    variables: Dict[str, Any],
+    template_type: prompt_types.PromptType,
+) -> str:
+    if not template:
+        return ""
+
+    try:
+        return prompt_template.PromptTemplate(
+            template,
+            validate_placeholders=False,
+            type=template_type,
+        ).format(**variables)
+    except Exception:
+        # Fall back to the raw template if formatting fails so evaluation keeps running.
+        return template
+
+
+def render_text_part(
+    part: prompt_types.ContentPart,
+    variables: Dict[str, Any],
+    template_type: prompt_types.PromptType,
+) -> Optional[prompt_types.ContentPart]:
+    text_template = part.get("text", "")
+    rendered_text = _render_template_string(text_template, variables, template_type)
+    return {"type": "text", "text": rendered_text}
+
+
+def render_image_url_part(
+    part: prompt_types.ContentPart,
+    variables: Dict[str, Any],
+    template_type: prompt_types.PromptType,
+) -> Optional[prompt_types.ContentPart]:
+    image_dict = part.get("image_url", {})
+    if not isinstance(image_dict, dict):
+        return None
+
+    url_template = image_dict.get("url", "")
+    rendered_url = _render_template_string(url_template, variables, template_type)
+    if not rendered_url:
+        return None
+
+    rendered_image: Dict[str, Any] = {"url": rendered_url}
+    if "detail" in image_dict:
+        rendered_image["detail"] = image_dict["detail"]
+
+    return {"type": "image_url", "image_url": rendered_image}
+
+
+def render_video_url_part(
+    part: prompt_types.ContentPart,
+    variables: Dict[str, Any],
+    template_type: prompt_types.PromptType,
+) -> Optional[prompt_types.ContentPart]:
+    """
+    Render a ``video_url`` part and preserve optional metadata.
+
+    In addition to the rendered ``url`` we keep:
+
+    - ``detail``: free-form provider hints (mirrors the image renderer semantics).
+    - ``mime_type``: the content type callers expect the downstream model to load.
+    - ``duration``: client-supplied duration in seconds to give hosts extra context.
+    - ``format``: a short format label (``mp4``, ``webm``, etc.) when known.
+    """
+    video_dict = part.get("video_url", {})
+    if not isinstance(video_dict, dict):
+        return None
+
+    url_template = video_dict.get("url", "")
+    rendered_url = _render_template_string(url_template, variables, template_type)
+    if not rendered_url:
+        return None
+
+    rendered_video: Dict[str, Any] = {"url": rendered_url}
+    for key in ("detail", "mime_type", "duration", "format"):
+        if key in video_dict:
+            rendered_video[key] = video_dict[key]
+
+    return {"type": "video_url", "video_url": rendered_video}
+
+
+def _extract_placeholders_from_string(
+    text: str, template_type: prompt_types.PromptType
+) -> Set[str]:
+    """
+    Extract placeholder keys from a string template.
+    Only supports Mustache templates for now.
+    """
+    if template_type == prompt_types.PromptType.MUSTACHE:
+        pattern = r"\{\{(.*?)\}\}"
+        return set(re.findall(pattern, text))
+    return set()
+
+
+content_renderer_registry.register_default_chat_part_renderer("text", render_text_part)
+content_renderer_registry.register_default_chat_part_renderer(
+    "image_url",
+    render_image_url_part,
+    modality="vision",
+    placeholder=("<<<image>>>", "<<</image>>>"),
+)
+content_renderer_registry.register_default_chat_part_renderer(
+    "video_url",
+    render_video_url_part,
+    modality="video",
+    placeholder=("<<<video>>>", "<<</video>>>"),
+)

opik/api_objects/prompt/chat/content_renderer_registry.py

@@ -0,0 +1,203 @@
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Tuple, Union
+
+from .. import types as prompt_types
+
+
+class ChatContentRendererRegistry:
+    """
+    Registry that knows how to render structured content parts.
+    """
+
+    def __init__(self) -> None:
+        self._part_renderers: MutableMapping[str, prompt_types.RendererFn] = {}
+        self._part_modalities: MutableMapping[
+            str, Optional[prompt_types.ModalityName]
+        ] = {}
+        self._modality_placeholders: MutableMapping[
+            prompt_types.ModalityName, Tuple[str, str]
+        ] = {
+            "vision": ("<<<image>>>", "<<</image>>>"),
+            "video": ("<<<video>>>", "<<</video>>>"),
+        }
+        self._default_placeholder: Tuple[str, str] = ("<<<media>>>", "<<</media>>>")
+        self._placeholder_value_limit = 500
+
+    def register_part_renderer(
+        self,
+        part_type: str,
+        renderer: prompt_types.RendererFn,
+        *,
+        modality: Optional[prompt_types.ModalityName] = None,
+        placeholder: Optional[Tuple[str, str]] = None,
+    ) -> None:
+        """
+        Register or override the renderer responsible for a structured content part.
+        """
+        normalized_part = part_type.lower()
+        self._part_renderers[normalized_part] = renderer
+        self._part_modalities[normalized_part] = modality
+
+        if modality and placeholder:
+            self._modality_placeholders[modality] = placeholder
+
+    def render_content(
+        self,
+        content: prompt_types.MessageContent,
+        variables: Dict[str, Any],
+        template_type: prompt_types.PromptType,
+        *,
+        supported_modalities: Optional[prompt_types.SupportedModalities] = None,
+    ) -> prompt_types.MessageContent:
+        if supported_modalities is None:
+            modality_flags: Dict[str, bool] = {}
+        else:
+            modality_flags = {
+                modality: bool(is_supported)
+                for modality, is_supported in supported_modalities.items()
+            }
+
+        if isinstance(content, str):
+            # Resolved in ChatPromptTemplate
+            return content
+
+        if not isinstance(content, list):
+            return str(content)
+
+        if not content:
+            return []
+
+        rendered_parts = self._render_structured_content(
+            content=content, variables=variables, template_type=template_type
+        )
+
+        if not rendered_parts:
+            return []
+
+        if self._should_flatten(rendered_parts, modality_flags):
+            return self._flatten_parts_to_text(rendered_parts, modality_flags)
+
+        return rendered_parts
+
+    def normalize_template_type(
+        self, template_type: Union[str, prompt_types.PromptType]
+    ) -> prompt_types.PromptType:
+        if isinstance(template_type, prompt_types.PromptType):
+            return template_type
+        try:
+            return prompt_types.PromptType(template_type)
+        except ValueError:
+            return prompt_types.PromptType.MUSTACHE
+
+    def infer_modalities(
+        self, content: prompt_types.MessageContent
+    ) -> set[prompt_types.ModalityName]:
+        if not isinstance(content, list):
+            return set()
+        modalities: set[prompt_types.ModalityName] = set()
+        for part in content:
+            if not isinstance(part, dict):
+                continue
+            part_type = part.get("type", "").lower()
+            modality = self._part_modalities.get(part_type)
+            if modality:
+                modalities.add(modality)
+        return modalities
+
+    def _render_structured_content(
+        self,
+        content: List[Any],
+        variables: Dict[str, Any],
+        template_type: prompt_types.PromptType,
+    ) -> List[prompt_types.ContentPart]:
+        rendered_parts: List[prompt_types.ContentPart] = []
+        for part in content:
+            if not isinstance(part, dict):
+                continue
+            part_type = part.get("type", "").lower()
+            renderer = self._part_renderers.get(part_type)
+            if renderer is None:
+                continue
+            rendered_part = renderer(part, variables, template_type)
+            if rendered_part is None:
+                continue
+            rendered_parts.append(rendered_part)
+        return rendered_parts
+
+    def _should_flatten(
+        self, parts: List[prompt_types.ContentPart], modality_flags: Mapping[str, bool]
+    ) -> bool:
+        for part in parts:
+            modality = self._part_modalities.get(part.get("type", "").lower())
+            if modality and not modality_flags.get(modality, False):
+                return True
+        return False
+
+    def _flatten_parts_to_text(
+        self, parts: List[prompt_types.ContentPart], modality_flags: Mapping[str, bool]
+    ) -> str:
+        segments: List[str] = []
+        for part in parts:
+            part_type = part.get("type", "").lower()
+            if part_type == "text":
+                text = part.get("text", "")
+                if text:
+                    segments.append(text)
+                continue
+            modality = self._part_modalities.get(part_type)
+            if modality and modality_flags.get(modality, False):
+                segments.append(str(part))
+                continue
+            if modality and modality in self._modality_placeholders:
+                prefix, suffix = self._modality_placeholders[modality]
+            else:
+                prefix, suffix = self._default_placeholder
+            placeholder_value = self._extract_placeholder_value(part)
+            if placeholder_value:
+                truncated = self._truncate_placeholder_value(placeholder_value)
+                segments.append(f"{prefix}{truncated}{suffix}")
+        return "\n\n".join(segment for segment in segments if segment)
+
+    @staticmethod
+    def _extract_placeholder_value(part: prompt_types.ContentPart) -> str:
+        part_type = part.get("type", "").lower()
+        if part_type == "image_url":
+            image_dict = part.get("image_url", {})
+            if isinstance(image_dict, dict):
+                return str(image_dict.get("url", "")).strip()
+        if part_type == "video_url":
+            video_dict = part.get("video_url", {})
+            if isinstance(video_dict, dict):
+                return str(video_dict.get("url", "")).strip()
+        return str(part)
+
+    def _truncate_placeholder_value(self, value: str) -> str:
+        if len(value) <= self._placeholder_value_limit:
+            return value
+        if self._placeholder_value_limit <= 3:
+            return value[: self._placeholder_value_limit]
+        return value[: self._placeholder_value_limit - 3] + "..."
+
+
+DEFAULT_CHAT_RENDERER_REGISTRY = ChatContentRendererRegistry()
+
+
+def register_default_chat_part_renderer(
+    part_type: str,
+    renderer: prompt_types.RendererFn,
+    *,
+    modality: Optional[prompt_types.ModalityName] = None,
+    placeholder: Optional[Tuple[str, str]] = None,
+) -> None:
+    DEFAULT_CHAT_RENDERER_REGISTRY.register_part_renderer(
+        part_type,
+        renderer,
+        modality=modality,
+        placeholder=placeholder,
+    )
+
+
+__all__ = [
+    "ChatContentRendererRegistry",
+    "DEFAULT_CHAT_RENDERER_REGISTRY",
+    "register_default_chat_part_renderer",
+]