opik 1.9.33__tar.gz → 1.9.38__tar.gz

{opik-1.9.33/src/opik.egg-info → opik-1.9.38}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik
-Version: 1.9.33
+Version: 1.9.38
 Summary: Comet tool for logging and evaluating LLM traces
 Home-page: https://www.comet.com
 Author: Comet ML Inc.
@@ -284,6 +284,7 @@ The easiest way to log traces is to use one of our direct integrations. Opik sup
 | Microsoft Agent Framework (Python) | Log traces for Microsoft Agent Framework calls | [Documentation](https://www.comet.com/docs/opik/integrations/microsoft-agent-framework?utm_source=opik&utm_medium=github&utm_content=agent_framework_link&utm_campaign=opik)              |
 | Microsoft Agent Framework (.NET) | Log traces for Microsoft Agent Framework .NET calls | [Documentation](https://www.comet.com/docs/opik/integrations/microsoft-agent-framework-dotnet?utm_source=opik&utm_medium=github&utm_content=agent_framework_dotnet_link&utm_campaign=opik) |
 | Mistral AI            | Log traces for Mistral AI LLM calls                     | [Documentation](https://www.comet.com/docs/opik/integrations/mistral?utm_source=opik&utm_medium=github&utm_content=mistral_link&utm_campaign=opik)                             |
+| n8n                   | Log traces for n8n workflow executions                  | [Documentation](https://www.comet.com/docs/opik/integrations/n8n?utm_source=opik&utm_medium=github&utm_content=n8n_link&utm_campaign=opik)                                     |
 | Novita AI             | Log traces for Novita AI LLM calls                      | [Documentation](https://www.comet.com/docs/opik/integrations/novita-ai?utm_source=opik&utm_medium=github&utm_content=novita_ai_link&utm_campaign=opik)                         |
 | Ollama                | Log traces for Ollama LLM calls                         | [Documentation](https://www.comet.com/docs/opik/integrations/ollama?utm_source=opik&utm_medium=github&utm_content=ollama_link&utm_campaign=opik)                               |
 | OpenAI (Python)       | Log traces for OpenAI LLM calls                         | [Documentation](https://www.comet.com/docs/opik/integrations/openai?utm_source=opik&utm_medium=github&utm_content=openai_link&utm_campaign=opik)                               |

{opik-1.9.33 → opik-1.9.38}/src/opik/__init__.py

@@ -6,7 +6,7 @@ from .api_objects.experiment.experiment_item import (
     ExperimentItemReferences,
 )
 from .api_objects.opik_client import Opik
-from .api_objects.prompt import Prompt
+from .api_objects.prompt import Prompt, ChatPrompt
 from .api_objects.prompt.types import PromptType
 from .api_objects.span import Span
 from .api_objects.trace import Trace
@@ -55,6 +55,7 @@ __all__ = [
     "llm_unit",
     "configure",
     "Prompt",
+    "ChatPrompt",
     "PromptType",
     "LLMProvider",
     "reset_tracing_to_config_default",

{opik-1.9.33 → opik-1.9.38}/src/opik/api_objects/dataset/rest_operations.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import List
 from opik.rest_api import OpikApi
 import opik.exceptions as exceptions

{opik-1.9.33 → opik-1.9.38}/src/opik/api_objects/experiment/experiment.py

@@ -1,14 +1,17 @@
 import functools
 import logging
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 
 from opik.message_processing.batching import sequence_splitter
 from opik.message_processing import messages, streamer
 from opik.rest_api import client as rest_api_client
-from opik.rest_api.types import experiment_public
+from opik.rest_api import types as rest_api_types
 from . import experiment_item, experiments_client
 from .. import constants, helpers
-from ...api_objects.prompt import Prompt
+from ...api_objects.prompt import base_prompt
+
+if TYPE_CHECKING:
+    from opik.evaluation.metrics import score_result
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +25,7 @@ class Experiment:
         rest_client: rest_api_client.OpikApi,
         streamer: streamer.Streamer,
         experiments_client: experiments_client.ExperimentsClient,
-        prompts: Optional[List[Prompt]] = None,
+        prompts: Optional[List[base_prompt.BasePrompt]] = None,
     ) -> None:
         self._id = id
         self._name = name
@@ -60,7 +63,7 @@ class Experiment:
     def experiments_rest_client(self) -> rest_api_client.ExperimentsClient:
         return self._rest_client.experiments
 
-    def get_experiment_data(self) -> experiment_public.ExperimentPublic:
+    def get_experiment_data(self) -> rest_api_types.experiment_public.ExperimentPublic:
         return self._rest_client.experiments.get_experiment_by_id(id=self.id)
 
     def insert(
@@ -123,3 +126,26 @@ class Experiment:
             truncate=truncate,
             max_results=max_results,
         )
+
+    def log_experiment_scores(
+        self,
+        score_results: List["score_result.ScoreResult"],
+    ) -> None:
+        """Log experiment-level scores to the backend."""
+        experiment_scores: List[rest_api_types.ExperimentScore] = []
+
+        for score_result_ in score_results:
+            if score_result_.scoring_failed:
+                continue
+
+            experiment_score = rest_api_types.ExperimentScore(
+                name=score_result_.name,
+                value=score_result_.value,
+            )
+            experiment_scores.append(experiment_score)
+
+        if experiment_scores:
+            self._rest_client.experiments.update_experiment(
+                id=self.id,
+                experiment_scores=experiment_scores,
+            )

{opik-1.9.33 → opik-1.9.38}/src/opik/api_objects/experiment/helpers.py

@@ -2,7 +2,7 @@ import logging
 import opik.jsonable_encoder as jsonable_encoder
 from typing import Any, Dict, List, Mapping, Optional, Tuple
 import copy
-from .. import prompt
+from ..prompt import base_prompt
 
 LOGGER = logging.getLogger(__name__)
 
@@ -11,7 +11,7 @@ PromptVersion = Dict[str, str]
 
 def build_metadata_and_prompt_versions(
     experiment_config: Optional[Dict[str, Any]],
-    prompts: Optional[List[prompt.Prompt]],
+    prompts: Optional[List[base_prompt.BasePrompt]],
 ) -> Tuple[Optional[Dict[str, Any]], Optional[List[PromptVersion]]]:
     prompt_versions: Optional[List[PromptVersion]] = None
 
@@ -37,9 +37,20 @@ def build_metadata_and_prompt_versions(
         prompt_versions = []
         experiment_config["prompts"] = {}
 
-        for prompt in prompts:
-            prompt_versions.append({"id": prompt.__internal_api__version_id__})
-            experiment_config["prompts"][prompt.name] = prompt.prompt
+        for prompt_obj in prompts:
+            prompt_versions.append({"id": prompt_obj.__internal_api__version_id__})
+            # Use __internal_api__to_info_dict__() to get the prompt content in a consistent way
+            prompt_info = prompt_obj.__internal_api__to_info_dict__()
+            # Extract the template/messages from the version dict
+            if "version" in prompt_info:
+                if "template" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["template"]
+                elif "messages" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["messages"]
 
     if experiment_config == {}:
         return None, None
@@ -50,9 +61,9 @@ def build_metadata_and_prompt_versions(
 
 
 def handle_prompt_args(
-    prompt: Optional[prompt.Prompt] = None,
-    prompts: Optional[List[prompt.Prompt]] = None,
-) -> Optional[List[prompt.Prompt]]:
+    prompt: Optional[base_prompt.BasePrompt] = None,
+    prompts: Optional[List[base_prompt.BasePrompt]] = None,
+) -> Optional[List[base_prompt.BasePrompt]]:
     if prompts is not None and len(prompts) > 0 and prompt is not None:
         LOGGER.warning(
             "Arguments `prompt` and `prompts` are mutually exclusive, `prompts` will be used`."

{opik-1.9.33 → opik-1.9.38}/src/opik/api_objects/opik_client.py

@@ -24,8 +24,8 @@ from .dataset import rest_operations as dataset_rest_operations
 from .experiment import experiments_client
 from .experiment import helpers as experiment_helpers
 from .experiment import rest_operations as experiment_rest_operations
-from .prompt import Prompt, PromptType
-from .prompt.client import PromptClient
+from . import prompt as prompt_module
+from .prompt import client as prompt_client
 from .threads import threads_client
 from .trace import migration as trace_migration, trace_client
 from .. import (
@@ -917,8 +917,8 @@ class Opik:
         dataset_name: str,
         name: Optional[str] = None,
         experiment_config: Optional[Dict[str, Any]] = None,
-        prompt: Optional[Prompt] = None,
-        prompts: Optional[List[Prompt]] = None,
+        prompt: Optional[prompt_module.base_prompt.BasePrompt] = None,
+        prompts: Optional[List[prompt_module.base_prompt.BasePrompt]] = None,
         type: Literal["regular", "trial", "mini-batch"] = "regular",
         optimization_id: Optional[str] = None,
     ) -> experiment.Experiment:
@@ -1394,70 +1394,188 @@ class Opik:
         name: str,
         prompt: str,
         metadata: Optional[Dict[str, Any]] = None,
-        type: PromptType = PromptType.MUSTACHE,
-    ) -> Prompt:
+        type: prompt_module.PromptType = prompt_module.PromptType.MUSTACHE,
+    ) -> prompt_module.Prompt:
         """
-        Creates a new prompt with the given name and template.
-        If a prompt with the same name already exists, it will create a new version of the existing prompt if the templates differ.
+        Creates a new text prompt with the given name and template.
+        If a text prompt with the same name already exists, it will create a new version of the existing prompt if the templates differ.
 
         Parameters:
             name: The name of the prompt.
             prompt: The template content of the prompt.
             metadata: Optional metadata to be included in the prompt.
+            type: The template type (MUSTACHE or JINJA2).
 
         Returns:
             A Prompt object containing details of the created or retrieved prompt.
 
         Raises:
-            ApiError: If there is an error during the creation of the prompt and the status code is not 409.
+            PromptTemplateStructureMismatch: If a chat prompt with the same name already exists (template structure is immutable).
+            ApiError: If there is an error during the creation of the prompt.
         """
-        prompt_client = PromptClient(self._rest_client)
-        prompt_version = prompt_client.create_prompt(
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        prompt_version = prompt_client_.create_prompt(
             name=name, prompt=prompt, metadata=metadata, type=type
         )
-        return Prompt.from_fern_prompt_version(name, prompt_version)
+        return prompt_module.Prompt.from_fern_prompt_version(name, prompt_version)
+
+    def create_chat_prompt(
+        self,
+        name: str,
+        messages: List[Dict[str, Any]],
+        metadata: Optional[Dict[str, Any]] = None,
+        type: prompt_module.PromptType = prompt_module.PromptType.MUSTACHE,
+    ) -> prompt_module.ChatPrompt:
+        """
+        Creates a new chat prompt with the given name and message templates.
+        If a chat prompt with the same name already exists, it will create a new version if the messages differ.
+
+        Parameters:
+            name: The name of the chat prompt.
+            messages: List of message dictionaries with 'role' and 'content' fields.
+            metadata: Optional metadata to be included in the prompt.
+            type: The template type (MUSTACHE or JINJA2).
+
+        Returns:
+            A ChatPrompt object containing details of the created or retrieved chat prompt.
+
+        Raises:
+            PromptTemplateStructureMismatch: If a text prompt with the same name already exists (template structure is immutable).
+            ApiError: If there is an error during the creation of the prompt.
+        """
+        return prompt_module.ChatPrompt(
+            name=name, messages=messages, metadata=metadata, type=type
+        )
 
     def get_prompt(
         self,
         name: str,
         commit: Optional[str] = None,
-    ) -> Optional[Prompt]:
+    ) -> Optional[prompt_module.Prompt]:
         """
-        Retrieve the prompt detail for a given prompt name and commit version.
+        Retrieve a text prompt by name and optional commit version.
+
+        This method only returns text prompts.
 
         Parameters:
             name: The name of the prompt.
             commit: An optional commit version of the prompt. If not provided, the latest version is retrieved.
 
         Returns:
-            Prompt: The details of the specified prompt.
+            Prompt: The details of the specified text prompt, or None if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a chat prompt (template structure mismatch).
         """
-        prompt_client = PromptClient(self._rest_client)
-        fern_prompt_version = prompt_client.get_prompt(name=name, commit=commit)
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        fern_prompt_version = prompt_client_.get_prompt(
+            name=name, commit=commit, raise_if_not_template_structure="text"
+        )
+
         if fern_prompt_version is None:
             return None
 
-        return Prompt.from_fern_prompt_version(name, fern_prompt_version)
+        return prompt_module.Prompt.from_fern_prompt_version(name, fern_prompt_version)
 
-    def get_prompt_history(self, name: str) -> List[Prompt]:
+    def get_chat_prompt(
+        self,
+        name: str,
+        commit: Optional[str] = None,
+    ) -> Optional[prompt_module.ChatPrompt]:
         """
-        Retrieve all the prompt versions history for a given prompt name.
+        Retrieve a chat prompt by name and optional commit version.
+
+        This method only returns chat prompts.
+
+        Parameters:
+            name: The name of the prompt.
+            commit: An optional commit version of the prompt. If not provided, the latest version is retrieved.
+
+        Returns:
+            ChatPrompt: The details of the specified chat prompt, or None if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a text prompt (template structure mismatch).
+        """
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        fern_prompt_version = prompt_client_.get_prompt(
+            name=name, commit=commit, raise_if_not_template_structure="chat"
+        )
+
+        if fern_prompt_version is None:
+            return None
+
+        return prompt_module.ChatPrompt.from_fern_prompt_version(
+            name, fern_prompt_version
+        )
+
+    def get_prompt_history(self, name: str) -> List[prompt_module.Prompt]:
+        """
+        Retrieve all text prompt versions history for a given prompt name.
+
+        Parameters:
+            name: The name of the prompt.
+
+        Returns:
+            List[Prompt]: A list of text Prompt instances for the given name, or an empty list if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a chat prompt (template structure mismatch).
+        """
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+
+        # First, validate that this is a text prompt by trying to get the latest version
+        # Let PromptTemplateStructureMismatch exception propagate - this is a hard error
+        latest_version = prompt_client_.get_prompt(
+            name=name, raise_if_not_template_structure="text"
+        )
+
+        if latest_version is None:
+            return []
+
+        # Now get all versions (we know it's a text prompt)
+        fern_prompt_versions = prompt_client_.get_all_prompt_versions(name=name)
+
+        result = [
+            prompt_module.Prompt.from_fern_prompt_version(name, version)
+            for version in fern_prompt_versions
+        ]
+        return result
+
+    def get_chat_prompt_history(self, name: str) -> List[prompt_module.ChatPrompt]:
+        """
+        Retrieve all chat prompt versions history for a given prompt name.
 
         Parameters:
             name: The name of the prompt.
 
         Returns:
-            List[Prompt]: A list of Prompt instances for the given name.
+            List[ChatPrompt]: A list of ChatPrompt instances for the given name, or an empty list if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a text prompt (template structure mismatch).
         """
-        prompt_client = PromptClient(self._rest_client)
-        fern_prompt_versions = prompt_client.get_all_prompt_versions(name=name)
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+
+        # First, validate that this is a chat prompt by trying to get the latest version
+        # Let PromptTemplateStructureMismatch exception propagate - this is a hard error
+        latest_version = prompt_client_.get_prompt(
+            name=name, raise_if_not_template_structure="chat"
+        )
+
+        if latest_version is None:
+            return []
+
+        # Now get all versions (we know it's a chat prompt)
+        fern_prompt_versions = prompt_client_.get_all_prompt_versions(name=name)
+
         result = [
-            Prompt.from_fern_prompt_version(name, version)
+            prompt_module.ChatPrompt.from_fern_prompt_version(name, version)
             for version in fern_prompt_versions
         ]
         return result
 
-    def get_all_prompts(self, name: str) -> List[Prompt]:
+    def get_all_prompts(self, name: str) -> List[prompt_module.Prompt]:
         """
         DEPRECATED: Please use Opik.get_prompt_history() instead.
         Retrieve all the prompt versions history for a given prompt name.
@@ -1466,16 +1584,18 @@ class Opik:
             name: The name of the prompt.
 
         Returns:
-            List[Prompt]: A list of Prompt instances for the given name.
+            List[prompt_module.Prompt]: A list of Prompt instances for the given name.
         """
         LOGGER.warning(
             "Opik.get_all_prompts() is deprecated. Please use Opik.get_prompt_history() instead."
         )
         return self.get_prompt_history(name)
 
-    def search_prompts(self, filter_string: Optional[str] = None) -> List[Prompt]:
+    def search_prompts(
+        self, filter_string: Optional[str] = None
+    ) -> List[Union[prompt_module.Prompt, prompt_module.ChatPrompt]]:
         """
-        Retrieve the latest prompt versions for the given search parameters.
+        Retrieve the latest prompt versions (both string and chat prompts) for the given search parameters.
 
         Parameters:
             filter_string: A filter string to narrow down the search using Opik Query Language (OQL).
@@ -1485,11 +1605,13 @@ class Opik:
                 - `id`, `name`: String fields
                 - `tags`: List field (use "contains" operator only)
                 - `created_by`: String field
+                - `template_structure`: String field ("string" or "chat")
 
                 Supported operators by column:
                 - `id`: =, !=, contains, not_contains, starts_with, ends_with, >, <
                 - `name`: =, !=, contains, not_contains, starts_with, ends_with, >, <
                 - `created_by`: =, !=, contains, not_contains, starts_with, ends_with, >, <
+                - `template_structure`: =, !=
                 - `tags`: contains (only)
 
                 Examples:
@@ -1498,23 +1620,36 @@ class Opik:
                 - `name contains "summary"` - Filter by name substring
                 - `created_by = "user@example.com"` - Filter by creator
                 - `id starts_with "prompt_"` - Filter by ID prefix
+                - `template_structure = "text"` - Only text prompts
+                - `template_structure = "chat"` - Only chat prompts
 
-                If not provided, all prompts matching the name filter will be returned.
+                If not provided, all prompts (both text and chat) will be returned.
 
         Returns:
-            List[Prompt]: A list of Prompt instances found.
-        """
-        parsed_filters = None
-        if filter_string:
-            oql = opik_query_language.OpikQueryLanguage(filter_string)
-            parsed_filters = oql.get_filter_expressions()
-
-        prompt_client = PromptClient(self._rest_client)
-        name_and_versions = prompt_client.search_prompts(parsed_filters=parsed_filters)
-        prompts: List[Prompt] = [
-            Prompt.from_fern_prompt_version(prompt_name, version)
-            for (prompt_name, version) in name_and_versions
-        ]
+            List[Union[Prompt, ChatPrompt]]: A list of Prompt and/or ChatPrompt instances found.
+        """
+        oql = opik_query_language.OpikQueryLanguage(filter_string or "")
+        parsed_filters = oql.get_filter_expressions()
+
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        search_results = prompt_client_.search_prompts(parsed_filters=parsed_filters)
+
+        # Convert to Prompt or ChatPrompt objects based on template_structure
+        prompts: List[Union[prompt_module.Prompt, prompt_module.ChatPrompt]] = []
+        for result in search_results:
+            if result.template_structure == "chat":
+                prompts.append(
+                    prompt_module.ChatPrompt.from_fern_prompt_version(
+                        result.name, result.prompt_version_detail
+                    )
+                )
+            else:
+                prompts.append(
+                    prompt_module.Prompt.from_fern_prompt_version(
+                        result.name, result.prompt_version_detail
+                    )
+                )
+
         return prompts
 
     def create_optimization(

{opik-1.9.33 → opik-1.9.38}/src/opik/api_objects/opik_query_language.py

@@ -29,6 +29,7 @@ COLUMNS = {
     "type": "string",
     "model": "string",
     "provider": "string",
+    "template_structure": "string",
 }
 
 SUPPORTED_OPERATORS = {
@@ -121,6 +122,14 @@ SUPPORTED_OPERATORS = {
         ">",
         "<",
     ],
+    "template_structure": [
+        "=",
+        "contains",
+        "not_contains",
+        "starts_with",
+        "ends_with",
+        "!=",
+    ],
 }
 
 

opik-1.9.38/src/opik/api_objects/prompt/__init__.py

@@ -0,0 +1,17 @@
+from .text.prompt import Prompt
+from .text.prompt_template import PromptTemplate
+from .chat.chat_prompt import ChatPrompt
+from .chat.chat_prompt_template import ChatPromptTemplate
+from .types import PromptType
+from .base_prompt import BasePrompt
+from .base_prompt_template import BasePromptTemplate
+
+__all__ = [
+    "PromptType",
+    "Prompt",
+    "ChatPrompt",
+    "PromptTemplate",
+    "ChatPromptTemplate",
+    "BasePrompt",
+    "BasePromptTemplate",
+]

opik-1.9.38/src/opik/api_objects/prompt/base_prompt.py

@@ -0,0 +1,69 @@
+"""
+Base class for prompts.
+
+Defines abstract interface that both string and chat prompt variants must implement.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any, Dict, Optional
+
+from . import types as prompt_types
+
+
+class BasePrompt(ABC):
+    """
+    Abstract base class for prompts (string and chat).
+
+    All prompt implementations must provide common properties and methods
+    for interacting with the backend API.
+    """
+
+    @property
+    @abstractmethod
+    def name(self) -> str:
+        """The name of the prompt."""
+        pass
+
+    @property
+    @abstractmethod
+    def commit(self) -> Optional[str]:
+        """The commit hash of the prompt version."""
+        pass
+
+    @property
+    @abstractmethod
+    def metadata(self) -> Optional[Dict[str, Any]]:
+        """The metadata dictionary associated with the prompt."""
+        pass
+
+    @property
+    @abstractmethod
+    def type(self) -> prompt_types.PromptType:
+        """The prompt type (MUSTACHE or JINJA2)."""
+        pass
+
+    # Internal API fields for backend synchronization
+    __internal_api__prompt_id__: str
+    __internal_api__version_id__: str
+
+    @abstractmethod
+    def format(self, *args: Any, **kwargs: Any) -> Any:
+        """
+        Format the prompt with the provided variables.
+
+        Returns:
+            Formatted output. Type depends on the implementation:
+            - Prompt returns str
+            - ChatPrompt returns List[Dict[str, MessageContent]]
+        """
+        pass
+
+    @abstractmethod
+    def __internal_api__to_info_dict__(self) -> Dict[str, Any]:
+        """
+        Convert the prompt to an info dictionary for serialization.
+
+        Returns:
+            Dictionary containing prompt metadata and version information.
+        """
+        pass

opik-1.9.38/src/opik/api_objects/prompt/base_prompt_template.py

@@ -0,0 +1,29 @@
+"""
+Base class for prompt templates.
+
+Defines abstract interface that both string and chat template variants must implement.
+"""
+
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BasePromptTemplate(ABC):
+    """
+    Abstract base class for prompt templates (string and chat).
+
+    All prompt template implementations must provide a format method
+    that takes variables and returns formatted output.
+    """
+
+    @abstractmethod
+    def format(self, *args: Any, **kwargs: Any) -> Any:
+        """
+        Format the template with the provided variables.
+
+        Returns:
+            Formatted output. Type depends on the implementation:
+            - PromptTemplate returns str
+            - ChatPromptTemplate returns List[Dict[str, MessageContent]]
+        """
+        pass

opik-1.9.38/src/opik/api_objects/prompt/chat/__init__.py

@@ -0,0 +1 @@
+# Empty - all exports handled by parent __init__.py