opik 1.9.26__py3-none-any.whl → 1.9.41__py3-none-any.whl

opik/__init__.py

@@ -6,18 +6,23 @@ from .api_objects.experiment.experiment_item import (
     ExperimentItemReferences,
 )
 from .api_objects.opik_client import Opik
-from .api_objects.prompt import Prompt
+from .api_objects.prompt import Prompt, ChatPrompt
 from .api_objects.prompt.types import PromptType
 from .api_objects.span import Span
 from .api_objects.trace import Trace
 from .configurator.configure import configure
 from .decorator.tracker import flush_tracker, track
-from .evaluation import evaluate, evaluate_experiment, evaluate_prompt
+from .evaluation import (
+    evaluate,
+    evaluate_experiment,
+    evaluate_on_dict_items,
+    evaluate_prompt,
+)
 from .integrations.sagemaker import auth as sagemaker_auth
 from .plugins.pytest.decorator import llm_unit
 from .types import LLMProvider
 from . import opik_context
-from .decorator.tracing_runtime_config import (
+from .tracing_runtime_config import (
     is_tracing_active,
     reset_tracing_to_config_default,
     set_tracing_active,
@@ -37,6 +42,7 @@ __all__ = [
     "evaluate",
     "evaluate_prompt",
     "evaluate_experiment",
+    "evaluate_on_dict_items",
     "ExperimentItemContent",
     "ExperimentItemReferences",
     "track",
@@ -49,6 +55,7 @@ __all__ = [
     "llm_unit",
     "configure",
     "Prompt",
+    "ChatPrompt",
     "PromptType",
     "LLMProvider",
     "reset_tracing_to_config_default",

opik/api_objects/dataset/rest_operations.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from typing import List
 from opik.rest_api import OpikApi
 import opik.exceptions as exceptions

opik/api_objects/experiment/experiment.py

@@ -1,14 +1,17 @@
 import functools
 import logging
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 
 from opik.message_processing.batching import sequence_splitter
 from opik.message_processing import messages, streamer
 from opik.rest_api import client as rest_api_client
-from opik.rest_api.types import experiment_public
+from opik.rest_api import types as rest_api_types
 from . import experiment_item, experiments_client
 from .. import constants, helpers
-from ...api_objects.prompt import Prompt
+from ...api_objects.prompt import base_prompt
+
+if TYPE_CHECKING:
+    from opik.evaluation.metrics import score_result
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +25,7 @@ class Experiment:
         rest_client: rest_api_client.OpikApi,
         streamer: streamer.Streamer,
         experiments_client: experiments_client.ExperimentsClient,
-        prompts: Optional[List[Prompt]] = None,
+        prompts: Optional[List[base_prompt.BasePrompt]] = None,
     ) -> None:
         self._id = id
         self._name = name
@@ -60,7 +63,7 @@ class Experiment:
     def experiments_rest_client(self) -> rest_api_client.ExperimentsClient:
         return self._rest_client.experiments
 
-    def get_experiment_data(self) -> experiment_public.ExperimentPublic:
+    def get_experiment_data(self) -> rest_api_types.experiment_public.ExperimentPublic:
         return self._rest_client.experiments.get_experiment_by_id(id=self.id)
 
     def insert(
@@ -123,3 +126,26 @@ class Experiment:
             truncate=truncate,
             max_results=max_results,
         )
+
+    def log_experiment_scores(
+        self,
+        score_results: List["score_result.ScoreResult"],
+    ) -> None:
+        """Log experiment-level scores to the backend."""
+        experiment_scores: List[rest_api_types.ExperimentScore] = []
+
+        for score_result_ in score_results:
+            if score_result_.scoring_failed:
+                continue
+
+            experiment_score = rest_api_types.ExperimentScore(
+                name=score_result_.name,
+                value=score_result_.value,
+            )
+            experiment_scores.append(experiment_score)
+
+        if experiment_scores:
+            self._rest_client.experiments.update_experiment(
+                id=self.id,
+                experiment_scores=experiment_scores,
+            )

opik/api_objects/experiment/helpers.py

@@ -1,8 +1,12 @@
+import copy
 import logging
-import opik.jsonable_encoder as jsonable_encoder
 from typing import Any, Dict, List, Mapping, Optional, Tuple
-import copy
-from .. import prompt
+
+from opik import id_helpers
+import opik.jsonable_encoder as jsonable_encoder
+
+from ..prompt import base_prompt
+
 
 LOGGER = logging.getLogger(__name__)
 
@@ -11,7 +15,7 @@ PromptVersion = Dict[str, str]
 
 def build_metadata_and_prompt_versions(
     experiment_config: Optional[Dict[str, Any]],
-    prompts: Optional[List[prompt.Prompt]],
+    prompts: Optional[List[base_prompt.BasePrompt]],
 ) -> Tuple[Optional[Dict[str, Any]], Optional[List[PromptVersion]]]:
     prompt_versions: Optional[List[PromptVersion]] = None
 
@@ -37,9 +41,20 @@ def build_metadata_and_prompt_versions(
         prompt_versions = []
         experiment_config["prompts"] = {}
 
-        for prompt in prompts:
-            prompt_versions.append({"id": prompt.__internal_api__version_id__})
-            experiment_config["prompts"][prompt.name] = prompt.prompt
+        for prompt_obj in prompts:
+            prompt_versions.append({"id": prompt_obj.__internal_api__version_id__})
+            # Use __internal_api__to_info_dict__() to get the prompt content in a consistent way
+            prompt_info = prompt_obj.__internal_api__to_info_dict__()
+            # Extract the template/messages from the version dict
+            if "version" in prompt_info:
+                if "template" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["template"]
+                elif "messages" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["messages"]
 
     if experiment_config == {}:
         return None, None
@@ -50,9 +65,9 @@ def build_metadata_and_prompt_versions(
 
 
 def handle_prompt_args(
-    prompt: Optional[prompt.Prompt] = None,
-    prompts: Optional[List[prompt.Prompt]] = None,
-) -> Optional[List[prompt.Prompt]]:
+    prompt: Optional[base_prompt.BasePrompt] = None,
+    prompts: Optional[List[base_prompt.BasePrompt]] = None,
+) -> Optional[List[base_prompt.BasePrompt]]:
     if prompts is not None and len(prompts) > 0 and prompt is not None:
         LOGGER.warning(
             "Arguments `prompt` and `prompts` are mutually exclusive, `prompts` will be used`."
@@ -63,3 +78,12 @@ def handle_prompt_args(
         prompts = None
 
     return prompts
+
+
+def generate_unique_experiment_name(experiment_name_prefix: Optional[str]) -> str:
+    if experiment_name_prefix is None:
+        return id_helpers.generate_random_alphanumeric_string(12)
+
+    return (
+        f"{experiment_name_prefix}-{id_helpers.generate_random_alphanumeric_string(6)}"
+    )

opik/api_objects/local_recording.py

@@ -1,6 +1,6 @@
 import contextlib
 from typing import Iterator, List
-
+from typing import Optional
 from . import opik_client
 from ..message_processing import message_processors_chain
 from ..message_processing.emulation import local_emulator_message_processor, models
@@ -33,9 +33,13 @@ class _LocalRecordingHandle:
 
 
 @contextlib.contextmanager
-def record_traces_locally() -> Iterator[_LocalRecordingHandle]:
+def record_traces_locally(
+    client: Optional[opik_client.Opik] = None,
+) -> Iterator[_LocalRecordingHandle]:
     """Enable local recording of traces/spans within the context.
 
+    Args:
+        client: Optional Opik client to use for recording. If not provided, the default session client will be used.
     Usage:
         with opik.record_traces_locally() as storage:
             # your code that creates traces/spans
@@ -44,7 +48,8 @@ def record_traces_locally() -> Iterator[_LocalRecordingHandle]:
     Yields a handle with `span_trees` and `trace_trees` properties that flush
     the client before reading, ensuring all events are captured.
     """
-    client = opik_client.get_client_cached()
+    if client is None:
+        client = opik_client.get_client_cached()
 
     # Disallow nested/local concurrent recordings in the same process
     existing_local = message_processors_chain.get_local_emulator_message_processor(

opik/api_objects/opik_client.py

@@ -24,8 +24,8 @@ from .dataset import rest_operations as dataset_rest_operations
 from .experiment import experiments_client
 from .experiment import helpers as experiment_helpers
 from .experiment import rest_operations as experiment_rest_operations
-from .prompt import Prompt, PromptType
-from .prompt.client import PromptClient
+from . import prompt as prompt_module
+from .prompt import client as prompt_client
 from .threads import threads_client
 from .trace import migration as trace_migration, trace_client
 from .. import (
@@ -917,8 +917,8 @@ class Opik:
         dataset_name: str,
         name: Optional[str] = None,
         experiment_config: Optional[Dict[str, Any]] = None,
-        prompt: Optional[Prompt] = None,
-        prompts: Optional[List[Prompt]] = None,
+        prompt: Optional[prompt_module.base_prompt.BasePrompt] = None,
+        prompts: Optional[List[prompt_module.base_prompt.BasePrompt]] = None,
         type: Literal["regular", "trial", "mini-batch"] = "regular",
         optimization_id: Optional[str] = None,
     ) -> experiment.Experiment:
@@ -972,6 +972,42 @@ class Opik:
 
         return experiment_
 
+    def update_experiment(
+        self,
+        id: str,
+        name: Optional[str] = None,
+        experiment_config: Optional[Dict[str, Any]] = None,
+    ) -> None:
+        """
+        Update an experiment's name and/or configuration.
+
+        Args:
+            id: The experiment ID.
+            name: The new name for the experiment. If None, the name will not be updated.
+            experiment_config: The new configuration for the experiment. If None, the configuration will not be updated.
+
+        Raises:
+            ValueError: if id is None or empty, or if both name and experiment_config are None
+        """
+        if not id:
+            raise ValueError(
+                f"id must be provided and can not be None or empty, id: {id}"
+            )
+
+        if name is None and experiment_config is None:
+            raise ValueError(
+                "At least one of 'name' or 'experiment_config' must be provided"
+            )
+
+        # Only include parameters that are provided to avoid clearing fields
+        request_params: Dict[str, Any] = {}
+        if name is not None:
+            request_params["name"] = name
+        if experiment_config is not None:
+            request_params["metadata"] = experiment_config
+
+        self._rest_client.experiments.update_experiment(id, **request_params)
+
     def get_experiment_by_name(self, name: str) -> experiment.Experiment:
         """
         Returns an existing experiment by its name.
@@ -991,7 +1027,7 @@ class Opik:
 
         return experiment.Experiment(
             id=experiment_public.id,
-            name=name,
+            name=experiment_public.name,
             dataset_name=experiment_public.dataset_name,
             rest_client=self._rest_client,
             streamer=self._streamer,
@@ -1000,10 +1036,11 @@ class Opik:
 
     def get_experiments_by_name(self, name: str) -> List[experiment.Experiment]:
         """
-        Returns a list of existing experiments by its name.
+        Returns a list of existing experiments containing the given string in their name.
+        Search is case-insensitive.
 
         Args:
-            name: The name of the experiment(s).
+            name: The string to search for in the experiment names.
 
         Returns:
             List[experiment.Experiment]: List of existing experiments.
@@ -1016,7 +1053,7 @@ class Opik:
         for public_experiment in experiments_public:
             experiment_ = experiment.Experiment(
                 id=public_experiment.id,
-                name=name,
+                name=public_experiment.name,
                 dataset_name=public_experiment.dataset_name,
                 rest_client=self._rest_client,
                 streamer=self._streamer,
@@ -1357,70 +1394,188 @@ class Opik:
         name: str,
         prompt: str,
         metadata: Optional[Dict[str, Any]] = None,
-        type: PromptType = PromptType.MUSTACHE,
-    ) -> Prompt:
+        type: prompt_module.PromptType = prompt_module.PromptType.MUSTACHE,
+    ) -> prompt_module.Prompt:
         """
-        Creates a new prompt with the given name and template.
-        If a prompt with the same name already exists, it will create a new version of the existing prompt if the templates differ.
+        Creates a new text prompt with the given name and template.
+        If a text prompt with the same name already exists, it will create a new version of the existing prompt if the templates differ.
 
         Parameters:
             name: The name of the prompt.
             prompt: The template content of the prompt.
             metadata: Optional metadata to be included in the prompt.
+            type: The template type (MUSTACHE or JINJA2).
 
         Returns:
             A Prompt object containing details of the created or retrieved prompt.
 
         Raises:
-            ApiError: If there is an error during the creation of the prompt and the status code is not 409.
+            PromptTemplateStructureMismatch: If a chat prompt with the same name already exists (template structure is immutable).
+            ApiError: If there is an error during the creation of the prompt.
         """
-        prompt_client = PromptClient(self._rest_client)
-        prompt_version = prompt_client.create_prompt(
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        prompt_version = prompt_client_.create_prompt(
             name=name, prompt=prompt, metadata=metadata, type=type
         )
-        return Prompt.from_fern_prompt_version(name, prompt_version)
+        return prompt_module.Prompt.from_fern_prompt_version(name, prompt_version)
+
+    def create_chat_prompt(
+        self,
+        name: str,
+        messages: List[Dict[str, Any]],
+        metadata: Optional[Dict[str, Any]] = None,
+        type: prompt_module.PromptType = prompt_module.PromptType.MUSTACHE,
+    ) -> prompt_module.ChatPrompt:
+        """
+        Creates a new chat prompt with the given name and message templates.
+        If a chat prompt with the same name already exists, it will create a new version if the messages differ.
+
+        Parameters:
+            name: The name of the chat prompt.
+            messages: List of message dictionaries with 'role' and 'content' fields.
+            metadata: Optional metadata to be included in the prompt.
+            type: The template type (MUSTACHE or JINJA2).
+
+        Returns:
+            A ChatPrompt object containing details of the created or retrieved chat prompt.
+
+        Raises:
+            PromptTemplateStructureMismatch: If a text prompt with the same name already exists (template structure is immutable).
+            ApiError: If there is an error during the creation of the prompt.
+        """
+        return prompt_module.ChatPrompt(
+            name=name, messages=messages, metadata=metadata, type=type
+        )
 
     def get_prompt(
         self,
         name: str,
         commit: Optional[str] = None,
-    ) -> Optional[Prompt]:
+    ) -> Optional[prompt_module.Prompt]:
         """
-        Retrieve the prompt detail for a given prompt name and commit version.
+        Retrieve a text prompt by name and optional commit version.
+
+        This method only returns text prompts.
 
         Parameters:
             name: The name of the prompt.
             commit: An optional commit version of the prompt. If not provided, the latest version is retrieved.
 
         Returns:
-            Prompt: The details of the specified prompt.
+            Prompt: The details of the specified text prompt, or None if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a chat prompt (template structure mismatch).
         """
-        prompt_client = PromptClient(self._rest_client)
-        fern_prompt_version = prompt_client.get_prompt(name=name, commit=commit)
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        fern_prompt_version = prompt_client_.get_prompt(
+            name=name, commit=commit, raise_if_not_template_structure="text"
+        )
+
         if fern_prompt_version is None:
             return None
 
-        return Prompt.from_fern_prompt_version(name, fern_prompt_version)
+        return prompt_module.Prompt.from_fern_prompt_version(name, fern_prompt_version)
 
-    def get_prompt_history(self, name: str) -> List[Prompt]:
+    def get_chat_prompt(
+        self,
+        name: str,
+        commit: Optional[str] = None,
+    ) -> Optional[prompt_module.ChatPrompt]:
         """
-        Retrieve all the prompt versions history for a given prompt name.
+        Retrieve a chat prompt by name and optional commit version.
+
+        This method only returns chat prompts.
+
+        Parameters:
+            name: The name of the prompt.
+            commit: An optional commit version of the prompt. If not provided, the latest version is retrieved.
+
+        Returns:
+            ChatPrompt: The details of the specified chat prompt, or None if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a text prompt (template structure mismatch).
+        """
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        fern_prompt_version = prompt_client_.get_prompt(
+            name=name, commit=commit, raise_if_not_template_structure="chat"
+        )
+
+        if fern_prompt_version is None:
+            return None
+
+        return prompt_module.ChatPrompt.from_fern_prompt_version(
+            name, fern_prompt_version
+        )
+
+    def get_prompt_history(self, name: str) -> List[prompt_module.Prompt]:
+        """
+        Retrieve all text prompt versions history for a given prompt name.
 
         Parameters:
             name: The name of the prompt.
 
         Returns:
-            List[Prompt]: A list of Prompt instances for the given name.
+            List[Prompt]: A list of text Prompt instances for the given name, or an empty list if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a chat prompt (template structure mismatch).
         """
-        prompt_client = PromptClient(self._rest_client)
-        fern_prompt_versions = prompt_client.get_all_prompt_versions(name=name)
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+
+        # First, validate that this is a text prompt by trying to get the latest version
+        # Let PromptTemplateStructureMismatch exception propagate - this is a hard error
+        latest_version = prompt_client_.get_prompt(
+            name=name, raise_if_not_template_structure="text"
+        )
+
+        if latest_version is None:
+            return []
+
+        # Now get all versions (we know it's a text prompt)
+        fern_prompt_versions = prompt_client_.get_all_prompt_versions(name=name)
+
         result = [
-            Prompt.from_fern_prompt_version(name, version)
+            prompt_module.Prompt.from_fern_prompt_version(name, version)
             for version in fern_prompt_versions
         ]
         return result
 
-    def get_all_prompts(self, name: str) -> List[Prompt]:
+    def get_chat_prompt_history(self, name: str) -> List[prompt_module.ChatPrompt]:
+        """
+        Retrieve all chat prompt versions history for a given prompt name.
+
+        Parameters:
+            name: The name of the prompt.
+
+        Returns:
+            List[ChatPrompt]: A list of ChatPrompt instances for the given name, or an empty list if not found.
+
+        Raises:
+            PromptTemplateStructureMismatch: If the prompt exists but is a text prompt (template structure mismatch).
+        """
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+
+        # First, validate that this is a chat prompt by trying to get the latest version
+        # Let PromptTemplateStructureMismatch exception propagate - this is a hard error
+        latest_version = prompt_client_.get_prompt(
+            name=name, raise_if_not_template_structure="chat"
+        )
+
+        if latest_version is None:
+            return []
+
+        # Now get all versions (we know it's a chat prompt)
+        fern_prompt_versions = prompt_client_.get_all_prompt_versions(name=name)
+
+        result = [
+            prompt_module.ChatPrompt.from_fern_prompt_version(name, version)
+            for version in fern_prompt_versions
+        ]
+        return result
+
+    def get_all_prompts(self, name: str) -> List[prompt_module.Prompt]:
         """
         DEPRECATED: Please use Opik.get_prompt_history() instead.
         Retrieve all the prompt versions history for a given prompt name.
@@ -1429,16 +1584,18 @@ class Opik:
             name: The name of the prompt.
 
         Returns:
-            List[Prompt]: A list of Prompt instances for the given name.
+            List[prompt_module.Prompt]: A list of Prompt instances for the given name.
         """
         LOGGER.warning(
             "Opik.get_all_prompts() is deprecated. Please use Opik.get_prompt_history() instead."
         )
         return self.get_prompt_history(name)
 
-    def search_prompts(self, filter_string: Optional[str] = None) -> List[Prompt]:
+    def search_prompts(
+        self, filter_string: Optional[str] = None
+    ) -> List[Union[prompt_module.Prompt, prompt_module.ChatPrompt]]:
         """
-        Retrieve the latest prompt versions for the given search parameters.
+        Retrieve the latest prompt versions (both string and chat prompts) for the given search parameters.
 
         Parameters:
             filter_string: A filter string to narrow down the search using Opik Query Language (OQL).
@@ -1448,11 +1605,13 @@ class Opik:
                 - `id`, `name`: String fields
                 - `tags`: List field (use "contains" operator only)
                 - `created_by`: String field
+                - `template_structure`: String field ("string" or "chat")
 
                 Supported operators by column:
                 - `id`: =, !=, contains, not_contains, starts_with, ends_with, >, <
                 - `name`: =, !=, contains, not_contains, starts_with, ends_with, >, <
                 - `created_by`: =, !=, contains, not_contains, starts_with, ends_with, >, <
+                - `template_structure`: =, !=
                 - `tags`: contains (only)
 
                 Examples:
@@ -1461,23 +1620,36 @@ class Opik:
                 - `name contains "summary"` - Filter by name substring
                 - `created_by = "user@example.com"` - Filter by creator
                 - `id starts_with "prompt_"` - Filter by ID prefix
+                - `template_structure = "text"` - Only text prompts
+                - `template_structure = "chat"` - Only chat prompts
 
-                If not provided, all prompts matching the name filter will be returned.
+                If not provided, all prompts (both text and chat) will be returned.
 
         Returns:
-            List[Prompt]: A list of Prompt instances found.
-        """
-        parsed_filters = None
-        if filter_string:
-            oql = opik_query_language.OpikQueryLanguage(filter_string)
-            parsed_filters = oql.get_filter_expressions()
-
-        prompt_client = PromptClient(self._rest_client)
-        name_and_versions = prompt_client.search_prompts(parsed_filters=parsed_filters)
-        prompts: List[Prompt] = [
-            Prompt.from_fern_prompt_version(prompt_name, version)
-            for (prompt_name, version) in name_and_versions
-        ]
+            List[Union[Prompt, ChatPrompt]]: A list of Prompt and/or ChatPrompt instances found.
+        """
+        oql = opik_query_language.OpikQueryLanguage(filter_string or "")
+        parsed_filters = oql.get_filter_expressions()
+
+        prompt_client_ = prompt_client.PromptClient(self._rest_client)
+        search_results = prompt_client_.search_prompts(parsed_filters=parsed_filters)
+
+        # Convert to Prompt or ChatPrompt objects based on template_structure
+        prompts: List[Union[prompt_module.Prompt, prompt_module.ChatPrompt]] = []
+        for result in search_results:
+            if result.template_structure == "chat":
+                prompts.append(
+                    prompt_module.ChatPrompt.from_fern_prompt_version(
+                        result.name, result.prompt_version_detail
+                    )
+                )
+            else:
+                prompts.append(
+                    prompt_module.Prompt.from_fern_prompt_version(
+                        result.name, result.prompt_version_detail
+                    )
+                )
+
         return prompts
 
     def create_optimization(

opik/api_objects/opik_query_language.py

@@ -29,6 +29,7 @@ COLUMNS = {
     "type": "string",
     "model": "string",
     "provider": "string",
+    "template_structure": "string",
 }
 
 SUPPORTED_OPERATORS = {
@@ -121,6 +122,14 @@ SUPPORTED_OPERATORS = {
         ">",
         "<",
     ],
+    "template_structure": [
+        "=",
+        "contains",
+        "not_contains",
+        "starts_with",
+        "ends_with",
+        "!=",
+    ],
 }
 
 

opik/api_objects/prompt/__init__.py

@@ -1,9 +1,17 @@
-from .prompt import Prompt
+from .text.prompt import Prompt
+from .text.prompt_template import PromptTemplate
+from .chat.chat_prompt import ChatPrompt
+from .chat.chat_prompt_template import ChatPromptTemplate
 from .types import PromptType
-from .chat_prompt_template import ChatPromptTemplate
+from .base_prompt import BasePrompt
+from .base_prompt_template import BasePromptTemplate
 
 __all__ = [
-    "Prompt",
     "PromptType",
+    "Prompt",
+    "ChatPrompt",
+    "PromptTemplate",
     "ChatPromptTemplate",
+    "BasePrompt",
+    "BasePromptTemplate",
 ]