opik 1.9.5__py3-none-any.whl → 1.9.39__py3-none-any.whl

opik/__init__.py

@@ -6,18 +6,23 @@ from .api_objects.experiment.experiment_item import (
     ExperimentItemReferences,
 )
 from .api_objects.opik_client import Opik
-from .api_objects.prompt import Prompt
+from .api_objects.prompt import Prompt, ChatPrompt
 from .api_objects.prompt.types import PromptType
 from .api_objects.span import Span
 from .api_objects.trace import Trace
 from .configurator.configure import configure
 from .decorator.tracker import flush_tracker, track
-from .evaluation import evaluate, evaluate_experiment, evaluate_prompt
+from .evaluation import (
+    evaluate,
+    evaluate_experiment,
+    evaluate_on_dict_items,
+    evaluate_prompt,
+)
 from .integrations.sagemaker import auth as sagemaker_auth
 from .plugins.pytest.decorator import llm_unit
 from .types import LLMProvider
 from . import opik_context
-from .decorator.tracing_runtime_config import (
+from .tracing_runtime_config import (
     is_tracing_active,
     reset_tracing_to_config_default,
     set_tracing_active,
@@ -37,6 +42,7 @@ __all__ = [
     "evaluate",
     "evaluate_prompt",
     "evaluate_experiment",
+    "evaluate_on_dict_items",
     "ExperimentItemContent",
     "ExperimentItemReferences",
     "track",
@@ -49,6 +55,7 @@ __all__ = [
     "llm_unit",
     "configure",
     "Prompt",
+    "ChatPrompt",
     "PromptType",
     "LLMProvider",
     "reset_tracing_to_config_default",

opik/anonymizer/__init__.py

@@ -0,0 +1,5 @@
+from .anonymizer import Anonymizer
+from .factory import create_anonymizer
+from .recursive_anonymizer import RecursiveAnonymizer
+
+__all__ = ["Anonymizer", "create_anonymizer", "RecursiveAnonymizer"]

opik/anonymizer/anonymizer.py

@@ -0,0 +1,12 @@
+import abc
+from typing import Dict, Any, Union, List
+
+AnonymizerDataType = Union[Dict[str, Any], str, List[Any]]
+
+
+class Anonymizer(abc.ABC):
+    """Abstract base class for anonymizing sensitive data in various data structures."""
+
+    @abc.abstractmethod
+    def anonymize(self, data: AnonymizerDataType, **kwargs: Any) -> AnonymizerDataType:
+        pass

opik/anonymizer/factory.py

@@ -0,0 +1,80 @@
+from typing import Union, List, Dict, Callable, Tuple
+
+from . import anonymizer, rules_anonymizer, rules
+
+RulesType = Union[
+    List[Dict[str, str]],
+    List[Tuple[str, str]],
+    List[Callable[[str], str]],
+    List[Union[Dict[str, str], Tuple[str, str], Callable[[str], str]]],
+    Dict[str, str],
+    Tuple[str, str],
+    Callable[[str], str],
+]
+
+
+def create_anonymizer(
+    anonymizer_rules: RulesType, max_depth: int = 10
+) -> anonymizer.Anonymizer:
+    """Create an anonymizer with the specified rules.
+
+    Args:
+        anonymizer_rules: Anonymizer rules specification in various formats:
+            - Dict with "regex" and "replace" keys for a single regex rule
+            - Tuple with (regex, replacement) for a single regex rule
+            - Callable that takes a string and returns anonymized string
+            - List of any of the above for multiple rules
+        max_depth: Maximum recursion depth for nested data structures.
+
+    Returns:
+        An Anonymizer instance configured with the specified rules.
+
+    Raises:
+        ValueError: If a rule format is invalid.
+    """
+    rule_objects: List[rules.Rule] = []
+
+    if callable(anonymizer_rules):
+        # Single function rule
+        rule_objects.append(rules.FunctionRule(anonymizer_rules))
+    elif isinstance(anonymizer_rules, dict):
+        # Single dictionary rule
+        _check_dictionary_rule(anonymizer_rules)
+        rule_objects.append(
+            rules.RegexRule(anonymizer_rules["regex"], anonymizer_rules["replace"])
+        )
+    elif isinstance(anonymizer_rules, tuple):
+        # Single tuple rule
+        _check_tuple_rule(anonymizer_rules)
+        regex_pattern, replacement = anonymizer_rules
+        rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+    elif isinstance(anonymizer_rules, list):
+        # List of rules
+        for rule in anonymizer_rules:
+            if callable(rule) and not isinstance(rule, (dict, tuple)):
+                rule_objects.append(rules.FunctionRule(rule))
+            elif isinstance(rule, dict):
+                _check_dictionary_rule(rule)
+                rule_objects.append(rules.RegexRule(rule["regex"], rule["replace"]))
+            elif isinstance(rule, tuple):
+                _check_tuple_rule(rule)
+                regex_pattern, replacement = rule
+                rule_objects.append(rules.RegexRule(regex_pattern, replacement))
+            else:
+                raise ValueError(f"Unsupported rule type in list: {type(rule)}")
+    else:
+        raise ValueError(f"Unsupported rules type: {type(anonymizer_rules)}")
+
+    return rules_anonymizer.RulesAnonymizer(rule_objects, max_depth=max_depth)
+
+
+def _check_dictionary_rule(rule: Dict[str, str]) -> None:
+    if "regex" not in rule or "replace" not in rule:
+        raise ValueError("Dictionary rule must have 'regex' and 'replace' keys")
+
+
+def _check_tuple_rule(rule: Tuple[str, str]) -> None:
+    if len(rule) != 2:
+        raise ValueError(
+            "Tuple rule must have exactly 2 elements: (regex, replacement)"
+        )

opik/anonymizer/recursive_anonymizer.py

@@ -0,0 +1,64 @@
+import abc
+from typing import Any, Optional
+
+from . import anonymizer
+
+
+class RecursiveAnonymizer(anonymizer.Anonymizer):
+    """Abstract base class for anonymizing sensitive data in various data structures.
+
+    This class provides a framework for recursively anonymizing text content within
+    nested data structures such as dictionaries, lists, and strings. Subclasses must
+    implement the anonymize_text() method to define the specific anonymization logic.
+    """
+
+    def __init__(self, max_depth: int = 10):
+        """Initialize the Anonymizer with depth limiting.
+
+        Args:
+            max_depth: Maximum recursion depth to prevent infinite loops when
+                      processing deeply nested or circular data structures.
+                      Defaults to 10.
+        """
+        self.max_depth = max_depth
+
+    def anonymize(
+        self, data: anonymizer.AnonymizerDataType, **kwargs: Any
+    ) -> anonymizer.AnonymizerDataType:
+        return self._recursive_anonymize(data, **kwargs)
+
+    @abc.abstractmethod
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        pass
+
+    def _recursive_anonymize(
+        self,
+        data: anonymizer.AnonymizerDataType,
+        depth: int = 0,
+        field_name: Optional[str] = None,
+        **kwargs: Any,
+    ) -> anonymizer.AnonymizerDataType:
+        if depth >= self.max_depth:
+            return data
+
+        if field_name is None:
+            field_name = ""
+
+        if isinstance(data, str):
+            return self.anonymize_text(data, field_name=field_name, **kwargs)
+        elif isinstance(data, dict):
+            return {
+                key: self._recursive_anonymize(
+                    value, depth + 1, field_name=f"{field_name}.{key}", **kwargs
+                )
+                for key, value in data.items()
+            }
+        elif isinstance(data, list):
+            return [
+                self._recursive_anonymize(
+                    item, depth + 1, field_name=f"{field_name}.{i}", **kwargs
+                )
+                for i, item in enumerate(data)
+            ]
+        else:
+            return data

opik/anonymizer/rules.py

@@ -0,0 +1,56 @@
+import abc
+import re
+from typing import Callable
+
+
+class Rule(abc.ABC):
+    """Abstract base class for text anonymization rules.
+
+    Rules define specific patterns or conditions for anonymizing sensitive
+    information in text. Subclasses must implement the apply() method to
+    define the anonymization logic.
+    """
+
+    @abc.abstractmethod
+    def apply(self, text: str) -> str:
+        pass
+
+
+class RegexRule(Rule):
+    """A rule that uses regular expressions to find and replace patterns in text.
+
+    This rule compiles a regular expression pattern and applies it to input text,
+    replacing all matches with a specified replacement string.
+    """
+
+    def __init__(self, regex: str, replacement: str):
+        """Initialize the regex rule with a pattern and replacement.
+
+        Args:
+            regex: Regular expression pattern to match sensitive data.
+            replacement: String to replace matched patterns with.
+        """
+        self.pattern = re.compile(regex)
+        self.replacement = replacement
+
+    def apply(self, text: str) -> str:
+        return self.pattern.sub(self.replacement, text)
+
+
+class FunctionRule(Rule):
+    """A rule that applies a custom function to anonymize text.
+
+    This rule allows for flexible anonymization by accepting any callable
+    that takes a string as input and returns an anonymized string.
+    """
+
+    def __init__(self, func: Callable[[str], str]):
+        """Initialize the function rule with a custom anonymization function.
+
+        Args:
+            func: A callable that takes a string and returns an anonymized version.
+        """
+        self.func = func
+
+    def apply(self, text: str) -> str:
+        return self.func(text)

opik/anonymizer/rules_anonymizer.py

@@ -0,0 +1,35 @@
+from typing import List, Any
+
+from . import recursive_anonymizer, rules
+
+
+class RulesAnonymizer(recursive_anonymizer.RecursiveAnonymizer):
+    """An anonymizer that applies a list of rules sequentially to text data.
+
+    This class takes a list of Rule objects and applies them to
+    anonymize sensitive information in text.
+    """
+
+    def __init__(self, anonymizer_rules: List[rules.Rule], max_depth: int = 10):
+        """Initialize the RulesAnonymizer with a list of rules.
+
+        Args:
+            anonymizer_rules: List of Rule objects to apply for anonymization.
+            max_depth: Maximum recursion depth for nested data structures.
+        """
+        super().__init__(max_depth)
+        self.rules = anonymizer_rules
+
+    def anonymize_text(self, data: str, **kwargs: Any) -> str:
+        """Apply all rules sequentially to the input text.
+
+        Args:
+            data: The text to anonymize.
+
+        Returns:
+            The anonymized text after applying all rules.
+        """
+        result = data
+        for rule in self.rules:
+            result = rule.apply(result)
+        return result

opik/api_objects/dataset/rest_operations.py

@@ -1,9 +1,12 @@
+from __future__ import annotations
+
 from typing import List
 from opik.rest_api import OpikApi
 import opik.exceptions as exceptions
 from opik.message_processing import streamer
 from . import dataset
 from .. import experiment
+from ..experiment import experiments_client
 from ...rest_api.core.api_error import ApiError
 
 
@@ -60,6 +63,7 @@ def get_dataset_experiments(
     dataset_id: str,
     max_results: int,
     streamer: streamer.Streamer,
+    experiments_client: experiments_client.ExperimentsClient,
 ) -> List[experiment.Experiment]:
     page_size = 100
     experiments: List[experiment.Experiment] = []
@@ -83,6 +87,7 @@ def get_dataset_experiments(
                     dataset_name=experiment_.dataset_name,
                     rest_client=rest_client,
                     streamer=streamer,
+                    experiments_client=experiments_client,
                 )
             )
 

opik/api_objects/experiment/experiment.py

@@ -1,15 +1,17 @@
 import functools
 import logging
-from typing import List, Optional
+from typing import List, Optional, TYPE_CHECKING
 
-import opik.rest_api
 from opik.message_processing.batching import sequence_splitter
 from opik.message_processing import messages, streamer
 from opik.rest_api import client as rest_api_client
-from opik.rest_api.types import experiment_public
-from . import experiment_item
-from .. import constants, helpers, rest_stream_parser
-from ...api_objects.prompt import Prompt
+from opik.rest_api import types as rest_api_types
+from . import experiment_item, experiments_client
+from .. import constants, helpers
+from ...api_objects.prompt import base_prompt
+
+if TYPE_CHECKING:
+    from opik.evaluation.metrics import score_result
 
 LOGGER = logging.getLogger(__name__)
 
@@ -22,7 +24,8 @@ class Experiment:
         dataset_name: str,
         rest_client: rest_api_client.OpikApi,
         streamer: streamer.Streamer,
-        prompts: Optional[List[Prompt]] = None,
+        experiments_client: experiments_client.ExperimentsClient,
+        prompts: Optional[List[base_prompt.BasePrompt]] = None,
     ) -> None:
         self._id = id
         self._name = name
@@ -30,6 +33,7 @@ class Experiment:
         self._rest_client = rest_client
         self._prompts = prompts
         self._streamer = streamer
+        self._experiments_client = experiments_client
 
     @property
     def id(self) -> str:
@@ -59,7 +63,7 @@ class Experiment:
     def experiments_rest_client(self) -> rest_api_client.ExperimentsClient:
         return self._rest_client.experiments
 
-    def get_experiment_data(self) -> experiment_public.ExperimentPublic:
+    def get_experiment_data(self) -> rest_api_types.experiment_public.ExperimentPublic:
         return self._rest_client.experiments.get_experiment_by_id(id=self.id)
 
     def insert(
@@ -100,55 +104,48 @@ class Experiment:
 
     def get_items(
         self,
-        max_results: Optional[int] = None,
+        max_results: Optional[int] = 10000,
         truncate: bool = False,
     ) -> List[experiment_item.ExperimentItemContent]:
         """
-        Retrieves and returns a list of experiment items by streaming from the backend in batches, with an option to
-        truncate the results for each batch.
-
-        This method streams experiment items from a backend service in chunks up to the specified `max_results`
-        or until the available items are exhausted. It handles batch-wise retrieval and parsing, ensuring the client
-        receives a list of `ExperimentItemContent` objects, while respecting the constraints on maximum retrieval size
-        from the backend. If truncation is enabled, the backend may return truncated details for each item.
+        Retrieves and returns a list of experiment items for this experiment.
 
         Args:
-            max_results: Maximum number of experiment items to retrieve.
-            truncate: Whether to truncate the items returned by the backend.
+            max_results: Maximum number of experiment items to retrieve. Defaults to 10000 if not specified.
+            truncate: Whether to truncate the items returned by the backend. Defaults to False.
 
+        Returns:
+            List of ExperimentItemContent objects for this experiment.
         """
-        result: List[experiment_item.ExperimentItemContent] = []
-        max_endpoint_batch_size = rest_stream_parser.MAX_ENDPOINT_BATCH_SIZE
-
-        while True:
-            if max_results is None:
-                current_batch_size = max_endpoint_batch_size
-            else:
-                current_batch_size = min(
-                    max_results - len(result), max_endpoint_batch_size
-                )
-
-            items_stream = self._rest_client.experiments.stream_experiment_items(
-                experiment_name=self.name,
-                limit=current_batch_size,
-                last_retrieved_id=result[-1].id if len(result) > 0 else None,
-                truncate=truncate,
-            )
+        if max_results is None:
+            max_results = 10000  # TODO: remove this once we have a proper way to get all experiment items
+
+        return self._experiments_client.find_experiment_items_for_dataset(
+            dataset_name=self.dataset_name,
+            experiment_ids=[self.id],
+            truncate=truncate,
+            max_results=max_results,
+        )
 
-            experiment_item_compare_current_batch = (
-                rest_stream_parser.read_and_parse_stream(
-                    stream=items_stream,
-                    item_class=opik.rest_api.ExperimentItemCompare,
-                )
-            )
+    def log_experiment_scores(
+        self,
+        score_results: List["score_result.ScoreResult"],
+    ) -> None:
+        """Log experiment-level scores to the backend."""
+        experiment_scores: List[rest_api_types.ExperimentScore] = []
 
-            for item in experiment_item_compare_current_batch:
-                converted_item = experiment_item.ExperimentItemContent.from_rest_experiment_item_compare(
-                    value=item
-                )
-                result.append(converted_item)
+        for score_result_ in score_results:
+            if score_result_.scoring_failed:
+                continue
 
-            if current_batch_size > len(experiment_item_compare_current_batch):
-                break
+            experiment_score = rest_api_types.ExperimentScore(
+                name=score_result_.name,
+                value=score_result_.value,
+            )
+            experiment_scores.append(experiment_score)
 
-        return result
+        if experiment_scores:
+            self._rest_client.experiments.update_experiment(
+                id=self.id,
+                experiment_scores=experiment_scores,
+            )

opik/api_objects/experiment/helpers.py

@@ -1,8 +1,12 @@
+import copy
 import logging
-import opik.jsonable_encoder as jsonable_encoder
 from typing import Any, Dict, List, Mapping, Optional, Tuple
-import copy
-from .. import prompt
+
+from opik import id_helpers
+import opik.jsonable_encoder as jsonable_encoder
+
+from ..prompt import base_prompt
+
 
 LOGGER = logging.getLogger(__name__)
 
@@ -11,7 +15,7 @@ PromptVersion = Dict[str, str]
 
 def build_metadata_and_prompt_versions(
     experiment_config: Optional[Dict[str, Any]],
-    prompts: Optional[List[prompt.Prompt]],
+    prompts: Optional[List[base_prompt.BasePrompt]],
 ) -> Tuple[Optional[Dict[str, Any]], Optional[List[PromptVersion]]]:
     prompt_versions: Optional[List[PromptVersion]] = None
 
@@ -37,9 +41,20 @@ def build_metadata_and_prompt_versions(
         prompt_versions = []
         experiment_config["prompts"] = {}
 
-        for prompt in prompts:
-            prompt_versions.append({"id": prompt.__internal_api__version_id__})
-            experiment_config["prompts"][prompt.name] = prompt.prompt
+        for prompt_obj in prompts:
+            prompt_versions.append({"id": prompt_obj.__internal_api__version_id__})
+            # Use __internal_api__to_info_dict__() to get the prompt content in a consistent way
+            prompt_info = prompt_obj.__internal_api__to_info_dict__()
+            # Extract the template/messages from the version dict
+            if "version" in prompt_info:
+                if "template" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["template"]
+                elif "messages" in prompt_info["version"]:
+                    experiment_config["prompts"][prompt_obj.name] = prompt_info[
+                        "version"
+                    ]["messages"]
 
     if experiment_config == {}:
         return None, None
@@ -50,9 +65,9 @@ def build_metadata_and_prompt_versions(
 
 
 def handle_prompt_args(
-    prompt: Optional[prompt.Prompt] = None,
-    prompts: Optional[List[prompt.Prompt]] = None,
-) -> Optional[List[prompt.Prompt]]:
+    prompt: Optional[base_prompt.BasePrompt] = None,
+    prompts: Optional[List[base_prompt.BasePrompt]] = None,
+) -> Optional[List[base_prompt.BasePrompt]]:
     if prompts is not None and len(prompts) > 0 and prompt is not None:
         LOGGER.warning(
             "Arguments `prompt` and `prompts` are mutually exclusive, `prompts` will be used`."
@@ -63,3 +78,12 @@ def handle_prompt_args(
         prompts = None
 
     return prompts
+
+
+def generate_unique_experiment_name(experiment_name_prefix: Optional[str]) -> str:
+    if experiment_name_prefix is None:
+        return id_helpers.generate_random_alphanumeric_string(12)
+
+    return (
+        f"{experiment_name_prefix}-{id_helpers.generate_random_alphanumeric_string(6)}"
+    )

opik/api_objects/local_recording.py

@@ -1,6 +1,6 @@
 import contextlib
 from typing import Iterator, List
-
+from typing import Optional
 from . import opik_client
 from ..message_processing import message_processors_chain
 from ..message_processing.emulation import local_emulator_message_processor, models
@@ -33,9 +33,13 @@ class _LocalRecordingHandle:
 
 
 @contextlib.contextmanager
-def record_traces_locally() -> Iterator[_LocalRecordingHandle]:
+def record_traces_locally(
+    client: Optional[opik_client.Opik] = None,
+) -> Iterator[_LocalRecordingHandle]:
     """Enable local recording of traces/spans within the context.
 
+    Args:
+        client: Optional Opik client to use for recording. If not provided, the default session client will be used.
     Usage:
         with opik.record_traces_locally() as storage:
             # your code that creates traces/spans
@@ -44,7 +48,8 @@ def record_traces_locally() -> Iterator[_LocalRecordingHandle]:
     Yields a handle with `span_trees` and `trace_trees` properties that flush
     the client before reading, ensuring all events are captured.
     """
-    client = opik_client.get_client_cached()
+    if client is None:
+        client = opik_client.get_client_cached()
 
     # Disallow nested/local concurrent recordings in the same process
     existing_local = message_processors_chain.get_local_emulator_message_processor(