opik 1.6.4__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/models/model_capabilities.py

@@ -0,0 +1,187 @@
+"""
+Capability registry for evaluation models.
+
+The registry is designed to grow beyond vision support (e.g. audio in the future).
+"""
+
+from __future__ import annotations
+
+from typing import Callable, Dict, Iterable, Optional, Set
+
+CapabilityDetector = Callable[[str], bool]
+
+
+VISION_MODEL_PREFIXES: Set[str] = {
+    # OpenAI
+    "gpt-4-vision",
+    "gpt-4o",
+    "gpt-4o-mini",
+    "gpt-4-turbo",
+    "chatgpt-4o-latest",
+    "gpt-5-mini",
+    "gpt-4.1",
+    "gpt-4.1-mini",
+    "gpt-4.1-nano",
+    "gpt-4.1-preview",
+    # Anthropic
+    "claude-3",
+    "claude-3-5",
+    # Google
+    "gemini-1.5-pro",
+    "gemini-1.5-flash",
+    "gemini-pro-vision",
+    "gemini-2.0-flash",
+    # Meta
+    "llama-3.2-11b-vision",
+    "llama-3.2-90b-vision",
+    # Mistral
+    "pixtral",
+    # Misc
+    "qwen-vl",
+    "qwen2-vl",
+    "phi-3-vision",
+    "phi-3.5-vision",
+    "llava",
+    "cogvlm",
+    "yi-vl",
+}
+VISION_MODEL_PREFIXES = {prefix.lower() for prefix in VISION_MODEL_PREFIXES}
+VISION_MODEL_SUFFIXES: Set[str] = {"-vision", "-vl"}
+
+
+def _strip_provider_prefix(model_name: str) -> str:
+    if "/" not in model_name:
+        return model_name
+    _, suffix = model_name.split("/", 1)
+    return suffix
+
+
+def _litellm_supports_vision(model_name: str) -> bool:
+    try:
+        import litellm  # type: ignore
+
+        return litellm.supports_vision(model=model_name)
+    except Exception:
+        return False
+
+
+def vision_capability_detector(model_name: str) -> bool:
+    stripped = _strip_provider_prefix(model_name)
+    candidates = {model_name, stripped}
+    for candidate in candidates:
+        if _litellm_supports_vision(candidate):
+            return True
+        normalized = candidate.lower()
+        if any(normalized.startswith(prefix) for prefix in VISION_MODEL_PREFIXES):
+            return True
+        if any(normalized.endswith(suffix) for suffix in VISION_MODEL_SUFFIXES):
+            return True
+    return False
+
+
+def video_capability_detector(model_name: str) -> bool:
+    """
+    Heuristically determine whether a model accepts video inputs.
+
+    Providers rarely expose structured metadata for video support, so we fall back
+    to naming conventions (e.g. models whose names contain ``video`` or ``qwen``
+    + ``vl``). When those heuristics fail we delegate to the vision detector since
+    current SDK integrations treat video as an extension of multimodal/vision APIs.
+    """
+    stripped = _strip_provider_prefix(model_name)
+    candidates = {model_name, stripped}
+    for candidate in candidates:
+        normalized = candidate.lower()
+        if "video" in normalized:
+            return True
+        if "qwen" in normalized and "vl" in normalized:
+            return True
+    # TODO(opik): litellm/model metadata still treats video + image inputs the same.
+    # Fall back to the vision heuristic so we can keep this dedicated capability
+    # and tighten detection once providers expose richer metadata.
+    return vision_capability_detector(model_name)
+
+
+class ModelCapabilitiesRegistry:
+    """
+    Central registry for model capability detection.
+    """
+
+    def __init__(self) -> None:
+        self._capability_detectors: Dict[str, CapabilityDetector] = {}
+
+    def register_capability_detector(
+        self, capability: str, detector: CapabilityDetector
+    ) -> None:
+        """
+        Register a detector callable for a capability name.
+        """
+        self._capability_detectors[capability] = detector
+
+    def supports(self, capability: str, model_name: Optional[str]) -> bool:
+        """
+        Return True when the supplied model name supports the requested capability.
+        """
+        if not model_name:
+            return False
+
+        detector = self._capability_detectors.get(capability)
+        if detector is None:
+            return False
+
+        try:
+            return detector(model_name)
+        except Exception:
+            return False
+
+    def supports_vision(self, model_name: Optional[str]) -> bool:
+        """
+        Convenience wrapper for vision-capable detection.
+        """
+        return self.supports("vision", model_name)
+
+    def supports_video(self, model_name: Optional[str]) -> bool:
+        """
+        Convenience wrapper for video-capable detection.
+        """
+        return self.supports("video", model_name)
+
+    def add_vision_model(self, model_name: str) -> None:
+        # Extend the module-level registry used by vision_capability_detector
+        VISION_MODEL_PREFIXES.add(self._strip_provider_prefix(model_name).lower())
+
+    def add_vision_models(self, model_names: Iterable[str]) -> None:
+        for model_name in model_names:
+            self.add_vision_model(model_name)
+
+    def _supports_vision(self, model_name: str) -> bool:
+        return vision_capability_detector(model_name)
+
+    @staticmethod
+    def _strip_provider_prefix(model_name: str) -> str:
+        return _strip_provider_prefix(model_name)
+
+    @staticmethod
+    def _litellm_supports_vision(model_name: str) -> bool:
+        return _litellm_supports_vision(model_name)
+
+
+MODEL_CAPABILITIES_REGISTRY = ModelCapabilitiesRegistry()
+MODEL_CAPABILITIES_REGISTRY.register_capability_detector(
+    "vision", vision_capability_detector
+)
+MODEL_CAPABILITIES_REGISTRY.register_capability_detector(
+    "video", video_capability_detector
+)
+
+# Backwards compatibility shim for previous API which exposed a class with classmethods.
+ModelCapabilities = MODEL_CAPABILITIES_REGISTRY
+
+
+__all__ = [
+    "ModelCapabilitiesRegistry",
+    "MODEL_CAPABILITIES_REGISTRY",
+    "ModelCapabilities",
+    "vision_capability_detector",
+    "video_capability_detector",
+]

opik/evaluation/models/models_factory.py

@@ -1,13 +1,35 @@
-from typing import Optional, Any
+from typing import Optional, Any, Dict
 
 from .litellm import litellm_chat_model
 from . import base_model
 
-DEFAULT_GPT_MODEL_NAME = "gpt-4o"
+DEFAULT_GPT_MODEL_NAME = "gpt-5-nano"
+
+_MODEL_CACHE: Dict[Any, base_model.OpikBaseModel] = {}
+
+
+def _freeze(value: Any) -> Any:
+    if isinstance(value, dict):
+        return frozenset((k, _freeze(v)) for k, v in value.items())
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze(v) for v in value)
+    if isinstance(value, set):
+        return frozenset(_freeze(v) for v in value)
+    return value
+
+
+def _make_cache_key(model_name: str, model_kwargs: Dict[str, Any]) -> Any:
+    frozen_kwargs = frozenset((k, _freeze(v)) for k, v in model_kwargs.items())
+    return (model_name, frozen_kwargs)
 
 
 def get(model_name: Optional[str], **model_kwargs: Any) -> base_model.OpikBaseModel:
     if model_name is None:
         model_name = DEFAULT_GPT_MODEL_NAME
 
-    return litellm_chat_model.LiteLLMChatModel(model_name=model_name, **model_kwargs)
+    cache_key = _make_cache_key(model_name, model_kwargs)
+    if cache_key not in _MODEL_CACHE:
+        _MODEL_CACHE[cache_key] = litellm_chat_model.LiteLLMChatModel(
+            model_name=model_name, **model_kwargs
+        )
+    return _MODEL_CACHE[cache_key]

opik/evaluation/preprocessing.py

@@ -0,0 +1,92 @@
+"""Shared text preprocessing utilities for metrics."""
+
+from __future__ import annotations
+
+import re
+import string
+import unicodedata
+from typing import Callable, Literal
+
+try:  # optional dependency for emoji detection
+    import emoji
+except ImportError:  # pragma: no cover
+    emoji = None  # type: ignore
+
+_Normalizer = Callable[[str], str]
+
+
+def normalize_text(
+    text: str,
+    *,
+    lowercase: bool = True,
+    strip_accents: bool = False,
+    remove_punctuation: bool = False,
+    keep_emoji: bool = True,
+    normalize_form: Literal["NFC", "NFD", "NFKC", "NFKD"] = "NFKC",
+) -> str:
+    """Normalize text before metric processing.
+
+    Args:
+        text: Input string.
+        lowercase: Whether to lowercase the text.
+        strip_accents: Remove diacritical marks.
+        remove_punctuation: Strip ASCII punctuation.
+        keep_emoji: Preserve emoji characters; if False they are removed.
+        normalize_form: Unicode normalization form to apply (default NFKC).
+    """
+
+    normalized = unicodedata.normalize(normalize_form, text)
+    if lowercase:
+        normalized = normalized.lower()
+
+    if not keep_emoji:
+        normalized = _remove_emoji(normalized)
+
+    if strip_accents:
+        normalized = _strip_accents(normalized)
+
+    if remove_punctuation:
+        normalized = _remove_punctuation(normalized)
+
+    normalized = _collapse_whitespace(normalized)
+    return normalized.strip()
+
+
+def _remove_emoji(text: str) -> str:
+    if emoji is None:  # pragma: no cover
+        return "".join(
+            ch for ch in text if unicodedata.category(ch) not in {"So", "Sk"}
+        )
+    return emoji.replace_emoji(text, replace="")
+
+
+def _strip_accents(text: str) -> str:
+    decomposed = unicodedata.normalize("NFD", text)
+    return "".join(ch for ch in decomposed if unicodedata.category(ch) != "Mn")
+
+
+def _remove_punctuation(text: str) -> str:
+    translator = str.maketrans("", "", string.punctuation)
+    stripped = text.translate(translator)
+    return re.sub(
+        r"[\u2010-\u2015\u2018-\u201f\u2020-\u2027\u2030-\u2043]", "", stripped
+    )
+
+
+def _collapse_whitespace(text: str) -> str:
+    return re.sub(r"\s+", " ", text)
+
+
+DEFAULT_NORMALIZER: _Normalizer = normalize_text
+
+
+def ascii_normalizer(text: str) -> str:
+    return normalize_text(
+        text,
+        strip_accents=True,
+        remove_punctuation=True,
+        keep_emoji=False,
+    )
+
+
+ASCII_NORMALIZER: _Normalizer = ascii_normalizer

opik/evaluation/report.py

@@ -1,11 +1,11 @@
 from collections import defaultdict
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from rich import align, console, panel, table, text
 
 
-from .. import url_helpers
-from . import test_result
+from . import test_result, evaluation_result
+from .metrics import score_result
 
 
 def _format_time(seconds: float) -> str:
@@ -42,7 +42,10 @@ def _compute_average_scores(
 
 
 def display_experiment_results(
-    dataset_name: str, total_time: float, test_results: List[test_result.TestResult]
+    dataset_name: str,
+    total_time: float,
+    test_results: List[test_result.TestResult],
+    experiment_scores: Optional[List[score_result.ScoreResult]] = None,
 ) -> None:
     average_scores, failed_scores = _compute_average_scores(test_results)
     nb_items = len(test_results)
@@ -63,6 +66,14 @@ def display_experiment_results(
             score_strings += text.Text(f" - {failed_scores[name]} failed", style="red")
         score_strings += text.Text("\n")
 
+    # Add experiment scores if available
+    if experiment_scores:
+        for score in experiment_scores:
+            score_strings += text.Text(
+                f"{score.name}: {score.value:.4f}", style="green bold"
+            )
+            score_strings += text.Text("\n")
+
     aligned_test_results = align.Align.left(score_strings)
 
     # Combine table, time text, and test results
@@ -87,16 +98,63 @@ def display_experiment_results(
     console_container.print("Uploading results to Opik ... ")
 
 
-def display_experiment_link(
-    experiment_id: str, dataset_id: str, url_override: str
-) -> None:
+def display_experiment_link(experiment_url: str) -> None:
     console_container = console.Console()
 
-    experiment_url = url_helpers.get_experiment_url_by_id(
-        experiment_id=experiment_id,
-        dataset_id=dataset_id,
-        url_override=url_override,
-    )
     console_container.print(
         f"View the results [link={experiment_url}]in your Opik dashboard[/link]."
     )
+
+
+def display_evaluation_scores_statistics(
+    dataset_name: str,
+    evaluation_results: evaluation_result.EvaluationResult,
+) -> None:
+    """
+    Displays evaluation scores statistics for a given dataset.
+
+    The function generates a summary of evaluation scores including mean, max,
+    min, and optionally standard deviation for each metric in the evaluation
+    results. The summarized scores are formatted and presented in a table
+    within a panel for user clarity.
+
+    Args:
+        dataset_name: Name of the dataset for which evaluation statistics are
+            being displayed.
+        evaluation_results: An object containing evaluation results with
+            aggregated scores and statistical data.
+    """
+    aggregated_view = evaluation_results.aggregate_evaluation_scores()
+    if not aggregated_view.aggregated_scores:
+        return
+
+    # Create a table for the statistics
+    stats_table = table.Table()
+    stats_table.add_column("Name", style="cyan", no_wrap=True)
+    stats_table.add_column("Mean", justify="right", style="green")
+    stats_table.add_column("Min", justify="right", style="yellow")
+    stats_table.add_column("Max", justify="right", style="yellow")
+    stats_table.add_column("Std", justify="right", style="magenta")
+
+    # Add rows for each metric
+    for name, stats in aggregated_view.aggregated_scores.items():
+        std_value = f"{stats.std:.4f}" if stats.std is not None else "N/A"
+        stats_table.add_row(
+            name,
+            f"{stats.mean:.4f}",
+            f"{stats.min:.4f}",
+            f"{stats.max:.4f}",
+            std_value,
+        )
+
+    # Create a panel with the table inside
+    panel_content = panel.Panel(
+        stats_table,
+        title=f"Evaluation statistics for {dataset_name}",
+        title_align="left",
+        expand=False,
+    )
+
+    # Display results
+    console_container = console.Console()
+    console_container.print(panel_content)

opik/evaluation/rest_operations.py

@@ -1,11 +1,14 @@
+import logging
 from typing import List, Optional
 
-from opik.api_objects import experiment, opik_client
-from opik.types import FeedbackScoreDict
-from . import test_case, test_result
-from .metrics import arguments_helpers
+from opik.api_objects import dataset, experiment, opik_client
+from opik.types import BatchFeedbackScoreDict
+from . import test_case
+from .metrics import score_result
 from .types import ScoringKeyMappingType
 
+LOGGER = logging.getLogger(__name__)
+
 
 def get_experiment_with_unique_name(
     client: opik_client.Opik, experiment_name: str
@@ -34,63 +37,64 @@ def get_trace_project_name(client: opik_client.Opik, trace_id: str) -> str:
 
 
 def get_experiment_test_cases(
-    client: opik_client.Opik,
-    experiment_id: str,
-    dataset_id: str,
+    experiment_: experiment.Experiment,
+    dataset_: dataset.Dataset,
     scoring_key_mapping: Optional[ScoringKeyMappingType],
 ) -> List[test_case.TestCase]:
+    experiment_items = experiment_.get_items()
+
+    # Fetch dataset items to get input data for bulk-uploaded experiment items
+    dataset_items_by_id = {item["id"]: item for item in dataset_.get_items()}
+
     test_cases = []
-    page = 1
+    for item in experiment_items:
+        dataset_item_data = dataset_items_by_id.get(item.dataset_item_id)
 
-    while True:
-        experiment_items_page = (
-            client._rest_client.datasets.find_dataset_items_with_experiment_items(
-                id=dataset_id, experiment_ids=f'["{experiment_id}"]', page=page
+        if dataset_item_data is None:
+            LOGGER.error(
+                f"Unexpected error: Dataset item with id {item.dataset_item_id} not found, skipping experiment item {item.id}"
+            )
+            continue
+
+        if item.evaluation_task_output is None:
+            LOGGER.error(
+                f"Unexpected error: Evaluation task output is None for experiment item {item.id}, skipping experiment item"
+            )
+            continue
+
+        test_cases.append(
+            test_case.TestCase(
+                trace_id=item.trace_id,
+                dataset_item_id=item.dataset_item_id,
+                task_output=item.evaluation_task_output,
+                dataset_item_content=dataset_item_data,
             )
         )
-        if len(experiment_items_page.content) == 0:
-            break
-
-        for item in experiment_items_page.content:
-            experiment_item = item.experiment_items[0]
-
-            test_cases += [
-                test_case.TestCase(
-                    trace_id=experiment_item.trace_id,
-                    dataset_item_id=experiment_item.dataset_item_id,
-                    task_output=experiment_item.output,
-                    scoring_inputs=arguments_helpers.create_scoring_inputs(
-                        dataset_item=experiment_item.input,
-                        task_output=experiment_item.output,
-                        scoring_key_mapping=scoring_key_mapping,
-                    ),
-                )
-            ]
-
-        page += 1
 
     return test_cases
 
 
-def log_test_result_scores(
+def log_test_result_feedback_scores(
     client: opik_client.Opik,
-    test_result: test_result.TestResult,
+    score_results: List[score_result.ScoreResult],
+    trace_id: str,
     project_name: Optional[str],
 ) -> None:
-    all_trace_scores: List[FeedbackScoreDict] = []
+    all_trace_scores: List[BatchFeedbackScoreDict] = []
 
-    for score_result in test_result.score_results:
-        if score_result.scoring_failed:
+    for score_result_ in score_results:
+        if score_result_.scoring_failed:
             continue
 
-        trace_score = FeedbackScoreDict(
-            id=test_result.test_case.trace_id,
-            name=score_result.name,
-            value=score_result.value,
-            reason=score_result.reason,
+        trace_score = BatchFeedbackScoreDict(
+            id=trace_id,
+            name=score_result_.name,
+            value=score_result_.value,
+            reason=score_result_.reason,
         )
         all_trace_scores.append(trace_score)
 
-    client.log_traces_feedback_scores(
-        scores=all_trace_scores, project_name=project_name
-    )
+    if len(all_trace_scores) > 0:
+        client.log_traces_feedback_scores(
+            scores=all_trace_scores, project_name=project_name
+        )

opik/evaluation/samplers/__init__.py

@@ -0,0 +1,4 @@
+from .base_dataset_sampler import BaseDatasetSampler
+from .random_dataset_sampler import RandomDatasetSampler
+
+__all__ = ["BaseDatasetSampler", "RandomDatasetSampler"]

opik/evaluation/samplers/base_dataset_sampler.py

@@ -0,0 +1,40 @@
+import abc
+from typing import List
+
+
+from opik.api_objects.dataset import dataset_item
+
+
+class BaseDatasetSampler(abc.ABC):
+    """
+    Defines the BaseDatasetSampler for sampling dataset items.
+
+    This is an abstract base class that provides the definition
+    for dataset sampling. It requires implementation of the `sample`
+    method in subclasses, which specifies the sampling logic tailored
+    to specific needs.
+
+    Methods in this class are enforced to be redefined in any
+    concrete implementation.
+
+    """
+
+    @abc.abstractmethod
+    def sample(
+        self, data_item: List[dataset_item.DatasetItem]
+    ) -> List[dataset_item.DatasetItem]:
+        """
+        Samples and filters a list of dataset items according to a specific implementation.
+
+        Args:
+            data_item (List[dataset_item.DatasetItem]): A list of DatasetItem objects to be
+                sampled and filtered.
+
+        Returns:
+            List[dataset_item.DatasetItem]: A list of DatasetItem objects resulting
+                from the sampling process.
+
+        Raises:
+            NotImplementedError: If the method is not implemented in a subclass.
+        """
+        pass

opik/evaluation/samplers/random_dataset_sampler.py

@@ -0,0 +1,48 @@
+import random
+from typing import List, Optional
+
+from opik.api_objects.dataset import dataset_item
+
+from . import base_dataset_sampler
+
+
+class RandomDatasetSampler(base_dataset_sampler.BaseDatasetSampler):
+    def __init__(
+        self, max_samples: int, shuffle: bool = True, seed: Optional[int] = None
+    ) -> None:
+        """Samples a random subset of dataset items.
+
+        This class is a dataset sampler that selects a random subset of items from a dataset.
+        The number of items to sample can be specified, and shuffling can be enabled or disabled.
+        An optional random seed can be provided for reproducibility.
+
+        Args:
+            max_samples: The maximum number of samples to generate.
+            shuffle: Whether to shuffle the samples. Default is True, False provides a speedup
+                for large datasets.
+            seed: Seed for the random number generator. If None, then fresh, unpredictable
+                entropy will be pulled from the OS.
+        """
+        self.max_samples = max_samples
+        self.shuffle = shuffle
+        self.seed = seed
+
+    def sample(
+        self, data_items: List[dataset_item.DatasetItem]
+    ) -> List[dataset_item.DatasetItem]:
+        if len(data_items) == 0:
+            return []
+
+        # Create a random number generator with the specified seed
+        rng = random.Random(self.seed)
+
+        # Determine how many samples to take
+        sample_size = min(len(data_items), self.max_samples)
+
+        # Do sample first to avoid shuffling the entire dataset
+        items = rng.sample(data_items, sample_size)
+
+        if self.shuffle:
+            rng.shuffle(items)
+
+        return items