opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/preprocessing.py

@@ -0,0 +1,92 @@
+"""Shared text preprocessing utilities for metrics."""
+
+from __future__ import annotations
+
+import re
+import string
+import unicodedata
+from typing import Callable, Literal
+
+try:  # optional dependency for emoji detection
+    import emoji
+except ImportError:  # pragma: no cover
+    emoji = None  # type: ignore
+
+_Normalizer = Callable[[str], str]
+
+
+def normalize_text(
+    text: str,
+    *,
+    lowercase: bool = True,
+    strip_accents: bool = False,
+    remove_punctuation: bool = False,
+    keep_emoji: bool = True,
+    normalize_form: Literal["NFC", "NFD", "NFKC", "NFKD"] = "NFKC",
+) -> str:
+    """Normalize text before metric processing.
+
+    Args:
+        text: Input string.
+        lowercase: Whether to lowercase the text.
+        strip_accents: Remove diacritical marks.
+        remove_punctuation: Strip ASCII punctuation.
+        keep_emoji: Preserve emoji characters; if False they are removed.
+        normalize_form: Unicode normalization form to apply (default NFKC).
+    """
+
+    normalized = unicodedata.normalize(normalize_form, text)
+    if lowercase:
+        normalized = normalized.lower()
+
+    if not keep_emoji:
+        normalized = _remove_emoji(normalized)
+
+    if strip_accents:
+        normalized = _strip_accents(normalized)
+
+    if remove_punctuation:
+        normalized = _remove_punctuation(normalized)
+
+    normalized = _collapse_whitespace(normalized)
+    return normalized.strip()
+
+
+def _remove_emoji(text: str) -> str:
+    if emoji is None:  # pragma: no cover
+        return "".join(
+            ch for ch in text if unicodedata.category(ch) not in {"So", "Sk"}
+        )
+    return emoji.replace_emoji(text, replace="")
+
+
+def _strip_accents(text: str) -> str:
+    decomposed = unicodedata.normalize("NFD", text)
+    return "".join(ch for ch in decomposed if unicodedata.category(ch) != "Mn")
+
+
+def _remove_punctuation(text: str) -> str:
+    translator = str.maketrans("", "", string.punctuation)
+    stripped = text.translate(translator)
+    return re.sub(
+        r"[\u2010-\u2015\u2018-\u201f\u2020-\u2027\u2030-\u2043]", "", stripped
+    )
+
+
+def _collapse_whitespace(text: str) -> str:
+    return re.sub(r"\s+", " ", text)
+
+
+DEFAULT_NORMALIZER: _Normalizer = normalize_text
+
+
+def ascii_normalizer(text: str) -> str:
+    return normalize_text(
+        text,
+        strip_accents=True,
+        remove_punctuation=True,
+        keep_emoji=False,
+    )
+
+
+ASCII_NORMALIZER: _Normalizer = ascii_normalizer

opik/evaluation/report.py

@@ -1,11 +1,11 @@
 from collections import defaultdict
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 from rich import align, console, panel, table, text
 
 
-from .. import url_helpers
-from . import test_result
+from . import test_result, evaluation_result
+from .metrics import score_result
 
 
 def _format_time(seconds: float) -> str:
@@ -42,7 +42,10 @@ def _compute_average_scores(
 
 
 def display_experiment_results(
-    dataset_name: str, total_time: float, test_results: List[test_result.TestResult]
+    dataset_name: str,
+    total_time: float,
+    test_results: List[test_result.TestResult],
+    experiment_scores: Optional[List[score_result.ScoreResult]] = None,
 ) -> None:
     average_scores, failed_scores = _compute_average_scores(test_results)
     nb_items = len(test_results)
@@ -63,6 +66,14 @@ def display_experiment_results(
             score_strings += text.Text(f" - {failed_scores[name]} failed", style="red")
         score_strings += text.Text("\n")
 
+    # Add experiment scores if available
+    if experiment_scores:
+        for score in experiment_scores:
+            score_strings += text.Text(
+                f"{score.name}: {score.value:.4f}", style="green bold"
+            )
+            score_strings += text.Text("\n")
+
     aligned_test_results = align.Align.left(score_strings)
 
     # Combine table, time text, and test results
@@ -87,16 +98,63 @@ def display_experiment_results(
     console_container.print("Uploading results to Opik ... ")
 
 
-def display_experiment_link(
-    experiment_id: str, dataset_id: str, url_override: str
-) -> None:
+def display_experiment_link(experiment_url: str) -> None:
     console_container = console.Console()
 
-    experiment_url = url_helpers.get_experiment_url_by_id(
-        experiment_id=experiment_id,
-        dataset_id=dataset_id,
-        url_override=url_override,
-    )
     console_container.print(
         f"View the results [link={experiment_url}]in your Opik dashboard[/link]."
     )
+
+
+def display_evaluation_scores_statistics(
+    dataset_name: str,
+    evaluation_results: evaluation_result.EvaluationResult,
+) -> None:
+    """
+    Displays evaluation scores statistics for a given dataset.
+
+    The function generates a summary of evaluation scores including mean, max,
+    min, and optionally standard deviation for each metric in the evaluation
+    results. The summarized scores are formatted and presented in a table
+    within a panel for user clarity.
+
+    Args:
+        dataset_name: Name of the dataset for which evaluation statistics are
+            being displayed.
+        evaluation_results: An object containing evaluation results with
+            aggregated scores and statistical data.
+    """
+    aggregated_view = evaluation_results.aggregate_evaluation_scores()
+    if not aggregated_view.aggregated_scores:
+        return
+
+    # Create a table for the statistics
+    stats_table = table.Table()
+    stats_table.add_column("Name", style="cyan", no_wrap=True)
+    stats_table.add_column("Mean", justify="right", style="green")
+    stats_table.add_column("Min", justify="right", style="yellow")
+    stats_table.add_column("Max", justify="right", style="yellow")
+    stats_table.add_column("Std", justify="right", style="magenta")
+
+    # Add rows for each metric
+    for name, stats in aggregated_view.aggregated_scores.items():
+        std_value = f"{stats.std:.4f}" if stats.std is not None else "N/A"
+        stats_table.add_row(
+            name,
+            f"{stats.mean:.4f}",
+            f"{stats.min:.4f}",
+            f"{stats.max:.4f}",
+            std_value,
+        )
+
+    # Create a panel with the table inside
+    panel_content = panel.Panel(
+        stats_table,
+        title=f"Evaluation statistics for {dataset_name}",
+        title_align="left",
+        expand=False,
+    )
+
+    # Display results
+    console_container = console.Console()
+    console_container.print(panel_content)

opik/evaluation/rest_operations.py

@@ -1,11 +1,14 @@
+import logging
 from typing import List, Optional
 
-from opik.api_objects import experiment, opik_client
-from opik.types import FeedbackScoreDict
-from . import test_case, test_result
-from .metrics import arguments_helpers
+from opik.api_objects import dataset, experiment, opik_client
+from opik.types import BatchFeedbackScoreDict
+from . import test_case
+from .metrics import score_result
 from .types import ScoringKeyMappingType
 
+LOGGER = logging.getLogger(__name__)
+
 
 def get_experiment_with_unique_name(
     client: opik_client.Opik, experiment_name: str
@@ -34,63 +37,64 @@ def get_trace_project_name(client: opik_client.Opik, trace_id: str) -> str:
 
 
 def get_experiment_test_cases(
-    client: opik_client.Opik,
-    experiment_id: str,
-    dataset_id: str,
+    experiment_: experiment.Experiment,
+    dataset_: dataset.Dataset,
     scoring_key_mapping: Optional[ScoringKeyMappingType],
 ) -> List[test_case.TestCase]:
+    experiment_items = experiment_.get_items()
+
+    # Fetch dataset items to get input data for bulk-uploaded experiment items
+    dataset_items_by_id = {item["id"]: item for item in dataset_.get_items()}
+
     test_cases = []
-    page = 1
+    for item in experiment_items:
+        dataset_item_data = dataset_items_by_id.get(item.dataset_item_id)
 
-    while True:
-        experiment_items_page = (
-            client._rest_client.datasets.find_dataset_items_with_experiment_items(
-                id=dataset_id, experiment_ids=f'["{experiment_id}"]', page=page
+        if dataset_item_data is None:
+            LOGGER.error(
+                f"Unexpected error: Dataset item with id {item.dataset_item_id} not found, skipping experiment item {item.id}"
+            )
+            continue
+
+        if item.evaluation_task_output is None:
+            LOGGER.error(
+                f"Unexpected error: Evaluation task output is None for experiment item {item.id}, skipping experiment item"
+            )
+            continue
+
+        test_cases.append(
+            test_case.TestCase(
+                trace_id=item.trace_id,
+                dataset_item_id=item.dataset_item_id,
+                task_output=item.evaluation_task_output,
+                dataset_item_content=dataset_item_data,
             )
         )
-        if len(experiment_items_page.content) == 0:
-            break
-
-        for item in experiment_items_page.content:
-            experiment_item = item.experiment_items[0]
-
-            test_cases += [
-                test_case.TestCase(
-                    trace_id=experiment_item.trace_id,
-                    dataset_item_id=experiment_item.dataset_item_id,
-                    task_output=experiment_item.output,
-                    scoring_inputs=arguments_helpers.create_scoring_inputs(
-                        dataset_item=experiment_item.input,
-                        task_output=experiment_item.output,
-                        scoring_key_mapping=scoring_key_mapping,
-                    ),
-                )
-            ]
-
-        page += 1
 
     return test_cases
 
 
-def log_test_result_scores(
+def log_test_result_feedback_scores(
     client: opik_client.Opik,
-    test_result: test_result.TestResult,
+    score_results: List[score_result.ScoreResult],
+    trace_id: str,
     project_name: Optional[str],
 ) -> None:
-    all_trace_scores: List[FeedbackScoreDict] = []
+    all_trace_scores: List[BatchFeedbackScoreDict] = []
 
-    for score_result in test_result.score_results:
-        if score_result.scoring_failed:
+    for score_result_ in score_results:
+        if score_result_.scoring_failed:
             continue
 
-        trace_score = FeedbackScoreDict(
-            id=test_result.test_case.trace_id,
-            name=score_result.name,
-            value=score_result.value,
-            reason=score_result.reason,
+        trace_score = BatchFeedbackScoreDict(
+            id=trace_id,
+            name=score_result_.name,
+            value=score_result_.value,
+            reason=score_result_.reason,
         )
         all_trace_scores.append(trace_score)
 
-    client.log_traces_feedback_scores(
-        scores=all_trace_scores, project_name=project_name
-    )
+    if len(all_trace_scores) > 0:
+        client.log_traces_feedback_scores(
+            scores=all_trace_scores, project_name=project_name
+        )

opik/evaluation/samplers/__init__.py

@@ -0,0 +1,4 @@
+from .base_dataset_sampler import BaseDatasetSampler
+from .random_dataset_sampler import RandomDatasetSampler
+
+__all__ = ["BaseDatasetSampler", "RandomDatasetSampler"]

opik/evaluation/samplers/base_dataset_sampler.py

@@ -0,0 +1,40 @@
+import abc
+from typing import List
+
+
+from opik.api_objects.dataset import dataset_item
+
+
+class BaseDatasetSampler(abc.ABC):
+    """
+    Defines the BaseDatasetSampler for sampling dataset items.
+
+    This is an abstract base class that provides the definition
+    for dataset sampling. It requires implementation of the `sample`
+    method in subclasses, which specifies the sampling logic tailored
+    to specific needs.
+
+    Methods in this class are enforced to be redefined in any
+    concrete implementation.
+
+    """
+
+    @abc.abstractmethod
+    def sample(
+        self, data_item: List[dataset_item.DatasetItem]
+    ) -> List[dataset_item.DatasetItem]:
+        """
+        Samples and filters a list of dataset items according to a specific implementation.
+
+        Args:
+            data_item (List[dataset_item.DatasetItem]): A list of DatasetItem objects to be
+                sampled and filtered.
+
+        Returns:
+            List[dataset_item.DatasetItem]: A list of DatasetItem objects resulting
+                from the sampling process.
+
+        Raises:
+            NotImplementedError: If the method is not implemented in a subclass.
+        """
+        pass

opik/evaluation/samplers/random_dataset_sampler.py

@@ -0,0 +1,48 @@
+import random
+from typing import List, Optional
+
+from opik.api_objects.dataset import dataset_item
+
+from . import base_dataset_sampler
+
+
+class RandomDatasetSampler(base_dataset_sampler.BaseDatasetSampler):
+    def __init__(
+        self, max_samples: int, shuffle: bool = True, seed: Optional[int] = None
+    ) -> None:
+        """Samples a random subset of dataset items.
+
+        This class is a dataset sampler that selects a random subset of items from a dataset.
+        The number of items to sample can be specified, and shuffling can be enabled or disabled.
+        An optional random seed can be provided for reproducibility.
+
+        Args:
+            max_samples: The maximum number of samples to generate.
+            shuffle: Whether to shuffle the samples. Default is True, False provides a speedup
+                for large datasets.
+            seed: Seed for the random number generator. If None, then fresh, unpredictable
+                entropy will be pulled from the OS.
+        """
+        self.max_samples = max_samples
+        self.shuffle = shuffle
+        self.seed = seed
+
+    def sample(
+        self, data_items: List[dataset_item.DatasetItem]
+    ) -> List[dataset_item.DatasetItem]:
+        if len(data_items) == 0:
+            return []
+
+        # Create a random number generator with the specified seed
+        rng = random.Random(self.seed)
+
+        # Determine how many samples to take
+        sample_size = min(len(data_items), self.max_samples)
+
+        # Do sample first to avoid shuffling the entire dataset
+        items = rng.sample(data_items, sample_size)
+
+        if self.shuffle:
+            rng.shuffle(items)
+
+        return items

opik/evaluation/score_statistics.py

@@ -0,0 +1,66 @@
+import dataclasses
+import math
+import statistics
+from collections import defaultdict
+from typing import List, Optional, Dict
+
+from opik.evaluation import test_result
+
+
+@dataclasses.dataclass
+class ScoreStatistics:
+    """Statistics for a single score metric across multiple trials."""
+
+    mean: float
+    max: float
+    min: float
+    values: List[float]
+    std: Optional[float] = None  # Standard deviation (None if count < 2)
+
+
+def calculate_aggregated_statistics(
+    evaluation_results: List[test_result.TestResult],
+) -> Dict[str, ScoreStatistics]:
+    """
+    Calculate mean, max, and min scores for each score name in the evaluation test results.
+
+    Args:
+        evaluation_results: List of TestResult objects to be aggregated
+
+    Returns:
+        Dict mapping score names to their aggregated statistics
+    """
+    if not evaluation_results:
+        return {}
+
+    # Group scores by name across all trials
+    scores_by_name = defaultdict(list)
+
+    for test_result_ in evaluation_results:
+        for score_result in test_result_.score_results:
+            # Only include successful scores with valid values
+            if not score_result.scoring_failed and _is_valid_score_value(
+                score_result.value
+            ):
+                scores_by_name[score_result.name].append(score_result.value)
+
+    # Calculate aggregated statistics for each score name
+    aggregated_scores = {}
+    for score_name, values in scores_by_name.items():
+        if values:
+            std = statistics.stdev(values) if len(values) >= 2 else None
+
+            aggregated_scores[score_name] = ScoreStatistics(
+                mean=statistics.mean(values),
+                max=max(values),
+                min=min(values),
+                values=values.copy(),  # Store the actual values used
+                std=std,
+            )
+
+    return aggregated_scores
+
+
+def _is_valid_score_value(value: float) -> bool:
+    """Check if a score value is valid for statistical calculations."""
+    return isinstance(value, (int, float)) and math.isfinite(value)

opik/evaluation/scorers/__init__.py

@@ -0,0 +1,4 @@
+from .scorer_function import ScorerFunction
+from .scorer_wrapper_metric import ScorerWrapperMetric
+
+__all__ = ["ScorerFunction", "ScorerWrapperMetric"]

opik/evaluation/scorers/scorer_function.py

@@ -0,0 +1,55 @@
+import inspect
+from typing import Any, Dict, Optional, Protocol, Union, List
+
+from opik.evaluation.metrics import score_result
+from opik.message_processing.emulation import models
+
+
+class ScorerFunctionProtocol(Protocol):
+    """
+    Represents a protocol defining the structure for a scorer function.
+
+    This protocol serves as a contract for implementing scorer functions used in
+    evaluating tasks. A scorer function adhering to this protocol should take
+    dataset item data, task outputs, and optionally a task span model as input
+    parameters and return a scoring result.
+    """
+
+    def __call__(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        task_span: Optional[models.SpanModel] = None,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]: ...
+
+
+ScorerFunction = ScorerFunctionProtocol
+
+
+EXPECTED_SCORER_FUNCTION_PARAMETERS = ["dataset_item", "task_outputs"]
+
+
+def validate_scorer_function(scorer_function: ScorerFunction) -> None:
+    if not callable(scorer_function):
+        raise ValueError("scorer_function must be a callable function")
+
+    parameters = inspect.signature(scorer_function).parameters
+    names = set(parameters.keys())
+
+    # Check if it has both dataset_item and task_outputs
+    has_dataset_item_and_task_outputs = all(
+        param in names for param in EXPECTED_SCORER_FUNCTION_PARAMETERS
+    )
+
+    # Check if it has at least one task_span parameter
+    has_task_span = "task_span" in names
+
+    if not (has_dataset_item_and_task_outputs or has_task_span):
+        raise ValueError(
+            f"scorer_function must have either both 'dataset_item' and 'task_outputs' parameters "
+            f"or at least one 'task_span' parameter. Found parameters: {list(names)}"
+        )
+
+
+def has_task_span_in_parameters(scorer_function: ScorerFunction) -> bool:
+    return "task_span" in inspect.signature(scorer_function).parameters

opik/evaluation/scorers/scorer_wrapper_metric.py

@@ -0,0 +1,130 @@
+from typing import Any, Callable, Dict, Optional, List, Union
+
+from opik.evaluation.metrics import base_metric, score_result
+
+from . import scorer_function
+from ...message_processing.emulation import models
+
+
+class ScorerWrapperMetric(base_metric.BaseMetric):
+    """
+    A wrapper metric that adapts a ScorerFunction to the BaseMetric interface.
+
+    This class allows using ScorerFunction instances as BaseMetric instances,
+    providing compatibility between the two interfaces.
+
+    Args:
+        scorer: The ScorerFunction to wrap
+        name: Optional name for the metric. If not provided, uses the class name.
+        track: Whether to track the metric. Defaults to True.
+        project_name: Optional project name for tracking.
+
+    Raises:
+        ValueError if the scorer function is invalid.
+
+    Example:
+        >>> def my_scorer(dataset_item: Dict[str, Any], task_outputs: Dict[str, Any]) -> score_result.ScoreResult:
+        >>>     return score_result.ScoreResult(name="my_metric", value=1.0)
+        >>>
+        >>> wrapper = ScorerWrapperMetric(scorer_function=my_scorer, name="wrapped_scorer")
+        >>> result = wrapper.score(dataset_item={"text": "hello"}, task_outputs={"text": "hello"})
+    """
+
+    def __init__(
+        self,
+        scorer: scorer_function.ScorerFunction,
+        name: str,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(name=name, track=track, project_name=project_name)
+        self.scorer = scorer
+
+        # validate scorer function
+        scorer_function.validate_scorer_function(scorer)
+
+    def score(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        **kwargs: Any,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Score using the wrapped ScorerFunction.
+
+        Args:
+            dataset_item: The dataset item data to score against
+            task_outputs: The output dictionary to be scored - can be the output of LLM task, etc.
+            **kwargs: Additional keyword arguments (ignored by the scorer function)
+
+        Returns:
+            ScoreResult from the wrapped scorer function
+        """
+        return self.scorer(dataset_item=dataset_item, task_outputs=task_outputs)
+
+
+class ScorerWrapperMetricTaskSpan(ScorerWrapperMetric):
+    def __init__(
+        self,
+        scorer: scorer_function.ScorerFunction,
+        name: str,
+        track: bool = True,
+        project_name: Optional[str] = None,
+    ) -> None:
+        super().__init__(
+            scorer=scorer, name=name, track=track, project_name=project_name
+        )
+
+    def score(
+        self,
+        dataset_item: Dict[str, Any],
+        task_outputs: Dict[str, Any],
+        task_span: Optional[models.SpanModel] = None,
+        **kwargs: Any,
+    ) -> Union[score_result.ScoreResult, List[score_result.ScoreResult]]:
+        """
+        Score using the wrapped ScorerFunction.
+
+        Args:
+            dataset_item: The dataset item data to score against
+            task_outputs: The output dictionary to be scored - can be the output of LLM task, etc.
+            task_span: The collected task span data.
+            **kwargs: Additional keyword arguments (ignored by the scorer function)
+
+        Returns:
+            ScoreResult from the wrapped scorer function
+        """
+        if task_span is not None and scorer_function.has_task_span_in_parameters(
+            self.scorer
+        ):
+            return self.scorer(
+                dataset_item=dataset_item,
+                task_outputs=task_outputs,
+                task_span=task_span,
+            )
+
+        return self.scorer(dataset_item=dataset_item, task_outputs=task_outputs)
+
+
+def _scorer_name(scorer: Callable) -> str:
+    return scorer.__name__
+
+
+def wrap_scorer_functions(
+    scorer_functions: List[scorer_function.ScorerFunction], project_name: Optional[str]
+) -> List[base_metric.BaseMetric]:
+    metrics: List[base_metric.BaseMetric] = []
+    for f in scorer_functions:
+        name = _scorer_name(f)
+        if scorer_function.has_task_span_in_parameters(f):
+            metrics.append(
+                ScorerWrapperMetricTaskSpan(
+                    scorer=f, project_name=project_name, name=name
+                )
+            )
+        else:
+            metrics.append(
+                ScorerWrapperMetric(scorer=f, project_name=project_name, name=name)
+            )
+
+    return metrics