opik 1.8.39__py3-none-any.whl → 1.9.71__py3-none-any.whl

opik/evaluation/evaluator.py

@@ -1,37 +1,98 @@
 import logging
 import time
-from typing import Any, Callable, Dict, List, Optional, Union
+from typing import Any, Callable, Dict, List, Optional, Union, cast
 
-from .. import Prompt
+from ..api_objects.prompt import base_prompt
 from ..api_objects import opik_client
 from ..api_objects import dataset, experiment
 from ..api_objects.experiment import helpers as experiment_helpers
-from ..api_objects.prompt import prompt_template
-from . import asyncio_support, engine, evaluation_result, report, rest_operations
-from .metrics import base_metric
-from .models import base_model, models_factory
-from .types import LLMTask, ScoringKeyMappingType
+from ..api_objects.prompt.chat import chat_prompt_template
+from ..api_objects.prompt import types as prompt_types
+from . import (
+    asyncio_support,
+    engine,
+    evaluation_result,
+    report,
+    rest_operations,
+    samplers,
+)
+from .metrics import base_metric, score_result
+from .models import ModelCapabilities, base_model, models_factory
+from .scorers import scorer_function, scorer_wrapper_metric
+from . import test_result
+from .types import ExperimentScoreFunction, LLMTask, ScoringKeyMappingType
+from .. import url_helpers
 
 LOGGER = logging.getLogger(__name__)
+MODALITY_SUPPORT_DOC_URL = (
+    "https://www.comet.com/docs/opik/evaluation/evaluate_multimodal"
+)
+
+
+def _try_notifying_about_experiment_completion(
+    experiment: experiment.Experiment,
+) -> None:
+    try:
+        experiment.experiments_rest_client.finish_experiments(ids=[experiment.id])
+    except Exception:
+        LOGGER.debug(
+            "Failed to notify backend about the experiment completion. Experiment ID: %s",
+            experiment.id,
+            exc_info=True,
+        )
+
+
+def _compute_experiment_scores(
+    experiment_scoring_functions: List[ExperimentScoreFunction],
+    test_results: List[test_result.TestResult],
+) -> List[score_result.ScoreResult]:
+    """Compute experiment-level scores from test results."""
+    if not experiment_scoring_functions or not test_results:
+        return []
+
+    all_scores: List[score_result.ScoreResult] = []
+    for score_function in experiment_scoring_functions:
+        try:
+            scores = score_function(test_results)
+            # Handle Union[ScoreResult, List[ScoreResult]]
+            if isinstance(scores, list):
+                all_scores.extend(scores)
+            else:
+                all_scores.append(scores)
+        except Exception as e:
+            LOGGER.warning(
+                "Failed to compute experiment score: %s",
+                e,
+                exc_info=True,
+            )
+
+    return all_scores
 
 
 def evaluate(
     dataset: dataset.Dataset,
     task: LLMTask,
     scoring_metrics: Optional[List[base_metric.BaseMetric]] = None,
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]] = None,
+    experiment_name_prefix: Optional[str] = None,
     experiment_name: Optional[str] = None,
     project_name: Optional[str] = None,
     experiment_config: Optional[Dict[str, Any]] = None,
     verbose: int = 1,
     nb_samples: Optional[int] = None,
     task_threads: int = 16,
-    prompt: Optional[Prompt] = None,
-    prompts: Optional[List[Prompt]] = None,
+    prompt: Optional[base_prompt.BasePrompt] = None,
+    prompts: Optional[List[base_prompt.BasePrompt]] = None,
     scoring_key_mapping: Optional[ScoringKeyMappingType] = None,
     dataset_item_ids: Optional[List[str]] = None,
+    dataset_sampler: Optional[samplers.BaseDatasetSampler] = None,
+    trial_count: int = 1,
+    experiment_scoring_functions: Optional[List[ExperimentScoreFunction]] = None,
 ) -> evaluation_result.EvaluationResult:
     """
-    Performs task evaluation on a given dataset.
+    Performs task evaluation on a given dataset. You can use either `scoring_metrics` or `scorer_functions` to calculate
+    evaluation metrics. The scorer functions doesn't require `scoring_key_mapping` and use reserved parameters
+    to receive inputs and outputs from the task.
 
     Args:
         dataset: An Opik dataset instance
@@ -39,6 +100,10 @@ def evaluate(
         task: A callable object that takes dict with dataset item content
             as input and returns dict which will later be used for scoring.
 
+        experiment_name_prefix: The prefix to be added to automatically generated experiment names to make them unique
+            but grouped under the same prefix. For example, if you set `experiment_name_prefix="my-experiment"`,
+            the first experiment created will be named `my-experiment-<unique-random-part>`.
+
         experiment_name: The name of the experiment associated with evaluation run.
             If None, a generated name will be used.
 
@@ -53,8 +118,16 @@ def evaluate(
             are mandatory in `task`-returned dictionary.
             If no value provided, the experiment won't have any scoring metrics.
 
+        scoring_functions: List of scorer functions to be executed during evaluation.
+            Each scorer function includes a scoring method that accepts predefined
+            arguments supplied by the evaluation engine:
+                • dataset_item — a dictionary containing the dataset item content,
+                • task_outputs — a dictionary containing the LLM task output.
+                • task_span - the data collected during the LLM task execution [optional].
+
         verbose: an integer value that controls evaluation output logs such as summary and tqdm progress bar.
-            0 - no outputs, 1 - outputs are enabled (default).
+            0 - no outputs, 1 - outputs are enabled (default), 2 - outputs are enabled and detailed statistics
+            are displayed.
 
         nb_samples: number of samples to evaluate. If no value is provided, all samples in the dataset will be evaluated.
 
@@ -73,9 +146,20 @@ def evaluate(
             `{"input": "user_question"}` to map the "user_question" key to "input".
 
         dataset_item_ids: list of dataset item ids to evaluate. If not provided, all samples in the dataset will be evaluated.
+
+        dataset_sampler: An instance of a dataset sampler that will be used to sample dataset items for evaluation.
+            If not provided, all samples in the dataset will be evaluated.
+
+        trial_count: number of times to run the task and evaluate the task output for every dataset item.
+
+        experiment_scoring_functions: List of callable functions that compute experiment-level scores.
+            Each function takes a list of TestResult objects and returns a list of ScoreResult objects.
+            These scores are computed after all test results are collected and represent aggregate
+            metrics across the entire experiment.
     """
-    if scoring_metrics is None:
-        scoring_metrics = []
+    experiment_scoring_functions = (
+        [] if experiment_scoring_functions is None else experiment_scoring_functions
+    )
 
     checked_prompts = experiment_helpers.handle_prompt_args(
         prompt=prompt,
@@ -84,6 +168,11 @@ def evaluate(
 
     client = opik_client.get_client_cached()
 
+    experiment_name = _use_or_create_experiment_name(
+        experiment_name=experiment_name,
+        experiment_name_prefix=experiment_name_prefix,
+    )
+
     experiment = client.create_experiment(
         name=experiment_name,
         dataset_name=dataset.name,
@@ -91,6 +180,13 @@ def evaluate(
         prompts=checked_prompts,
     )
 
+    # wrap scoring functions if any
+    scoring_metrics = _wrap_scoring_functions(
+        scoring_functions=scoring_functions,
+        scoring_metrics=scoring_metrics,
+        project_name=project_name,
+    )
+
     return _evaluate_task(
         client=client,
         experiment=experiment,
@@ -103,6 +199,9 @@ def evaluate(
         task_threads=task_threads,
         scoring_key_mapping=scoring_key_mapping,
         dataset_item_ids=dataset_item_ids,
+        dataset_sampler=dataset_sampler,
+        trial_count=trial_count,
+        experiment_scoring_functions=experiment_scoring_functions,
     )
 
 
@@ -119,6 +218,9 @@ def _evaluate_task(
     task_threads: int,
     scoring_key_mapping: Optional[ScoringKeyMappingType],
     dataset_item_ids: Optional[List[str]],
+    dataset_sampler: Optional[samplers.BaseDatasetSampler],
+    trial_count: int,
+    experiment_scoring_functions: List[ExperimentScoreFunction],
 ) -> evaluation_result.EvaluationResult:
     start_time = time.time()
 
@@ -126,51 +228,82 @@ def _evaluate_task(
         evaluation_engine = engine.EvaluationEngine(
             client=client,
             project_name=project_name,
-            experiment_=experiment,
             scoring_metrics=scoring_metrics,
             workers=task_threads,
             verbose=verbose,
             scoring_key_mapping=scoring_key_mapping,
         )
-        test_results = evaluation_engine.evaluate_llm_tasks(
+        test_results = evaluation_engine.evaluate_llm_task_on_dataset(
             dataset_=dataset,
             task=task,
             nb_samples=nb_samples,
             dataset_item_ids=dataset_item_ids,
+            dataset_sampler=dataset_sampler,
+            trial_count=trial_count,
+            experiment_=experiment,
         )
 
     total_time = time.time() - start_time
 
-    if verbose == 1:
-        report.display_experiment_results(dataset.name, total_time, test_results)
+    # Compute experiment scores
+    computed_experiment_scores = _compute_experiment_scores(
+        experiment_scoring_functions=experiment_scoring_functions,
+        test_results=test_results,
+    )
 
-    report.display_experiment_link(
+    if verbose >= 1:
+        report.display_experiment_results(
+            dataset.name, total_time, test_results, computed_experiment_scores
+        )
+
+    experiment_url = url_helpers.get_experiment_url_by_id(
         experiment_id=experiment.id,
         dataset_id=dataset.id,
         url_override=client.config.url_override,
     )
 
+    report.display_experiment_link(experiment_url=experiment_url)
+
     client.flush()
 
+    _try_notifying_about_experiment_completion(experiment)
+
+    # Log experiment scores to backend
+    if computed_experiment_scores:
+        experiment.log_experiment_scores(score_results=computed_experiment_scores)
+
     evaluation_result_ = evaluation_result.EvaluationResult(
         dataset_id=dataset.id,
         experiment_id=experiment.id,
         experiment_name=experiment.name,
         test_results=test_results,
+        experiment_url=experiment_url,
+        trial_count=trial_count,
+        experiment_scores=computed_experiment_scores,
     )
 
+    if verbose >= 2:
+        report.display_evaluation_scores_statistics(
+            dataset_name=dataset.name,
+            evaluation_results=evaluation_result_,
+        )
+
     return evaluation_result_
 
 
 def evaluate_experiment(
     experiment_name: str,
     scoring_metrics: List[base_metric.BaseMetric],
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]] = None,
     scoring_threads: int = 16,
     verbose: int = 1,
     scoring_key_mapping: Optional[ScoringKeyMappingType] = None,
     experiment_id: Optional[str] = None,
+    experiment_scoring_functions: Optional[List[ExperimentScoreFunction]] = None,
 ) -> evaluation_result.EvaluationResult:
-    """Update existing experiment with new evaluation metrics.
+    """Update the existing experiment with new evaluation metrics. You can use either `scoring_metrics` or `scorer_functions` to calculate
+    evaluation metrics. The scorer functions doesn't require `scoring_key_mapping` and use reserved parameters
+    to receive inputs and outputs from the task.
 
     Args:
         experiment_name: The name of the experiment to update.
@@ -181,15 +314,32 @@ def evaluate_experiment(
             of the `score` method in metrics that you need to find out which keys
             are mandatory in `task`-returned dictionary.
 
+        scoring_functions: List of scorer functions to be executed during evaluation.
+            Each scorer function includes a scoring method that accepts predefined
+            arguments supplied by the evaluation engine:
+                • dataset_item — a dictionary containing the dataset item content,
+                • task_outputs — a dictionary containing the LLM task output.
+                • task_span - the data collected during the LLM task execution [optional].
+
         scoring_threads: amount of thread workers to run scoring metrics.
 
         verbose: an integer value that controls evaluation output logs such as summary and tqdm progress bar.
 
         scoring_key_mapping: A dictionary that allows you to rename keys present in either the dataset item or the task output
-            so that they match the keys expected by the scoring metrics. For example if you have a dataset item with the following content:
+            so that they match the keys expected by the scoring metrics. For example, if you have a dataset item with the following content:
             {"user_question": "What is Opik ?"} and a scoring metric that expects a key "input", you can use scoring_key_mapping
             `{"input": "user_question"}` to map the "user_question" key to "input".
+
+        experiment_id: The ID of the experiment to evaluate. If not provided, the experiment will be evaluated based on the experiment name.
+
+        experiment_scoring_functions: List of callable functions that compute experiment-level scores.
+            Each function takes a list of TestResult objects and returns a list of ScoreResult objects.
+            These scores are computed after all test results are collected and represent aggregate
+            metrics across the entire experiment.
     """
+    experiment_scoring_functions = (
+        [] if experiment_scoring_functions is None else experiment_scoring_functions
+    )
     start_time = time.time()
 
     client = opik_client.get_client_cached()
@@ -202,10 +352,11 @@ def evaluate_experiment(
             client=client, experiment_name=experiment_name
         )
 
+    dataset_ = client.get_dataset(name=experiment.dataset_name)
+
     test_cases = rest_operations.get_experiment_test_cases(
-        client=client,
-        experiment_id=experiment.id,
-        dataset_id=experiment.dataset_id,
+        experiment_=experiment,
+        dataset_=dataset_,
         scoring_key_mapping=scoring_key_mapping,
     )
     first_trace_id = test_cases[0].trace_id
@@ -213,11 +364,17 @@ def evaluate_experiment(
         client=client, trace_id=first_trace_id
     )
 
+    # wrap scoring functions if any
+    scoring_metrics = _wrap_scoring_functions(
+        scoring_functions=scoring_functions,
+        scoring_metrics=scoring_metrics,
+        project_name=project_name,
+    )
+
     with asyncio_support.async_http_connections_expire_immediately():
         evaluation_engine = engine.EvaluationEngine(
             client=client,
             project_name=project_name,
-            experiment_=experiment,
             scoring_metrics=scoring_metrics,
             workers=scoring_threads,
             verbose=verbose,
@@ -229,50 +386,104 @@ def evaluate_experiment(
 
     total_time = time.time() - start_time
 
-    if verbose == 1:
+    # Compute experiment scores
+    computed_experiment_scores = _compute_experiment_scores(
+        experiment_scoring_functions=experiment_scoring_functions,
+        test_results=test_results,
+    )
+
+    if verbose >= 1:
         report.display_experiment_results(
-            experiment.dataset_name, total_time, test_results
+            dataset_.name,
+            total_time,
+            test_results,
+            computed_experiment_scores,
         )
 
-    report.display_experiment_link(
-        dataset_id=experiment.dataset_id,
+    experiment_url = url_helpers.get_experiment_url_by_id(
         experiment_id=experiment.id,
+        dataset_id=dataset_.id,
         url_override=client.config.url_override,
     )
 
+    report.display_experiment_link(experiment_url=experiment_url)
+
+    _try_notifying_about_experiment_completion(experiment)
+
+    # Log experiment scores to backend
+    if computed_experiment_scores:
+        experiment.log_experiment_scores(score_results=computed_experiment_scores)
+
     evaluation_result_ = evaluation_result.EvaluationResult(
-        dataset_id=experiment.dataset_id,
+        dataset_id=dataset_.id,
         experiment_id=experiment.id,
         experiment_name=experiment.name,
         test_results=test_results,
+        experiment_url=experiment_url,
+        trial_count=1,
+        experiment_scores=computed_experiment_scores,
     )
 
+    if verbose >= 2:
+        report.display_evaluation_scores_statistics(
+            dataset_name=dataset_.name,
+            evaluation_results=evaluation_result_,
+        )
+
     return evaluation_result_
 
 
 def _build_prompt_evaluation_task(
     model: base_model.OpikBaseModel, messages: List[Dict[str, Any]]
 ) -> Callable[[Dict[str, Any]], Dict[str, Any]]:
-    def _prompt_evaluation_task(prompt_variables: Dict[str, Any]) -> Dict[str, Any]:
-        processed_messages = []
-        for message in messages:
-            processed_messages.append(
-                {
-                    "role": message["role"],
-                    "content": prompt_template.PromptTemplate(
-                        message["content"],
-                        validate_placeholders=False,
-                        type=prompt_variables.get("type", "mustache"),
-                    ).format(**prompt_variables),
-                }
-            )
+    supported_modalities = cast(
+        prompt_types.SupportedModalities,
+        {
+            "vision": ModelCapabilities.supports_vision(
+                getattr(model, "model_name", None)
+            ),
+            "video": ModelCapabilities.supports_video(
+                getattr(model, "model_name", None)
+            ),
+        },
+    )
+    # Disable placeholder validation since we pass all dataset item fields to format()
+    chat_prompt_template_ = chat_prompt_template.ChatPromptTemplate(
+        messages=messages, validate_placeholders=False
+    )
 
-        llm_output = model.generate_provider_response(messages=processed_messages)
+    required_modalities = chat_prompt_template_.required_modalities()
+    unsupported_modalities = {
+        modality
+        for modality in required_modalities
+        if not supported_modalities.get(modality, False)
+    }
+
+    if unsupported_modalities:
+        modalities_list = ", ".join(sorted(unsupported_modalities))
+        LOGGER.warning(
+            "Model '%s' does not support %s content. Multimedia parts will be flattened "
+            "to text placeholders. See %s for supported models and customization options.",
+            getattr(model, "model_name", "unknown"),
+            modalities_list,
+            MODALITY_SUPPORT_DOC_URL,
+        )
 
-        return {
-            "input": processed_messages,
-            "output": llm_output.choices[0].message.content,
-        }
+    def _prompt_evaluation_task(prompt_variables: Dict[str, Any]) -> Dict[str, Any]:
+        template_type_override = prompt_variables.get("type")
+        processed_messages = chat_prompt_template_.format(
+            variables=prompt_variables,
+            supported_modalities=supported_modalities,
+            template_type=template_type_override,
+        )
+
+        with base_model.get_provider_response(
+            model_provider=model, messages=processed_messages
+        ) as llm_output:
+            return {
+                "input": processed_messages,
+                "output": llm_output.choices[0].message.content,
+            }
 
     return _prompt_evaluation_task
 
@@ -282,14 +493,19 @@ def evaluate_prompt(
     messages: List[Dict[str, Any]],
     model: Optional[Union[str, base_model.OpikBaseModel]] = None,
     scoring_metrics: Optional[List[base_metric.BaseMetric]] = None,
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]] = None,
+    experiment_name_prefix: Optional[str] = None,
     experiment_name: Optional[str] = None,
     project_name: Optional[str] = None,
     experiment_config: Optional[Dict[str, Any]] = None,
     verbose: int = 1,
     nb_samples: Optional[int] = None,
     task_threads: int = 16,
-    prompt: Optional[Prompt] = None,
+    prompt: Optional[base_prompt.BasePrompt] = None,
     dataset_item_ids: Optional[List[str]] = None,
+    dataset_sampler: Optional[samplers.BaseDatasetSampler] = None,
+    trial_count: int = 1,
+    experiment_scoring_functions: Optional[List[ExperimentScoreFunction]] = None,
 ) -> evaluation_result.EvaluationResult:
     """
     Performs prompt evaluation on a given dataset.
@@ -304,6 +520,17 @@ def evaluate_prompt(
         scoring_metrics: List of metrics to calculate during evaluation.
             The LLM input and output will be passed as arguments to each metric `score(...)` method.
 
+        scoring_functions: List of scorer functions to be executed during evaluation.
+            Each scorer function includes a scoring method that accepts predefined
+            arguments supplied by the evaluation engine:
+                • dataset_item — a dictionary containing the dataset item content,
+                • task_outputs — a dictionary containing the LLM task output.
+                • task_span - the data collected during the LLM task execution [optional].
+
+        experiment_name_prefix: The prefix to be added to automatically generated experiment names to make them unique
+            but grouped under the same prefix. For example, if you set `experiment_name_prefix="my-experiment"`,
+            the first experiment created will be named `my-experiment-<unique-random-part>`.
+
         experiment_name: name of the experiment.
 
         project_name: The name of the project to log data
@@ -319,28 +546,48 @@ def evaluate_prompt(
         prompt: Prompt object to link with experiment.
 
         dataset_item_ids: list of dataset item ids to evaluate. If not provided, all samples in the dataset will be evaluated.
+
+        dataset_sampler: An instance of a dataset sampler that will be used to sample dataset items for evaluation.
+            If not provided, all samples in the dataset will be evaluated.
+
+        trial_count: number of times to execute the prompt and evaluate the LLM output for every dataset item.
+
+        experiment_scoring_functions: List of callable functions that compute experiment-level scores.
+            Each function takes a list of TestResult objects and returns a list of ScoreResult objects.
+            These scores are computed after all test results are collected and represent aggregate
+            metrics across the entire experiment.
     """
+    experiment_scoring_functions = (
+        [] if experiment_scoring_functions is None else experiment_scoring_functions
+    )
     if isinstance(model, str):
-        model = models_factory.get(model_name=model)
+        opik_model = models_factory.get(model_name=model)
     elif not isinstance(model, base_model.OpikBaseModel):
         raise ValueError("`model` must be either a string or an OpikBaseModel instance")
+    else:
+        opik_model = model
 
     if experiment_config is None:
-        experiment_config = {"prompt_template": messages, "model": model.model_name}
+        experiment_config = {
+            "prompt_template": messages,
+            "model": opik_model.model_name,
+        }
     else:
         if "prompt_template" not in experiment_config:
             experiment_config["prompt_template"] = messages
 
         if "model" not in experiment_config:
-            experiment_config["model"] = model.model_name
-
-    if scoring_metrics is None:
-        scoring_metrics = []
+            experiment_config["model"] = opik_model.model_name
 
     client = opik_client.get_client_cached()
 
     prompts = [prompt] if prompt else None
 
+    experiment_name = _use_or_create_experiment_name(
+        experiment_name=experiment_name,
+        experiment_name_prefix=experiment_name_prefix,
+    )
+
     experiment = client.create_experiment(
         name=experiment_name,
         dataset_name=dataset.name,
@@ -348,45 +595,79 @@ def evaluate_prompt(
         prompts=prompts,
     )
 
+    # wrap scoring functions if any
+    scoring_metrics = _wrap_scoring_functions(
+        scoring_functions=scoring_functions,
+        scoring_metrics=scoring_metrics,
+        project_name=project_name,
+    )
+
     start_time = time.time()
 
     with asyncio_support.async_http_connections_expire_immediately():
         evaluation_engine = engine.EvaluationEngine(
             client=client,
             project_name=project_name,
-            experiment_=experiment,
             scoring_metrics=scoring_metrics,
             workers=task_threads,
             verbose=verbose,
             scoring_key_mapping=None,
         )
-        test_results = evaluation_engine.evaluate_llm_tasks(
+        test_results = evaluation_engine.evaluate_llm_task_on_dataset(
             dataset_=dataset,
-            task=_build_prompt_evaluation_task(model=model, messages=messages),
+            task=_build_prompt_evaluation_task(model=opik_model, messages=messages),
             nb_samples=nb_samples,
             dataset_item_ids=dataset_item_ids,
+            dataset_sampler=dataset_sampler,
+            trial_count=trial_count,
+            experiment_=experiment,
         )
 
     total_time = time.time() - start_time
 
-    if verbose == 1:
-        report.display_experiment_results(dataset.name, total_time, test_results)
+    # Compute experiment scores
+    computed_experiment_scores = _compute_experiment_scores(
+        experiment_scoring_functions=experiment_scoring_functions,
+        test_results=test_results,
+    )
+
+    if verbose >= 1:
+        report.display_experiment_results(
+            dataset.name, total_time, test_results, computed_experiment_scores
+        )
 
-    report.display_experiment_link(
+    experiment_url = url_helpers.get_experiment_url_by_id(
         experiment_id=experiment.id,
         dataset_id=dataset.id,
         url_override=client.config.url_override,
     )
 
+    report.display_experiment_link(experiment_url=experiment_url)
+
     client.flush()
 
+    _try_notifying_about_experiment_completion(experiment)
+
+    # Log experiment scores to backend
+    if computed_experiment_scores:
+        experiment.log_experiment_scores(score_results=computed_experiment_scores)
+
     evaluation_result_ = evaluation_result.EvaluationResult(
         experiment_id=experiment.id,
         dataset_id=dataset.id,
         experiment_name=experiment.name,
         test_results=test_results,
+        experiment_url=experiment_url,
+        trial_count=trial_count,
+        experiment_scores=computed_experiment_scores,
     )
 
+    if verbose >= 2:
+        report.display_evaluation_scores_statistics(
+            dataset_name=dataset.name,
+            evaluation_results=evaluation_result_,
+        )
+
     return evaluation_result_
 
 
@@ -395,16 +676,21 @@ def evaluate_optimization_trial(
     dataset: dataset.Dataset,
     task: LLMTask,
     scoring_metrics: Optional[List[base_metric.BaseMetric]] = None,
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]] = None,
+    experiment_name_prefix: Optional[str] = None,
     experiment_name: Optional[str] = None,
     project_name: Optional[str] = None,
     experiment_config: Optional[Dict[str, Any]] = None,
     verbose: int = 1,
     nb_samples: Optional[int] = None,
     task_threads: int = 16,
-    prompt: Optional[Prompt] = None,
-    prompts: Optional[List[Prompt]] = None,
+    prompt: Optional[base_prompt.BasePrompt] = None,
+    prompts: Optional[List[base_prompt.BasePrompt]] = None,
     scoring_key_mapping: Optional[ScoringKeyMappingType] = None,
     dataset_item_ids: Optional[List[str]] = None,
+    dataset_sampler: Optional[samplers.BaseDatasetSampler] = None,
+    trial_count: int = 1,
+    experiment_scoring_functions: Optional[List[ExperimentScoreFunction]] = None,
 ) -> evaluation_result.EvaluationResult:
     """
     Performs task evaluation on a given dataset.
@@ -417,6 +703,17 @@ def evaluate_optimization_trial(
         task: A callable object that takes dict with dataset item content
             as input and returns dict which will later be used for scoring.
 
+        scoring_functions: List of scorer functions to be executed during evaluation.
+            Each scorer function includes a scoring method that accepts predefined
+            arguments supplied by the evaluation engine:
+                • dataset_item — a dictionary containing the dataset item content,
+                • task_outputs — a dictionary containing the LLM task output.
+                • task_span - the data collected during the LLM task execution [optional].
+
+        experiment_name_prefix: The prefix to be added to automatically generated experiment names to make them unique
+                    but grouped under the same prefix. For example, if you set `experiment_name_prefix="my-experiment"`,
+                    the first experiment created will be named `my-experiment-<unique-random-part>`.
+
         experiment_name: The name of the experiment associated with evaluation run.
             If None, a generated name will be used.
 
@@ -451,7 +748,21 @@ def evaluate_optimization_trial(
             `{"input": "user_question"}` to map the "user_question" key to "input".
 
         dataset_item_ids: list of dataset item ids to evaluate. If not provided, all samples in the dataset will be evaluated.
+
+        dataset_sampler: An instance of a dataset sampler that will be used to sample dataset items for evaluation.
+            If not provided, all samples in the dataset will be evaluated.
+
+        trial_count: number of times to execute the prompt and evaluate the LLM output for every dataset item.
+
+        experiment_scoring_functions: List of callable functions that compute experiment-level scores.
+            Each function takes a list of TestResult objects and returns a list of ScoreResult objects.
+            These scores are computed after all test results are collected and represent aggregate
+            metrics across the entire experiment.
     """
+    experiment_scoring_functions = (
+        [] if experiment_scoring_functions is None else experiment_scoring_functions
+    )
+
     if scoring_metrics is None:
         scoring_metrics = []
 
@@ -460,8 +771,20 @@ def evaluate_optimization_trial(
         prompts=prompts,
     )
 
+    # wrap scoring functions if any
+    scoring_metrics = _wrap_scoring_functions(
+        scoring_functions=scoring_functions,
+        scoring_metrics=scoring_metrics,
+        project_name=project_name,
+    )
+
     client = opik_client.get_client_cached()
 
+    experiment_name = _use_or_create_experiment_name(
+        experiment_name=experiment_name,
+        experiment_name_prefix=experiment_name_prefix,
+    )
+
     experiment = client.create_experiment(
         name=experiment_name,
         dataset_name=dataset.name,
@@ -483,4 +806,152 @@ def evaluate_optimization_trial(
         task_threads=task_threads,
         scoring_key_mapping=scoring_key_mapping,
         dataset_item_ids=dataset_item_ids,
+        dataset_sampler=dataset_sampler,
+        trial_count=trial_count,
+        experiment_scoring_functions=experiment_scoring_functions,
     )
+
+
+def evaluate_on_dict_items(
+    items: List[Dict[str, Any]],
+    task: LLMTask,
+    scoring_metrics: Optional[List[base_metric.BaseMetric]] = None,
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]] = None,
+    project_name: Optional[str] = None,
+    verbose: int = 0,
+    scoring_key_mapping: Optional[ScoringKeyMappingType] = None,
+    scoring_threads: int = 16,
+) -> evaluation_result.EvaluationResultOnDictItems:
+    """
+    Lightweight evaluation function that evaluates a task on dataset items (as dictionaries)
+    without requiring a Dataset object or creating an experiment.
+
+    This function is useful for optimization scenarios where you need to evaluate many
+    candidate solutions quickly using Opik's metric infrastructure. It creates traces for
+    tracking but doesn't require experiment setup or dataset management.
+
+    Args:
+        items: List of dataset item contents (dictionaries with the data to evaluate).
+
+        task: A callable object that takes dict with dataset item content
+            as input and returns dict which will later be used for scoring.
+
+        scoring_metrics: List of metrics to calculate during evaluation.
+            Each metric's `score(...)` method will be called with arguments taken from
+            the dataset item and task output.
+
+        scoring_functions: List of scorer functions to be executed during evaluation.
+            Each scorer function accepts predefined arguments:
+                • dataset_item — a dictionary containing the dataset item content,
+                • task_outputs — a dictionary containing the LLM task output.
+
+        project_name: The name of the project for logging traces.
+
+        verbose: Controls evaluation output logs and progress bars.
+            0 - no outputs (default), 1 - enable outputs.
+
+        scoring_key_mapping: A dictionary that allows you to rename keys present in either
+            the dataset item or the task output to match the keys expected by scoring metrics.
+
+        scoring_threads: Number of thread workers to run scoring metrics.
+
+    Returns:
+        EvaluationResultOnDictItems object containing test results and providing methods
+        to aggregate scores, similar to the regular evaluation result.
+
+    Example:
+        ```python
+        import opik
+        from opik.evaluation.metrics import Equals
+
+        items = [
+            {"input": "What is 2+2?", "expected_output": "4"},
+            {"input": "What is 3+3?", "expected_output": "6"},
+        ]
+
+        def my_task(item):
+            # Your LLM call here
+            question = item["input"]
+            # ... call model ...
+            return {"output": model_output}
+
+        result = opik.evaluate_on_dict_items(
+            items=items,
+            task=my_task,
+            scoring_metrics=[Equals()],
+            scoring_key_mapping={"reference": "expected_output"},
+        )
+
+        # Access individual test results
+        for test_result in result.test_results:
+            print(f"Score: {test_result.score_results[0].value}")
+
+        # Get aggregated statistics
+        aggregated = result.aggregate_evaluation_scores()
+        print(f"Mean equals score: {aggregated['equals_metric'].mean}")
+        ```
+    """
+    # Wrap scoring functions if any
+    scoring_metrics = _wrap_scoring_functions(
+        scoring_functions=scoring_functions,
+        scoring_metrics=scoring_metrics,
+        project_name=project_name,
+    )
+
+    if not scoring_metrics:
+        LOGGER.warning("No scoring metrics provided for items evaluation")
+        return evaluation_result.EvaluationResultOnDictItems(test_results=[])
+
+    client = opik_client.get_client_cached()
+
+    # Create evaluation engine
+    with asyncio_support.async_http_connections_expire_immediately():
+        evaluation_engine = engine.EvaluationEngine(
+            client=client,
+            project_name=project_name,
+            scoring_metrics=scoring_metrics,
+            workers=scoring_threads,
+            verbose=verbose,
+            scoring_key_mapping=scoring_key_mapping,
+        )
+
+        # Use the new evaluate_items method
+        test_results = evaluation_engine.evaluate_llm_task_on_dict_items(
+            items=items,
+            task=task,
+        )
+
+    return evaluation_result.EvaluationResultOnDictItems(
+        test_results=test_results,
+    )
+
+
+def _wrap_scoring_functions(
+    scoring_functions: Optional[List[scorer_function.ScorerFunction]],
+    scoring_metrics: Optional[List[base_metric.BaseMetric]],
+    project_name: Optional[str],
+) -> List[base_metric.BaseMetric]:
+    if scoring_functions:
+        function_metrics = scorer_wrapper_metric.wrap_scorer_functions(
+            scoring_functions, project_name=project_name
+        )
+        if scoring_metrics:
+            scoring_metrics.extend(function_metrics)
+        else:
+            scoring_metrics = function_metrics
+
+    return scoring_metrics if scoring_metrics else []
+
+
+def _use_or_create_experiment_name(
+    experiment_name: Optional[str], experiment_name_prefix: Optional[str]
+) -> Optional[str]:
+    if experiment_name:
+        return experiment_name
+
+    if experiment_name_prefix:
+        return experiment_helpers.generate_unique_experiment_name(
+            experiment_name_prefix
+        )
+    else:
+        return None