opik-optimizer 2.0.1__py3-none-any.whl → 2.1.0__py3-none-any.whl

opik_optimizer/__init__.py

@@ -18,6 +18,12 @@ from .meta_prompt_optimizer import MetaPromptOptimizer
 from .mipro_optimizer import MiproOptimizer
 from .optimization_config.configs import TaskConfig
 from .optimization_result import OptimizationResult
+from .parameter_optimizer import (
+    ParameterOptimizer,
+    ParameterSearchSpace,
+    ParameterSpec,
+    ParameterType,
+)
 
 __version__ = importlib.metadata.version("opik_optimizer")
 
@@ -34,9 +40,13 @@ __all__ = [
     "MetaPromptOptimizer",
     "MiproOptimizer",
     "EvolutionaryOptimizer",
+    "ParameterOptimizer",
     "OptimizationResult",
     "OptimizableAgent",
     "setup_logging",
     "datasets",
     "TaskConfig",
+    "ParameterSearchSpace",
+    "ParameterSpec",
+    "ParameterType",
 ]

opik_optimizer/base_optimizer.py

@@ -470,6 +470,39 @@ class BaseOptimizer(ABC):
             f"{self.__class__.__name__} does not implement optimize_mcp yet."
         )
 
+    def optimize_parameter(
+        self,
+        prompt: "chat_prompt.ChatPrompt",
+        dataset: Dataset,
+        metric: Callable,
+        parameter_space: Any,
+        experiment_config: dict | None = None,
+        n_trials: int | None = None,
+        n_samples: int | None = None,
+        agent_class: type[OptimizableAgent] | None = None,
+        **kwargs: Any,
+    ) -> optimization_result.OptimizationResult:
+        """
+        Optimize LLM call parameters such as temperature or top_k.
+
+        Args:
+            prompt: The chat prompt to evaluate with tuned parameters
+            dataset: Dataset providing evaluation examples
+            metric: Objective function to maximize
+            parameter_space: Definition of the search space for tunable parameters
+            experiment_config: Optional experiment metadata
+            n_trials: Number of trials to run (optimizer specific default if None)
+            n_samples: Number of dataset samples to evaluate per trial (None for all)
+            agent_class: Optional custom agent class to execute evaluations
+            **kwargs: Additional optimizer specific settings
+
+        Returns:
+            OptimizationResult: Structured result describing the best parameters found
+        """
+        raise NotImplementedError(
+            f"{self.__class__.__name__} does not implement optimize_parameter yet."
+        )
+
     def get_history(self) -> list[OptimizationRound]:
         """
         Get the optimization history.

opik_optimizer/optimization_result.py

@@ -8,6 +8,13 @@ import rich
 from .reporting_utils import get_console, get_link_text, get_optimization_run_url_by_id
 
 
+def _format_float(value: Any, digits: int = 6) -> str:
+    """Format float values with specified precision."""
+    if isinstance(value, float):
+        return f"{value:.{digits}f}"
+    return str(value)
+
+
 class OptimizationResult(pydantic.BaseModel):
     """Result oan optimization run."""
 
@@ -44,6 +51,50 @@ class OptimizationResult(pydantic.BaseModel):
     def model_dump(self, *kargs: Any, **kwargs: Any) -> dict[str, Any]:
         return super().model_dump(*kargs, **kwargs)
 
+    def get_optimized_model_kwargs(self) -> dict[str, Any]:
+        """
+        Extract optimized model_kwargs for use in other optimizers.
+
+        Returns:
+            Dictionary of optimized model kwargs, empty dict if not available
+        """
+        return self.details.get("optimized_model_kwargs", {})
+
+    def get_optimized_model(self) -> str | None:
+        """
+        Extract optimized model name.
+
+        Returns:
+            Model name string if available, None otherwise
+        """
+        return self.details.get("optimized_model")
+
+    def get_optimized_parameters(self) -> dict[str, Any]:
+        """
+        Extract optimized parameter values.
+
+        Returns:
+            Dictionary of optimized parameters, empty dict if not available
+        """
+        return self.details.get("optimized_parameters", {})
+
+    def apply_to_prompt(self, prompt: Any) -> Any:
+        """
+        Apply optimized parameters to a prompt.
+
+        Args:
+            prompt: ChatPrompt instance to apply optimizations to
+
+        Returns:
+            New ChatPrompt instance with optimized parameters applied
+        """
+        prompt_copy = prompt.copy()
+        if "optimized_model_kwargs" in self.details:
+            prompt_copy.model_kwargs = self.details["optimized_model_kwargs"]
+        if "optimized_model" in self.details:
+            prompt_copy.model = self.details["optimized_model"]
+        return prompt_copy
+
     def _calculate_improvement_str(self) -> str:
         """Helper to calculate improvement percentage string."""
         initial_s = self.initial_score
@@ -113,12 +164,97 @@ class OptimizationResult(pydantic.BaseModel):
             f"Final Best Score: {final_score_str}",
             f"Total Improvement:{improvement_str.rjust(max(0, 18 - len('Total Improvement:')))}",
             f"Rounds Completed: {rounds_ran}",
-            "\nFINAL OPTIMIZED PROMPT / STRUCTURE:",
-            "--------------------------------------------------------------------------------",
-            f"{final_prompt_display}",
-            "--------------------------------------------------------------------------------",
-            f"{separator}",
         ]
+
+        optimized_params = self.details.get("optimized_parameters") or {}
+        parameter_importance = self.details.get("parameter_importance") or {}
+        search_ranges = self.details.get("search_ranges") or {}
+        precision = self.details.get("parameter_precision", 6)
+
+        if optimized_params:
+
+            def _format_range(desc: dict[str, Any]) -> str:
+                if "min" in desc and "max" in desc:
+                    step_str = (
+                        f", step={_format_float(desc['step'], precision)}"
+                        if desc.get("step") is not None
+                        else ""
+                    )
+                    return f"[{_format_float(desc['min'], precision)}, {_format_float(desc['max'], precision)}{step_str}]"
+                if desc.get("choices"):
+                    return f"choices={desc['choices']}"
+                return str(desc)
+
+            rows = []
+            stage_order = [
+                record.get("stage")
+                for record in self.details.get("search_stages", [])
+                if record.get("stage") in search_ranges
+            ]
+            if not stage_order:
+                stage_order = sorted(search_ranges)
+
+            for name in sorted(optimized_params):
+                contribution = parameter_importance.get(name)
+                stage_ranges = []
+                for stage in stage_order:
+                    params = search_ranges.get(stage) or {}
+                    if name in params:
+                        stage_ranges.append(f"{stage}: {_format_range(params[name])}")
+                if not stage_ranges:
+                    for stage, params in search_ranges.items():
+                        if name in params:
+                            stage_ranges.append(
+                                f"{stage}: {_format_range(params[name])}"
+                            )
+                joined_ranges = "\n".join(stage_ranges) if stage_ranges else "N/A"
+                rows.append(
+                    {
+                        "parameter": name,
+                        "value": optimized_params[name],
+                        "contribution": contribution,
+                        "ranges": joined_ranges,
+                    }
+                )
+
+            if rows:
+                output.append("Parameter Summary:")
+                # Compute overall improvement fraction for gain calculation
+                total_improvement = None
+                if isinstance(self.initial_score, (int, float)) and isinstance(
+                    self.score, (int, float)
+                ):
+                    if self.initial_score != 0:
+                        total_improvement = (self.score - self.initial_score) / abs(
+                            self.initial_score
+                        )
+                    else:
+                        total_improvement = self.score
+                for row in rows:
+                    value_str = _format_float(row["value"], precision)
+                    contrib_val = row["contribution"]
+                    if contrib_val is not None:
+                        contrib_percent = contrib_val * 100
+                        gain_str = ""
+                        if total_improvement is not None:
+                            gain_value = contrib_val * total_improvement * 100
+                            gain_str = f" ({gain_value:+.2f}%)"
+                        contrib_str = f"{contrib_percent:.1f}%{gain_str}"
+                    else:
+                        contrib_str = "N/A"
+                    output.append(
+                        f"- {row['parameter']}: value={value_str}, contribution={contrib_str}, ranges=\n  {row['ranges']}"
+                    )
+
+        output.extend(
+            [
+                "\nFINAL OPTIMIZED PROMPT / STRUCTURE:",
+                "--------------------------------------------------------------------------------",
+                f"{final_prompt_display}",
+                "--------------------------------------------------------------------------------",
+                f"{separator}",
+            ]
+        )
         return "\n".join(output)
 
     def __rich__(self) -> rich.panel.Panel:
@@ -159,6 +295,11 @@ class OptimizationResult(pydantic.BaseModel):
             ),
         )
 
+        optimized_params = self.details.get("optimized_parameters") or {}
+        parameter_importance = self.details.get("parameter_importance") or {}
+        search_ranges = self.details.get("search_ranges") or {}
+        precision = self.details.get("parameter_precision", 6)
+
         # Display Chat Structure if available
         panel_title = "[bold]Final Optimized Prompt[/bold]"
         try:
@@ -190,7 +331,87 @@ class OptimizationResult(pydantic.BaseModel):
             prompt_renderable, title=panel_title, border_style="blue", padding=(1, 2)
         )
 
-        content_group = rich.console.Group(table, "\n", prompt_panel)
+        renderables: list[rich.console.RenderableType] = [table, "\n"]
+
+        if optimized_params:
+            summary_table = rich.table.Table(
+                title="Parameter Summary", show_header=True, title_style="bold"
+            )
+            summary_table.add_column("Parameter", justify="left", style="cyan")
+            summary_table.add_column("Value", justify="left")
+            summary_table.add_column("Importance", justify="left", style="magenta")
+            summary_table.add_column("Gain", justify="left", style="dim")
+            summary_table.add_column("Ranges", justify="left")
+
+            stage_order = [
+                record.get("stage")
+                for record in self.details.get("search_stages", [])
+                if record.get("stage") in search_ranges
+            ]
+            if not stage_order:
+                stage_order = sorted(search_ranges)
+
+            def _format_range(desc: dict[str, Any]) -> str:
+                if "min" in desc and "max" in desc:
+                    step_str = (
+                        f", step={_format_float(desc['step'], precision)}"
+                        if desc.get("step") is not None
+                        else ""
+                    )
+                    return f"[{_format_float(desc['min'], precision)}, {_format_float(desc['max'], precision)}{step_str}]"
+                if desc.get("choices"):
+                    return ",".join(map(str, desc["choices"]))
+                return str(desc)
+
+            total_improvement = None
+            if isinstance(self.initial_score, (int, float)) and isinstance(
+                self.score, (int, float)
+            ):
+                if self.initial_score != 0:
+                    total_improvement = (self.score - self.initial_score) / abs(
+                        self.initial_score
+                    )
+                else:
+                    total_improvement = self.score
+
+            for name in sorted(optimized_params):
+                value_str = _format_float(optimized_params[name], precision)
+                contrib_val = parameter_importance.get(name)
+                if contrib_val is not None:
+                    contrib_str = f"{contrib_val:.1%}"
+                    gain_str = (
+                        f"{contrib_val * total_improvement:+.2%}"
+                        if total_improvement is not None
+                        else "N/A"
+                    )
+                else:
+                    contrib_str = "N/A"
+                    gain_str = "N/A"
+                ranges_parts = []
+                for stage in stage_order:
+                    params = search_ranges.get(stage) or {}
+                    if name in params:
+                        ranges_parts.append(f"{stage}: {_format_range(params[name])}")
+                if not ranges_parts:
+                    for stage, params in search_ranges.items():
+                        if name in params:
+                            ranges_parts.append(
+                                f"{stage}: {_format_range(params[name])}"
+                            )
+
+                summary_table.add_row(
+                    name,
+                    value_str,
+                    contrib_str,
+                    gain_str,
+                    "\n".join(ranges_parts) if ranges_parts else "N/A",
+                )
+
+            renderables.extend([summary_table, "\n"])
+
+        renderables.append(prompt_panel)
+
+        content_group = rich.console.Group(*renderables)
 
         return rich.panel.Panel(
             content_group,

opik_optimizer/parameter_optimizer/__init__.py

@@ -0,0 +1,11 @@
+from .parameter_optimizer import ParameterOptimizer
+from .parameter_search_space import ParameterSearchSpace
+from .parameter_spec import ParameterSpec
+from .search_space_types import ParameterType
+
+__all__ = [
+    "ParameterOptimizer",
+    "ParameterSearchSpace",
+    "ParameterSpec",
+    "ParameterType",
+]

opik_optimizer/parameter_optimizer/parameter_optimizer.py

@@ -0,0 +1,382 @@
+"""Simple Optuna-based optimizer for model parameter tuning."""
+
+from collections.abc import Callable, Mapping
+from typing import Any
+
+import copy
+import logging
+from datetime import datetime
+
+import optuna
+from optuna import importance as optuna_importance
+from optuna.trial import Trial, TrialState
+
+from opik import Dataset
+
+from ..base_optimizer import BaseOptimizer
+from ..optimizable_agent import OptimizableAgent
+from ..optimization_config import chat_prompt
+from ..optimization_result import OptimizationResult
+from .parameter_search_space import ParameterSearchSpace
+from .search_space_types import ParameterType
+from .sensitivity_analysis import compute_sensitivity_from_trials
+
+logger = logging.getLogger(__name__)
+
+
+class ParameterOptimizer(BaseOptimizer):
+    """Optimizer that tunes model call parameters (temperature, top_p, etc.)."""
+
+    def __init__(
+        self,
+        model: str,
+        *,
+        default_n_trials: int = 20,
+        n_threads: int = 4,
+        seed: int = 42,
+        verbose: int = 1,
+        local_search_ratio: float = 0.3,
+        local_search_scale: float = 0.2,
+        **model_kwargs: Any,
+    ) -> None:
+        super().__init__(model=model, verbose=verbose, seed=seed, **model_kwargs)
+        self.default_n_trials = default_n_trials
+        self.n_threads = n_threads
+        self.local_search_ratio = max(0.0, min(local_search_ratio, 1.0))
+        self.local_search_scale = max(0.0, local_search_scale)
+
+        if self.verbose == 0:
+            logger.setLevel(logging.WARNING)
+        elif self.verbose == 1:
+            logger.setLevel(logging.INFO)
+        else:
+            logger.setLevel(logging.DEBUG)
+
+    def optimize_prompt(
+        self,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: Dataset,
+        metric: Callable[[Any, Any], float],
+        experiment_config: dict | None = None,
+        n_samples: int | None = None,
+        auto_continue: bool = False,
+        agent_class: type[OptimizableAgent] | None = None,
+        **kwargs: Any,
+    ) -> OptimizationResult:
+        raise NotImplementedError(
+            "ParameterOptimizer.optimize_prompt is not supported. "
+            "Use optimize_parameter(prompt, dataset, metric, parameter_space) instead, "
+            "where parameter_space is a ParameterSearchSpace or dict defining the parameters to optimize."
+        )
+
+    def optimize_parameter(
+        self,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: Dataset,
+        metric: Callable[[Any, Any], float],
+        parameter_space: ParameterSearchSpace | Mapping[str, Any],
+        experiment_config: dict | None = None,
+        n_trials: int | None = None,
+        n_samples: int | None = None,
+        agent_class: type[OptimizableAgent] | None = None,
+        **kwargs: Any,
+    ) -> OptimizationResult:
+        if not isinstance(parameter_space, ParameterSearchSpace):
+            parameter_space = ParameterSearchSpace.model_validate(parameter_space)
+
+        # After validation, parameter_space is guaranteed to be ParameterSearchSpace
+        assert isinstance(parameter_space, ParameterSearchSpace)  # for mypy
+
+        sampler = kwargs.pop("sampler", None)
+        callbacks = kwargs.pop("callbacks", None)
+        timeout = kwargs.pop("timeout", None)
+        local_trials_override = kwargs.pop("local_trials", None)
+        local_search_scale_override = kwargs.pop("local_search_scale", None)
+        if kwargs:
+            extra_keys = ", ".join(sorted(kwargs.keys()))
+            raise TypeError(f"Unsupported keyword arguments: {extra_keys}")
+
+        self.validate_optimization_inputs(prompt, dataset, metric)
+        self.configure_prompt_model(prompt)
+
+        base_model_kwargs = copy.deepcopy(prompt.model_kwargs or {})
+        base_prompt = prompt.copy()
+        base_prompt.model_kwargs = copy.deepcopy(base_model_kwargs)
+
+        metric_name = getattr(metric, "__name__", str(metric))
+
+        self.agent_class = self.setup_agent_class(base_prompt, agent_class)
+        baseline_score = self.evaluate_prompt(
+            prompt=base_prompt,
+            dataset=dataset,
+            metric=metric,
+            n_threads=self.n_threads,
+            verbose=self.verbose,
+            experiment_config=experiment_config,
+            n_samples=n_samples,
+            agent_class=self.agent_class,
+        )
+
+        history: list[dict[str, Any]] = [
+            {
+                "iteration": 0,
+                "timestamp": datetime.utcnow().isoformat(),
+                "parameters": {},
+                "score": baseline_score,
+                "model_kwargs": copy.deepcopy(base_prompt.model_kwargs or {}),
+                "model": base_prompt.model,
+                "type": "baseline",
+                "stage": "baseline",
+            }
+        ]
+
+        try:
+            optuna.logging.disable_default_handler()
+            optuna_logger = logging.getLogger("optuna")
+            optuna_logger.setLevel(logger.getEffectiveLevel())
+            optuna_logger.propagate = False
+        except Exception as exc:  # pragma: no cover - defensive safety
+            logger.warning("Could not configure Optuna logging: %s", exc)
+
+        sampler = sampler or optuna.samplers.TPESampler(seed=self.seed)
+        study = optuna.create_study(direction="maximize", sampler=sampler)
+
+        total_trials = self.default_n_trials if n_trials is None else n_trials
+        if total_trials < 0:
+            total_trials = 0
+
+        if local_trials_override is not None:
+            local_trials = min(max(int(local_trials_override), 0), total_trials)
+        else:
+            local_trials = int(total_trials * self.local_search_ratio)
+
+        global_trials = total_trials - local_trials
+        if total_trials > 0 and global_trials <= 0:
+            global_trials = 1
+            local_trials = max(0, total_trials - global_trials)
+
+        current_space = parameter_space
+        current_stage = "global"
+        stage_records: list[dict[str, Any]] = []
+        search_ranges: dict[str, dict[str, Any]] = {}
+
+        def objective(trial: Trial) -> float:
+            sampled_values = current_space.suggest(trial)
+            tuned_prompt = parameter_space.apply(
+                prompt,
+                sampled_values,
+                base_model_kwargs=base_model_kwargs,
+            )
+            tuned_agent_class = self.setup_agent_class(tuned_prompt, agent_class)
+            score = self.evaluate_prompt(
+                prompt=tuned_prompt,
+                dataset=dataset,
+                metric=metric,
+                n_threads=self.n_threads,
+                verbose=self.verbose,
+                experiment_config=experiment_config,
+                n_samples=n_samples,
+                agent_class=tuned_agent_class,
+            )
+            trial.set_user_attr("parameters", sampled_values)
+            trial.set_user_attr(
+                "model_kwargs", copy.deepcopy(tuned_prompt.model_kwargs)
+            )
+            trial.set_user_attr("model", tuned_prompt.model)
+            trial.set_user_attr("stage", current_stage)
+            return float(score)
+
+        global_range = parameter_space.describe()
+        stage_records.append(
+            {
+                "stage": "global",
+                "trials": global_trials,
+                "scale": 1.0,
+                "parameters": global_range,
+            }
+        )
+        search_ranges["global"] = global_range
+
+        if global_trials > 0:
+            study.optimize(
+                objective,
+                n_trials=global_trials,
+                timeout=timeout,
+                callbacks=callbacks,
+                show_progress_bar=False,
+            )
+
+        for trial in study.trials:
+            if trial.state != TrialState.COMPLETE or trial.value is None:
+                continue
+            timestamp = (
+                trial.datetime_complete or trial.datetime_start or datetime.utcnow()
+            )
+            history.append(
+                {
+                    "iteration": trial.number + 1,
+                    "timestamp": timestamp.isoformat(),
+                    "parameters": trial.user_attrs.get("parameters", {}),
+                    "score": float(trial.value),
+                    "model_kwargs": trial.user_attrs.get("model_kwargs"),
+                    "model": trial.user_attrs.get("model"),
+                    "stage": trial.user_attrs.get("stage", "global"),
+                }
+            )
+
+        best_score = baseline_score
+        best_parameters: dict[str, Any] = {}
+        best_model_kwargs = copy.deepcopy(base_prompt.model_kwargs or {})
+        best_model = base_prompt.model
+
+        completed_trials = [
+            trial
+            for trial in study.trials
+            if trial.state == TrialState.COMPLETE and trial.value is not None
+        ]
+        if completed_trials:
+            best_trial = max(completed_trials, key=lambda t: t.value)  # type: ignore[arg-type]
+            if best_trial.value is not None and best_trial.value > best_score:
+                best_score = float(best_trial.value)
+                best_parameters = best_trial.user_attrs.get("parameters", {})
+                best_model_kwargs = best_trial.user_attrs.get("model_kwargs", {})
+                best_model = best_trial.user_attrs.get("model", prompt.model)
+
+        local_space: ParameterSearchSpace | None = None
+        if (
+            local_trials > 0
+            and completed_trials
+            and any(
+                spec.distribution in {ParameterType.FLOAT, ParameterType.INT}
+                for spec in parameter_space.parameters
+            )
+        ):
+            local_scale = (
+                self.local_search_scale
+                if local_search_scale_override is None
+                else max(0.0, float(local_search_scale_override))
+            )
+
+            if best_parameters:
+                center_values = best_parameters
+            elif base_model_kwargs:
+                center_values = base_model_kwargs
+            else:
+                center_values = {}
+
+            if local_scale > 0 and center_values:
+                current_stage = "local"
+                local_space = parameter_space.narrow_around(center_values, local_scale)
+                local_range = local_space.describe()
+                stage_records.append(
+                    {
+                        "stage": "local",
+                        "trials": local_trials,
+                        "scale": local_scale,
+                        "parameters": local_range,
+                    }
+                )
+                search_ranges["local"] = local_range
+
+                current_space = local_space
+                study.optimize(
+                    objective,
+                    n_trials=local_trials,
+                    timeout=timeout,
+                    callbacks=callbacks,
+                    show_progress_bar=False,
+                )
+
+                completed_trials = [
+                    trial
+                    for trial in study.trials
+                    if trial.state == TrialState.COMPLETE and trial.value is not None
+                ]
+                if completed_trials:
+                    new_best = max(completed_trials, key=lambda t: t.value)  # type: ignore[arg-type]
+                    if new_best.value is not None and new_best.value > best_score:
+                        best_score = float(new_best.value)
+                        best_parameters = new_best.user_attrs.get("parameters", {})
+                        best_model_kwargs = new_best.user_attrs.get("model_kwargs", {})
+                        best_model = new_best.user_attrs.get("model", prompt.model)
+
+        else:
+            local_trials = 0
+
+        for trial in study.trials:
+            if trial.state != TrialState.COMPLETE or trial.value is None:
+                continue
+            timestamp = (
+                trial.datetime_complete or trial.datetime_start or datetime.utcnow()
+            )
+            if not any(entry["iteration"] == trial.number + 1 for entry in history):
+                history.append(
+                    {
+                        "iteration": trial.number + 1,
+                        "timestamp": timestamp.isoformat(),
+                        "parameters": trial.user_attrs.get("parameters", {}),
+                        "score": float(trial.value),
+                        "model_kwargs": trial.user_attrs.get("model_kwargs"),
+                        "model": trial.user_attrs.get("model"),
+                        "stage": trial.user_attrs.get("stage", current_stage),
+                    }
+                )
+
+        rounds_summary = [
+            {
+                "iteration": trial.number + 1,
+                "parameters": trial.user_attrs.get("parameters", {}),
+                "score": float(trial.value) if trial.value is not None else None,
+                "model": trial.user_attrs.get("model"),
+                "stage": trial.user_attrs.get("stage"),
+            }
+            for trial in completed_trials
+        ]
+
+        try:
+            importance = optuna_importance.get_param_importances(study)
+        except (ValueError, RuntimeError, ImportError):
+            # Falls back to custom sensitivity analysis if:
+            # - Study has insufficient data (ValueError/RuntimeError)
+            # - scikit-learn not installed (ImportError)
+            importance = {}
+
+        if not importance or all(value == 0 for value in importance.values()):
+            importance = compute_sensitivity_from_trials(
+                completed_trials, parameter_space.parameters
+            )
+
+        details = {
+            "initial_score": baseline_score,
+            "optimized_parameters": best_parameters,
+            "optimized_model_kwargs": best_model_kwargs,
+            "optimized_model": best_model,
+            "trials": history,
+            "parameter_space": parameter_space.model_dump(by_alias=True),
+            "n_trials": total_trials,
+            "model": best_model,
+            "rounds": rounds_summary,
+            "baseline_parameters": base_model_kwargs,
+            "temperature": best_model_kwargs.get("temperature"),
+            "local_trials": local_trials,
+            "global_trials": global_trials,
+            "search_stages": stage_records,
+            "search_ranges": search_ranges,
+            "parameter_importance": importance,
+            "parameter_precision": 6,
+        }
+
+        return OptimizationResult(
+            optimizer=self.__class__.__name__,
+            prompt=prompt.get_messages() if hasattr(prompt, "get_messages") else [],
+            initial_prompt=prompt.get_messages()
+            if hasattr(prompt, "get_messages")
+            else [],
+            initial_score=baseline_score,
+            score=best_score,
+            metric_name=metric_name,
+            details=details,
+            history=history,
+            llm_calls=self.llm_call_counter,
+            tool_calls=self.tool_call_counter,
+        )

opik_optimizer/parameter_optimizer/parameter_search_space.py

@@ -0,0 +1,125 @@
+"""Parameter search space for collections of tunable parameters."""
+
+from __future__ import annotations
+
+import copy
+from typing import Any
+from collections.abc import Mapping, Sequence
+
+from optuna.trial import Trial
+from pydantic import BaseModel, Field, model_validator
+
+from .parameter_spec import ParameterSpec
+
+
+class ParameterSearchSpace(BaseModel):
+    """Collection of parameters to explore during optimization."""
+
+    parameters: list[ParameterSpec] = Field(default_factory=list)
+
+    model_config = {
+        "extra": "forbid",
+    }
+
+    @model_validator(mode="before")
+    @classmethod
+    def _normalize(cls, data: Any) -> Any:
+        if isinstance(data, ParameterSearchSpace):
+            return data
+        if isinstance(data, Mapping):
+            if "parameters" in data:
+                return data
+            parameters = []
+            for name, spec in data.items():
+                if isinstance(spec, Mapping):
+                    spec_dict = dict(spec)
+                elif isinstance(spec, ParameterSpec):
+                    spec_dict = spec.model_dump()
+                else:
+                    raise TypeError(
+                        "Parameter definitions must be mappings or ParameterSpec instances"
+                    )
+                spec_dict.setdefault("name", name)
+                parameters.append(spec_dict)
+            return {"parameters": parameters}
+        if isinstance(data, Sequence):
+            return {"parameters": list(data)}
+        return data
+
+    @model_validator(mode="after")
+    def _validate(self) -> ParameterSearchSpace:
+        names = [spec.name for spec in self.parameters]
+        if len(names) != len(set(names)):
+            duplicates = {name for name in names if names.count(name) > 1}
+            raise ValueError(
+                f"Duplicate parameter names detected: {', '.join(sorted(duplicates))}"
+            )
+        if not self.parameters:
+            raise ValueError("Parameter search space cannot be empty")
+        return self
+
+    def suggest(self, trial: Trial) -> dict[str, Any]:
+        """Sample a set of parameter values using an Optuna trial."""
+        return {spec.name: spec.suggest(trial) for spec in self.parameters}
+
+    def apply(
+        self,
+        prompt: Any,  # ChatPrompt type
+        values: Mapping[str, Any],
+        *,
+        base_model_kwargs: dict[str, Any] | None = None,
+    ) -> Any:  # Returns ChatPrompt
+        """Return a prompt copy with sampled values applied."""
+        prompt_copy = prompt.copy()
+        if base_model_kwargs is not None:
+            prompt_copy.model_kwargs = copy.deepcopy(base_model_kwargs)
+        for spec in self.parameters:
+            if spec.name in values:
+                spec.apply_to_prompt(prompt_copy, values[spec.name])
+        return prompt_copy
+
+    def values_to_model_kwargs(
+        self,
+        values: Mapping[str, Any],
+        *,
+        base: dict[str, Any] | None = None,
+    ) -> dict[str, Any]:
+        """Produce a model_kwargs dictionary with sampled values applied."""
+        model_kwargs = copy.deepcopy(base) if base is not None else {}
+        for spec in self.parameters:
+            if spec.name in values:
+                spec.apply_to_model_kwargs(model_kwargs, values[spec.name])
+        return model_kwargs
+
+    def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
+        """Ensure dumping keeps parameter definitions accessible."""
+        return super().model_dump(*args, **kwargs)
+
+    def narrow_around(
+        self, values: Mapping[str, Any], scale: float
+    ) -> ParameterSearchSpace:
+        """Return a new search space narrowed around provided parameter values."""
+
+        narrowed: list[ParameterSpec] = []
+        for spec in self.parameters:
+            value = values.get(spec.name)
+            narrowed.append(spec.narrow(value, scale))
+        return ParameterSearchSpace(parameters=narrowed)
+
+    def describe(self) -> dict[str, dict[str, Any]]:
+        """Return a human-friendly description of each parameter range."""
+
+        summary: dict[str, dict[str, Any]] = {}
+        for spec in self.parameters:
+            entry: dict[str, Any] = {"type": spec.distribution.value}
+            if spec.distribution.value in {"float", "int"}:
+                entry["min"] = spec.low
+                entry["max"] = spec.high
+                if spec.step is not None:
+                    entry["step"] = spec.step
+                entry["scale"] = spec.scale
+            else:
+                if spec.choices is not None:
+                    entry["choices"] = list(spec.choices)
+            summary[spec.name] = entry
+        return summary

opik_optimizer/parameter_optimizer/parameter_spec.py

@@ -0,0 +1,214 @@
+"""Parameter specification for defining tunable parameters."""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Literal
+from collections.abc import Mapping, Sequence
+
+from optuna.trial import Trial
+from pydantic import BaseModel, Field, PrivateAttr, model_validator
+
+from .search_space_types import ParameterType, ResolvedTarget
+
+
+class ParameterSpec(BaseModel):
+    """Definition for a single tunable parameter."""
+
+    name: str
+    description: str | None = None
+    distribution: ParameterType = Field(alias="type")
+    low: float | None = Field(default=None, alias="min")
+    high: float | None = Field(default=None, alias="max")
+    step: float | None = None
+    scale: Literal["linear", "log"] = "linear"
+    choices: list[Any] | None = None
+    target: str | Sequence[str] | None = None
+    default: Any | None = None
+
+    model_config = {
+        "populate_by_name": True,
+        "extra": "forbid",
+    }
+
+    _resolved_target: ResolvedTarget | None = PrivateAttr(default=None)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _coerce_aliases(cls, data: Any) -> Any:
+        if isinstance(data, Mapping):
+            data = dict(data)
+            if "values" in data and "choices" not in data:
+                data["choices"] = data.pop("values")
+            if "selection" in data and "choices" not in data:
+                data["choices"] = data.pop("selection")
+            if "path" in data and "target" not in data:
+                data["target"] = data.pop("path")
+        return data
+
+    @model_validator(mode="after")
+    def _validate(self) -> ParameterSpec:
+        if self.distribution in {ParameterType.FLOAT, ParameterType.INT}:
+            if self.low is None or self.high is None:
+                raise ValueError(
+                    "'min' and 'max' must be provided for range parameters"
+                )
+            if self.low >= self.high:
+                raise ValueError("'min' must be less than 'max'")
+            if self.scale not in {"linear", "log"}:
+                raise ValueError("scale must be 'linear' or 'log'")
+            if self.scale == "log" and (self.low <= 0 or self.high <= 0):
+                raise ValueError("log-scaled parameters require positive bounds")
+            if self.step is not None and self.step <= 0:
+                raise ValueError("step must be positive when provided")
+
+            if self.distribution == ParameterType.INT:
+                object.__setattr__(self, "low", int(self.low))
+                object.__setattr__(self, "high", int(self.high))
+                if self.step is not None:
+                    object.__setattr__(self, "step", int(self.step))
+        elif self.distribution == ParameterType.CATEGORICAL:
+            if not self.choices:
+                raise ValueError("categorical parameters require non-empty 'choices'")
+        elif self.distribution == ParameterType.BOOL:
+            if not self.choices:
+                object.__setattr__(self, "choices", [False, True])
+        else:  # pragma: no cover - safety fallback
+            raise ValueError(f"Unsupported distribution: {self.distribution}")
+
+        object.__setattr__(self, "_resolved_target", self._resolve_target())
+        return self
+
+    @property
+    def target_path(self) -> ResolvedTarget:
+        if self._resolved_target is None:
+            self._resolved_target = self._resolve_target()
+        return self._resolved_target
+
+    def suggest(self, trial: Trial) -> Any:
+        """Return a sampled value for this parameter from Optuna."""
+        if self.distribution == ParameterType.FLOAT:
+            assert self.low is not None and self.high is not None  # validated earlier
+            return trial.suggest_float(
+                self.name,
+                float(self.low),
+                float(self.high),
+                step=self.step,
+                log=self.scale == "log",
+            )
+        if self.distribution == ParameterType.INT:
+            assert self.low is not None and self.high is not None  # validated earlier
+            return trial.suggest_int(
+                self.name,
+                int(self.low),
+                int(self.high),
+                step=int(self.step) if self.step is not None else 1,
+                log=self.scale == "log",
+            )
+        if self.distribution in {ParameterType.CATEGORICAL, ParameterType.BOOL}:
+            assert self.choices is not None  # guarded in validators
+            return trial.suggest_categorical(self.name, list(self.choices))
+        raise RuntimeError(f"Unsupported distribution type: {self.distribution}")
+
+    def apply_to_prompt(
+        self,
+        prompt: Any,
+        value: Any,  # ChatPrompt type
+    ) -> None:
+        """Apply a sampled value to the provided prompt instance."""
+        resolved = self.target_path
+        if resolved.root == "model":
+            if resolved.path:
+                raise ValueError("Nested paths under 'model' are not supported")
+            prompt.model = value
+            return
+
+        if prompt.model_kwargs is None:
+            prompt.model_kwargs = {}
+
+        self._assign_nested(prompt.model_kwargs, resolved.path, value)
+
+    def apply_to_model_kwargs(self, model_kwargs: dict[str, Any], value: Any) -> None:
+        """Apply a sampled value to a model_kwargs dictionary."""
+        resolved = self.target_path
+        if resolved.root != "model_kwargs":
+            return
+        self._assign_nested(model_kwargs, resolved.path, value)
+
+    def narrow(self, center: Any, scale: float) -> ParameterSpec:
+        """Return a narrowed version of the spec around the provided center."""
+
+        if center is None or scale <= 0:
+            return self
+
+        if self.distribution in {ParameterType.FLOAT, ParameterType.INT}:
+            if self.low is None or self.high is None:
+                return self
+
+            span = float(self.high) - float(self.low)
+            if span <= 0:
+                return self
+
+            half_window = span * float(scale) / 2
+            if half_window <= 0:
+                return self
+
+            center_val = float(center)
+            new_low = max(float(self.low), center_val - half_window)
+            new_high = min(float(self.high), center_val + half_window)
+
+            if self.distribution == ParameterType.INT:
+                new_low = math.floor(new_low)
+                new_high = math.ceil(new_high)
+                if new_low == new_high:
+                    new_high = min(int(self.high), new_low + 1)
+                if new_low == new_high:
+                    return self
+
+            if new_low >= new_high:
+                return self
+
+            spec_dict = self.model_dump(by_alias=True)
+            spec_dict["min"] = new_low
+            spec_dict["max"] = new_high
+            return ParameterSpec.model_validate(spec_dict)
+
+        # Non-numeric parameters remain unchanged
+        return self
+
+    def _assign_nested(
+        self, container: dict[str, Any], path: Sequence[str], value: Any
+    ) -> None:
+        if not path:
+            container[self.name] = value
+            return
+        current = container
+        for key in path[:-1]:
+            next_val = current.get(key)
+            if not isinstance(next_val, dict):
+                next_val = {}
+            current[key] = next_val
+            current = next_val
+        current[path[-1]] = value
+
+    def _resolve_target(self) -> ResolvedTarget:
+        target = self.target
+        if target is None:
+            return ResolvedTarget("model_kwargs", (self.name,))
+
+        if isinstance(target, str):
+            tokens = tuple(filter(None, (part.strip() for part in target.split("."))))
+        else:
+            tokens = tuple(target)
+
+        if not tokens:
+            return ResolvedTarget("model_kwargs", (self.name,))
+
+        root = tokens[0]
+        path = tokens[1:]
+
+        if root not in {"model", "model_kwargs"}:
+            root = "model_kwargs"
+            path = tokens
+
+        return ResolvedTarget(root, tuple(path))  # type: ignore[arg-type]

opik_optimizer/parameter_optimizer/search_space_types.py

@@ -0,0 +1,24 @@
+"""Type definitions for parameter search space."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Literal
+
+
+class ParameterType(str, Enum):
+    """Supported parameter distribution types."""
+
+    FLOAT = "float"
+    INT = "int"
+    CATEGORICAL = "categorical"
+    BOOL = "bool"
+
+
+@dataclass(frozen=True)
+class ResolvedTarget:
+    """Resolved target location for a parameter."""
+
+    root: Literal["model", "model_kwargs"]
+    path: tuple[str, ...]

opik_optimizer/parameter_optimizer/sensitivity_analysis.py

@@ -0,0 +1,71 @@
+"""Sensitivity analysis utilities for parameter optimization."""
+
+from __future__ import annotations
+
+import math
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from optuna.trial import Trial
+
+    from .search_space import ParameterSpec
+
+
+def compute_sensitivity_from_trials(
+    trials: list[Trial], specs: list[ParameterSpec]
+) -> dict[str, float]:
+    """
+    Compute parameter sensitivity from completed trials.
+
+    This function calculates a correlation-based sensitivity measure for each parameter
+    by analyzing how changes in parameter values correlate with changes in the objective
+    function values across trials.
+
+    Args:
+        trials: List of completed Optuna trials
+        specs: List of parameter specifications
+
+    Returns:
+        Dictionary mapping parameter names to sensitivity scores (0.0 to 1.0)
+    """
+    sensitivities: dict[str, float] = {}
+
+    for spec in specs:
+        param_name = spec.name
+        values: list[float] = []
+        scores: list[float] = []
+
+        for trial in trials:
+            if trial.value is None:
+                continue
+
+            raw_value = trial.params.get(param_name)
+            if isinstance(raw_value, bool):
+                processed = float(int(raw_value))
+            elif isinstance(raw_value, (int, float)):
+                processed = float(raw_value)
+            else:
+                continue
+
+            values.append(processed)
+            scores.append(float(trial.value))
+
+        if len(values) < 2 or len(set(values)) == 1:
+            sensitivities[param_name] = 0.0
+            continue
+
+        mean_val = sum(values) / len(values)
+        mean_score = sum(scores) / len(scores)
+
+        cov = sum((v - mean_val) * (s - mean_score) for v, s in zip(values, scores))
+        var_val = sum((v - mean_val) ** 2 for v in values)
+        var_score = sum((s - mean_score) ** 2 for s in scores)
+
+        if var_val <= 0 or var_score <= 0:
+            sensitivities[param_name] = 0.0
+            continue
+
+        corr = abs(cov) / math.sqrt(var_val * var_score)
+        sensitivities[param_name] = min(max(corr, 0.0), 1.0)
+
+    return sensitivities

{opik_optimizer-2.0.1.dist-info → opik_optimizer-2.1.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 2.0.1
+Version: 2.1.0
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML
@@ -32,6 +32,7 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: langgraph; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: scikit-learn; extra == "dev"
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file
@@ -51,6 +52,7 @@ The Opik Agent Optimizer refines your prompts to achieve better performance from
 * **MetaPromptOptimizer** - Employs meta-prompting techniques for optimization
 * **MiproOptimizer** - Implements MIPRO (Multi-Input Prompt Optimization) algorithm
 * **GepaOptimizer** - Leverages GEPA (Genetic-Pareto) optimization approach
+* **ParameterOptimizer** - Optimizes LLM call parameters (temperature, top_p, etc.) using Bayesian optimization
 
 ## 🎯 Key Features
 

{opik_optimizer-2.0.1.dist-info → opik_optimizer-2.1.0.dist-info}/RECORD

@@ -1,10 +1,10 @@
-opik_optimizer/__init__.py,sha256=bNNFoAJmORQ38vHPZsOb3m4Gb07-jjqPy-MbMIiTop8,1234
+opik_optimizer/__init__.py,sha256=Vhvp9GwFPPxIGLa6dV12GAdN9WFt3a4rK1FjRsh5kn4,1455
 opik_optimizer/_throttle.py,sha256=1JXIhYlo0IaqCgwmNB0Hnh9CYhYPkwRFdVGIcE7pVNg,1362
-opik_optimizer/base_optimizer.py,sha256=QPGLOzdlniSCIGOPpsz6_1CrvC0NEXmVQqmRIOwIv7E,20021
+opik_optimizer/base_optimizer.py,sha256=TKQknIvhJ1H5LOxhhkXIzjEepx3h0j0jyNsTGZ7EFLI,21410
 opik_optimizer/cache_config.py,sha256=Xd3NdUsL7bLQWoNe3pESqH4nHucU1iNTSGp-RqbwDog,599
 opik_optimizer/logging_config.py,sha256=TmxX0C1P20amxoXuiNQvlENOjdSNfWwvL8jFy206VWM,3837
 opik_optimizer/optimizable_agent.py,sha256=R0_BdwdHyZGWTw3oSvTg8FULDOYM8XaTiPNR3qV8DkQ,6344
-opik_optimizer/optimization_result.py,sha256=cscPGDNvkh88xAKhlU-nqaws-wXcYdJ7uwlCcshicwo,8112
+opik_optimizer/optimization_result.py,sha256=sG-Yr-hOaH9zx_I5S6_W3v6j8nPUhwYdS333jVM4Gus,17218
 opik_optimizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opik_optimizer/reporting_utils.py,sha256=dcECFmzZ_J-DKoukMDEE_fm7X8sdQyl_ijTddvQtepE,8287
 opik_optimizer/task_evaluator.py,sha256=1hILYwJLtn7XpPX96JjubnlMasmudVTHMVK3pmd22bE,4312
@@ -62,13 +62,19 @@ opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 opik_optimizer/optimization_config/chat_prompt.py,sha256=d3jwM1UvUeRQOSsYHa5GD842VO3JWjVDmB3ROUGp57c,7089
 opik_optimizer/optimization_config/configs.py,sha256=EGacRNnl6TeWuf8RNsxpP6Nh5JhogjK-JxKllK8dQr0,413
 opik_optimizer/optimization_config/mappers.py,sha256=4uBoPaIvCo4bqt_w-4rJyVe2LMAP_W7p6xxnDmGT-Sk,1724
+opik_optimizer/parameter_optimizer/__init__.py,sha256=Eg-LEFBJqnOFw7i2B_YH27CoIGDPb5y_q1ar-ZpjtYo,308
+opik_optimizer/parameter_optimizer/parameter_optimizer.py,sha256=eDd9tFQinz2lKsEJtikCBVzSWMK4saI9bhUY2NtDEg0,14955
+opik_optimizer/parameter_optimizer/parameter_search_space.py,sha256=rgTNK8HPbdDiVm4GVX2QESTmQPhPFj4UkxqZfAy9JAA,4659
+opik_optimizer/parameter_optimizer/parameter_spec.py,sha256=HzYT_dHBTfZtx403mY-Epv_IEqn4kYuYBZ6QUdkFRiY,8064
+opik_optimizer/parameter_optimizer/search_space_types.py,sha256=UajTA2QKikEWazokDNO7j141gc2WxxYYiDRnFFjXi6M,512
+opik_optimizer/parameter_optimizer/sensitivity_analysis.py,sha256=8KEMVMHsmcoiK21Cq1-We6_Pw_6LX9qBX9Az4-tmj_w,2146
 opik_optimizer/utils/__init__.py,sha256=Ee0SnTPOcwRwp93M6Lh-X913lfSIwnvCiYYh5cpdRQE,486
 opik_optimizer/utils/colbert.py,sha256=qSrzKUUGw7P92mLy4Ofug5pBGeTsHBLMJXlXSJSfKuo,8147
 opik_optimizer/utils/core.py,sha256=5GT1vp6fW8ICO42LHMX14BjR-xEb6afAKjM7b1Evx5M,15298
 opik_optimizer/utils/dataset_utils.py,sha256=dqRUGOekjeNWL0J15R8xFwLyKJDJynJXzVyQmt8rhHA,1464
 opik_optimizer/utils/prompt_segments.py,sha256=1zUITSccJ82Njac1rmANzim4WWM6rVac61mfluS7lFE,5931
-opik_optimizer-2.0.1.dist-info/licenses/LICENSE,sha256=V-0VHJOBdcA_teT8VymvsBUQ1-CZU6yJRmMEjec_8tA,11372
-opik_optimizer-2.0.1.dist-info/METADATA,sha256=cboRfRa4vLcDFOog_nVkWaOLH3Xj6ukBElF94At9Q74,12665
-opik_optimizer-2.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-opik_optimizer-2.0.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
-opik_optimizer-2.0.1.dist-info/RECORD,,
+opik_optimizer-2.1.0.dist-info/licenses/LICENSE,sha256=V-0VHJOBdcA_teT8VymvsBUQ1-CZU6yJRmMEjec_8tA,11372
+opik_optimizer-2.1.0.dist-info/METADATA,sha256=4ZKzeqD4iIGwgS2CKYRlF7MYK5Zink37AVOU_D3U9gY,12821
+opik_optimizer-2.1.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+opik_optimizer-2.1.0.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-2.1.0.dist-info/RECORD,,