opik-optimizer 2.0.1__py3-none-any.whl → 2.1.1__py3-none-any.whl

opik_optimizer/parameter_optimizer/parameter_spec.py

@@ -0,0 +1,214 @@
+"""Parameter specification for defining tunable parameters."""
+
+from __future__ import annotations
+
+import math
+from typing import Any, Literal
+from collections.abc import Mapping, Sequence
+
+from optuna.trial import Trial
+from pydantic import BaseModel, Field, PrivateAttr, model_validator
+
+from .search_space_types import ParameterType, ResolvedTarget
+
+
+class ParameterSpec(BaseModel):
+    """Definition for a single tunable parameter."""
+
+    name: str
+    description: str | None = None
+    distribution: ParameterType = Field(alias="type")
+    low: float | None = Field(default=None, alias="min")
+    high: float | None = Field(default=None, alias="max")
+    step: float | None = None
+    scale: Literal["linear", "log"] = "linear"
+    choices: list[Any] | None = None
+    target: str | Sequence[str] | None = None
+    default: Any | None = None
+
+    model_config = {
+        "populate_by_name": True,
+        "extra": "forbid",
+    }
+
+    _resolved_target: ResolvedTarget | None = PrivateAttr(default=None)
+
+    @model_validator(mode="before")
+    @classmethod
+    def _coerce_aliases(cls, data: Any) -> Any:
+        if isinstance(data, Mapping):
+            data = dict(data)
+            if "values" in data and "choices" not in data:
+                data["choices"] = data.pop("values")
+            if "selection" in data and "choices" not in data:
+                data["choices"] = data.pop("selection")
+            if "path" in data and "target" not in data:
+                data["target"] = data.pop("path")
+        return data
+
+    @model_validator(mode="after")
+    def _validate(self) -> ParameterSpec:
+        if self.distribution in {ParameterType.FLOAT, ParameterType.INT}:
+            if self.low is None or self.high is None:
+                raise ValueError(
+                    "'min' and 'max' must be provided for range parameters"
+                )
+            if self.low >= self.high:
+                raise ValueError("'min' must be less than 'max'")
+            if self.scale not in {"linear", "log"}:
+                raise ValueError("scale must be 'linear' or 'log'")
+            if self.scale == "log" and (self.low <= 0 or self.high <= 0):
+                raise ValueError("log-scaled parameters require positive bounds")
+            if self.step is not None and self.step <= 0:
+                raise ValueError("step must be positive when provided")
+
+            if self.distribution == ParameterType.INT:
+                object.__setattr__(self, "low", int(self.low))
+                object.__setattr__(self, "high", int(self.high))
+                if self.step is not None:
+                    object.__setattr__(self, "step", int(self.step))
+        elif self.distribution == ParameterType.CATEGORICAL:
+            if not self.choices:
+                raise ValueError("categorical parameters require non-empty 'choices'")
+        elif self.distribution == ParameterType.BOOL:
+            if not self.choices:
+                object.__setattr__(self, "choices", [False, True])
+        else:  # pragma: no cover - safety fallback
+            raise ValueError(f"Unsupported distribution: {self.distribution}")
+
+        object.__setattr__(self, "_resolved_target", self._resolve_target())
+        return self
+
+    @property
+    def target_path(self) -> ResolvedTarget:
+        if self._resolved_target is None:
+            self._resolved_target = self._resolve_target()
+        return self._resolved_target
+
+    def suggest(self, trial: Trial) -> Any:
+        """Return a sampled value for this parameter from Optuna."""
+        if self.distribution == ParameterType.FLOAT:
+            assert self.low is not None and self.high is not None  # validated earlier
+            return trial.suggest_float(
+                self.name,
+                float(self.low),
+                float(self.high),
+                step=self.step,
+                log=self.scale == "log",
+            )
+        if self.distribution == ParameterType.INT:
+            assert self.low is not None and self.high is not None  # validated earlier
+            return trial.suggest_int(
+                self.name,
+                int(self.low),
+                int(self.high),
+                step=int(self.step) if self.step is not None else 1,
+                log=self.scale == "log",
+            )
+        if self.distribution in {ParameterType.CATEGORICAL, ParameterType.BOOL}:
+            assert self.choices is not None  # guarded in validators
+            return trial.suggest_categorical(self.name, list(self.choices))
+        raise RuntimeError(f"Unsupported distribution type: {self.distribution}")
+
+    def apply_to_prompt(
+        self,
+        prompt: Any,
+        value: Any,  # ChatPrompt type
+    ) -> None:
+        """Apply a sampled value to the provided prompt instance."""
+        resolved = self.target_path
+        if resolved.root == "model":
+            if resolved.path:
+                raise ValueError("Nested paths under 'model' are not supported")
+            prompt.model = value
+            return
+
+        if prompt.model_kwargs is None:
+            prompt.model_kwargs = {}
+
+        self._assign_nested(prompt.model_kwargs, resolved.path, value)
+
+    def apply_to_model_kwargs(self, model_kwargs: dict[str, Any], value: Any) -> None:
+        """Apply a sampled value to a model_kwargs dictionary."""
+        resolved = self.target_path
+        if resolved.root != "model_kwargs":
+            return
+        self._assign_nested(model_kwargs, resolved.path, value)
+
+    def narrow(self, center: Any, scale: float) -> ParameterSpec:
+        """Return a narrowed version of the spec around the provided center."""
+
+        if center is None or scale <= 0:
+            return self
+
+        if self.distribution in {ParameterType.FLOAT, ParameterType.INT}:
+            if self.low is None or self.high is None:
+                return self
+
+            span = float(self.high) - float(self.low)
+            if span <= 0:
+                return self
+
+            half_window = span * float(scale) / 2
+            if half_window <= 0:
+                return self
+
+            center_val = float(center)
+            new_low = max(float(self.low), center_val - half_window)
+            new_high = min(float(self.high), center_val + half_window)
+
+            if self.distribution == ParameterType.INT:
+                new_low = math.floor(new_low)
+                new_high = math.ceil(new_high)
+                if new_low == new_high:
+                    new_high = min(int(self.high), new_low + 1)
+                if new_low == new_high:
+                    return self
+
+            if new_low >= new_high:
+                return self
+
+            spec_dict = self.model_dump(by_alias=True)
+            spec_dict["min"] = new_low
+            spec_dict["max"] = new_high
+            return ParameterSpec.model_validate(spec_dict)
+
+        # Non-numeric parameters remain unchanged
+        return self
+
+    def _assign_nested(
+        self, container: dict[str, Any], path: Sequence[str], value: Any
+    ) -> None:
+        if not path:
+            container[self.name] = value
+            return
+        current = container
+        for key in path[:-1]:
+            next_val = current.get(key)
+            if not isinstance(next_val, dict):
+                next_val = {}
+            current[key] = next_val
+            current = next_val
+        current[path[-1]] = value
+
+    def _resolve_target(self) -> ResolvedTarget:
+        target = self.target
+        if target is None:
+            return ResolvedTarget("model_kwargs", (self.name,))
+
+        if isinstance(target, str):
+            tokens = tuple(filter(None, (part.strip() for part in target.split("."))))
+        else:
+            tokens = tuple(target)
+
+        if not tokens:
+            return ResolvedTarget("model_kwargs", (self.name,))
+
+        root = tokens[0]
+        path = tokens[1:]
+
+        if root not in {"model", "model_kwargs"}:
+            root = "model_kwargs"
+            path = tokens
+
+        return ResolvedTarget(root, tuple(path))  # type: ignore[arg-type]

opik_optimizer/parameter_optimizer/search_space_types.py

@@ -0,0 +1,24 @@
+"""Type definitions for parameter search space."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+from typing import Literal
+
+
+class ParameterType(str, Enum):
+    """Supported parameter distribution types."""
+
+    FLOAT = "float"
+    INT = "int"
+    CATEGORICAL = "categorical"
+    BOOL = "bool"
+
+
+@dataclass(frozen=True)
+class ResolvedTarget:
+    """Resolved target location for a parameter."""
+
+    root: Literal["model", "model_kwargs"]
+    path: tuple[str, ...]

opik_optimizer/parameter_optimizer/sensitivity_analysis.py

@@ -0,0 +1,71 @@
+"""Sensitivity analysis utilities for parameter optimization."""
+
+from __future__ import annotations
+
+import math
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from optuna.trial import Trial
+
+    from .search_space import ParameterSpec
+
+
+def compute_sensitivity_from_trials(
+    trials: list[Trial], specs: list[ParameterSpec]
+) -> dict[str, float]:
+    """
+    Compute parameter sensitivity from completed trials.
+
+    This function calculates a correlation-based sensitivity measure for each parameter
+    by analyzing how changes in parameter values correlate with changes in the objective
+    function values across trials.
+
+    Args:
+        trials: List of completed Optuna trials
+        specs: List of parameter specifications
+
+    Returns:
+        Dictionary mapping parameter names to sensitivity scores (0.0 to 1.0)
+    """
+    sensitivities: dict[str, float] = {}
+
+    for spec in specs:
+        param_name = spec.name
+        values: list[float] = []
+        scores: list[float] = []
+
+        for trial in trials:
+            if trial.value is None:
+                continue
+
+            raw_value = trial.params.get(param_name)
+            if isinstance(raw_value, bool):
+                processed = float(int(raw_value))
+            elif isinstance(raw_value, (int, float)):
+                processed = float(raw_value)
+            else:
+                continue
+
+            values.append(processed)
+            scores.append(float(trial.value))
+
+        if len(values) < 2 or len(set(values)) == 1:
+            sensitivities[param_name] = 0.0
+            continue
+
+        mean_val = sum(values) / len(values)
+        mean_score = sum(scores) / len(scores)
+
+        cov = sum((v - mean_val) * (s - mean_score) for v, s in zip(values, scores))
+        var_val = sum((v - mean_val) ** 2 for v in values)
+        var_score = sum((s - mean_score) ** 2 for s in scores)
+
+        if var_val <= 0 or var_score <= 0:
+            sensitivities[param_name] = 0.0
+            continue
+
+        corr = abs(cov) / math.sqrt(var_val * var_score)
+        sensitivities[param_name] = min(max(corr, 0.0), 1.0)
+
+    return sensitivities

{opik_optimizer-2.0.1.dist-info → opik_optimizer-2.1.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 2.0.1
+Version: 2.1.1
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML
@@ -18,7 +18,7 @@ Requires-Dist: dspy<3
 Requires-Dist: gepa>=0.0.7
 Requires-Dist: ujson
 Requires-Dist: hf_xet
-Requires-Dist: litellm
+Requires-Dist: litellm<=1.75.6
 Requires-Dist: mcp>=1.0.0
 Requires-Dist: opik>=1.7.17
 Requires-Dist: optuna
@@ -32,6 +32,7 @@ Requires-Dist: pytest; extra == "dev"
 Requires-Dist: pytest-cov; extra == "dev"
 Requires-Dist: langgraph; extra == "dev"
 Requires-Dist: pre-commit; extra == "dev"
+Requires-Dist: scikit-learn; extra == "dev"
 Dynamic: author
 Dynamic: home-page
 Dynamic: license-file
@@ -51,6 +52,7 @@ The Opik Agent Optimizer refines your prompts to achieve better performance from
 * **MetaPromptOptimizer** - Employs meta-prompting techniques for optimization
 * **MiproOptimizer** - Implements MIPRO (Multi-Input Prompt Optimization) algorithm
 * **GepaOptimizer** - Leverages GEPA (Genetic-Pareto) optimization approach
+* **ParameterOptimizer** - Optimizes LLM call parameters (temperature, top_p, etc.) using Bayesian optimization
 
 ## 🎯 Key Features
 

{opik_optimizer-2.0.1.dist-info → opik_optimizer-2.1.1.dist-info}/RECORD

@@ -1,10 +1,10 @@
-opik_optimizer/__init__.py,sha256=bNNFoAJmORQ38vHPZsOb3m4Gb07-jjqPy-MbMIiTop8,1234
+opik_optimizer/__init__.py,sha256=VwryQ5bSOmJSl4CiacCIv_UF_In8Zho54fQ3FUR8pyk,1573
 opik_optimizer/_throttle.py,sha256=1JXIhYlo0IaqCgwmNB0Hnh9CYhYPkwRFdVGIcE7pVNg,1362
-opik_optimizer/base_optimizer.py,sha256=QPGLOzdlniSCIGOPpsz6_1CrvC0NEXmVQqmRIOwIv7E,20021
+opik_optimizer/base_optimizer.py,sha256=TKQknIvhJ1H5LOxhhkXIzjEepx3h0j0jyNsTGZ7EFLI,21410
 opik_optimizer/cache_config.py,sha256=Xd3NdUsL7bLQWoNe3pESqH4nHucU1iNTSGp-RqbwDog,599
 opik_optimizer/logging_config.py,sha256=TmxX0C1P20amxoXuiNQvlENOjdSNfWwvL8jFy206VWM,3837
 opik_optimizer/optimizable_agent.py,sha256=R0_BdwdHyZGWTw3oSvTg8FULDOYM8XaTiPNR3qV8DkQ,6344
-opik_optimizer/optimization_result.py,sha256=cscPGDNvkh88xAKhlU-nqaws-wXcYdJ7uwlCcshicwo,8112
+opik_optimizer/optimization_result.py,sha256=sG-Yr-hOaH9zx_I5S6_W3v6j8nPUhwYdS333jVM4Gus,17218
 opik_optimizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opik_optimizer/reporting_utils.py,sha256=dcECFmzZ_J-DKoukMDEE_fm7X8sdQyl_ijTddvQtepE,8287
 opik_optimizer/task_evaluator.py,sha256=1hILYwJLtn7XpPX96JjubnlMasmudVTHMVK3pmd22bE,4312
@@ -45,6 +45,12 @@ opik_optimizer/gepa_optimizer/__init__.py,sha256=XcPah5t4mop7UCFo69E9l45Mem49-it
 opik_optimizer/gepa_optimizer/adapter.py,sha256=KzPa4koq7aJhALMAOKPxAO4yWuEy_YbW7tGnqny3Hfo,5139
 opik_optimizer/gepa_optimizer/gepa_optimizer.py,sha256=DqTjkDeR7WEMt-53MSzDbD_72FRiwCCGxkwEyrLaQ4A,26430
 opik_optimizer/gepa_optimizer/reporting.py,sha256=F0cxYSjRuFAszgi3rgqwH1A-KH26kZOLtENP7x1xrQs,5154
+opik_optimizer/hierarchical_reflective_optimizer/__init__.py,sha256=9qM3kvfAaFy-Y6Tg19MXHJxpnF5DJQQwzr6oNsxaRBM,133
+opik_optimizer/hierarchical_reflective_optimizer/hierarchical_reflective_optimizer.py,sha256=Fs83ztOuPS8mkFvJAVmYok15DaXTk4Jqpoa9ImRl2t4,27256
+opik_optimizer/hierarchical_reflective_optimizer/hierarchical_root_cause_analyzer.py,sha256=GSIXUBxoS9LFnCXopS6B6wLSpmCYXA8Cv6ytELgEBoc,12709
+opik_optimizer/hierarchical_reflective_optimizer/prompts.py,sha256=XcOEI9eeEbTgKFsFiRWxvHdaByQkiN02bH2gTl3HX-Y,3853
+opik_optimizer/hierarchical_reflective_optimizer/reporting.py,sha256=LpHv_WBZCg2a0RhZaGwUmCch_-Dfk_rpuMxTckJMWTU,23234
+opik_optimizer/hierarchical_reflective_optimizer/types.py,sha256=bS-JAheX2FpJ4XAxoZi5PfjloG8L-B1LGQA1iLXZhW4,1031
 opik_optimizer/mcp_utils/__init__.py,sha256=BsWQT8nAa6JV6zcOD__OvPMepUS2IpJD4J2rnAXhpuU,710
 opik_optimizer/mcp_utils/mcp.py,sha256=UylgpTJsybszS433_kuTAgKH-PPde-VHjHVelMardFs,18466
 opik_optimizer/mcp_utils/mcp_second_pass.py,sha256=p2Knlxg1CKIZVMBbdskdRDqw1BRrnjM4gkcwAZtggm8,4519
@@ -62,13 +68,19 @@ opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 opik_optimizer/optimization_config/chat_prompt.py,sha256=d3jwM1UvUeRQOSsYHa5GD842VO3JWjVDmB3ROUGp57c,7089
 opik_optimizer/optimization_config/configs.py,sha256=EGacRNnl6TeWuf8RNsxpP6Nh5JhogjK-JxKllK8dQr0,413
 opik_optimizer/optimization_config/mappers.py,sha256=4uBoPaIvCo4bqt_w-4rJyVe2LMAP_W7p6xxnDmGT-Sk,1724
+opik_optimizer/parameter_optimizer/__init__.py,sha256=Eg-LEFBJqnOFw7i2B_YH27CoIGDPb5y_q1ar-ZpjtYo,308
+opik_optimizer/parameter_optimizer/parameter_optimizer.py,sha256=eDd9tFQinz2lKsEJtikCBVzSWMK4saI9bhUY2NtDEg0,14955
+opik_optimizer/parameter_optimizer/parameter_search_space.py,sha256=rgTNK8HPbdDiVm4GVX2QESTmQPhPFj4UkxqZfAy9JAA,4659
+opik_optimizer/parameter_optimizer/parameter_spec.py,sha256=HzYT_dHBTfZtx403mY-Epv_IEqn4kYuYBZ6QUdkFRiY,8064
+opik_optimizer/parameter_optimizer/search_space_types.py,sha256=UajTA2QKikEWazokDNO7j141gc2WxxYYiDRnFFjXi6M,512
+opik_optimizer/parameter_optimizer/sensitivity_analysis.py,sha256=8KEMVMHsmcoiK21Cq1-We6_Pw_6LX9qBX9Az4-tmj_w,2146
 opik_optimizer/utils/__init__.py,sha256=Ee0SnTPOcwRwp93M6Lh-X913lfSIwnvCiYYh5cpdRQE,486
 opik_optimizer/utils/colbert.py,sha256=qSrzKUUGw7P92mLy4Ofug5pBGeTsHBLMJXlXSJSfKuo,8147
 opik_optimizer/utils/core.py,sha256=5GT1vp6fW8ICO42LHMX14BjR-xEb6afAKjM7b1Evx5M,15298
 opik_optimizer/utils/dataset_utils.py,sha256=dqRUGOekjeNWL0J15R8xFwLyKJDJynJXzVyQmt8rhHA,1464
 opik_optimizer/utils/prompt_segments.py,sha256=1zUITSccJ82Njac1rmANzim4WWM6rVac61mfluS7lFE,5931
-opik_optimizer-2.0.1.dist-info/licenses/LICENSE,sha256=V-0VHJOBdcA_teT8VymvsBUQ1-CZU6yJRmMEjec_8tA,11372
-opik_optimizer-2.0.1.dist-info/METADATA,sha256=cboRfRa4vLcDFOog_nVkWaOLH3Xj6ukBElF94At9Q74,12665
-opik_optimizer-2.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-opik_optimizer-2.0.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
-opik_optimizer-2.0.1.dist-info/RECORD,,
+opik_optimizer-2.1.1.dist-info/licenses/LICENSE,sha256=V-0VHJOBdcA_teT8VymvsBUQ1-CZU6yJRmMEjec_8tA,11372
+opik_optimizer-2.1.1.dist-info/METADATA,sha256=CEmFvO0lRClgIYAItfH7V1XAJNBKcRwp8uq673-AzPo,12829
+opik_optimizer-2.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+opik_optimizer-2.1.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-2.1.1.dist-info/RECORD,,