opik-optimizer 0.7.0__tar.gz

opik_optimizer-0.7.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Comet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

opik_optimizer-0.7.0/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.7.0
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: info@comet.ml
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Provides-Extra: dev
+Requires-Dist: adalflow; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

opik_optimizer-0.7.0/README.md

@@ -0,0 +1,133 @@
+# Opik Optimizer
+
+The Opik Opitmizer can refine your prompts to get better performance
+from your LLMs. You can use a variety of algorithms, including:
+
+* FewShotBayesianOptimizer
+* MiproOptimizer
+* MetaPromptOptimizer
+
+## Quickstart
+
+
+[Open Quickstart Notebook in Colab](https://colab.research.google.com/github/comet-ml/opik/blob/main/sdks/opik_optimizer/notebooks/OpikOptimizerIntro.ipynb)
+
+
+## Setup
+
+1. Configure Opik:
+   ```bash
+   # Install Comet ML CLI
+   pip install opik
+
+   # Configure your API key
+   opik configure
+   # When prompted, enter your Opik API key
+   ```
+
+2. Set up your environment variables:
+   ```bash
+   # OpenAI API key for LLM access
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. Install the package:
+   ```bash
+   pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+   ```
+
+You'll need:
+
+1. An LLM model name
+2. An Opik Dataset (or Opik Dataset name)
+3. An Opik Metric (possibly a custom one)
+4. A starting prompt (string)
+
+## Example
+
+We have prepared some sample datasets for testing:
+
+* "tiny-test"
+* "halu-eval-300"
+* "hotpot-300"
+
+You can see how to use those below:
+
+```python
+from opik.evaluation.metrics import LevenshteinRatio
+from opik_optimizer.few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from opik_optimizer.demo import get_or_create_dataset
+
+from opik_optimizer import (
+    OptimizationConfig,
+    MetricConfig,
+    TaskConfig,
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+hot_pot_dataset = get_or_create_dataset("hotpot-300")
+
+# For chat prompts instruction doesn't need to contain input parameters from dataset examples.
+prompt_instruction = """
+Answer the question.
+"""
+
+initial_prompt_no_examples = [
+    {"role": "system", "content": prompt_instruction},
+    {"role": "user", "content": "{{question}}"},
+]
+
+optimizer = FewShotBayesianOptimizer(
+    model="gpt-4o-mini",
+    project_name="optimize-few-shot-bayesian-hotpot",
+    min_examples=3,
+    max_examples=8,
+    n_threads=16,
+    seed=42,
+)
+
+optimization_config = OptimizationConfig(
+    dataset=hot_pot_dataset,
+    objective=MetricConfig(
+        metric=LevenshteinRatio(),
+        inputs={
+            "output": from_llm_response_text(),
+            "reference": from_dataset_field(name="answer"),
+        },
+    ),
+    task=TaskConfig(
+        instruction_prompt=prompt_instruction,
+        input_dataset_fields=["question"],
+        output_dataset_field="answer",
+        use_chat_prompt=True,
+    ),
+)
+
+result = optimizer.optimize_prompt(optimization_config, n_trials=10)
+print(result)
+```
+
+More examples can be found in the `scripts` folder.
+
+## Installation
+
+```bash
+pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+```
+
+## Development
+
+To use the Opik Optimizer from source:
+
+```bash
+git clone git clone git@github.com:comet-ml/opik
+cd sdks/opik_optimizer
+pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+
+- Opik API key
+- OpenAI API key (or other LLM provider)

opik_optimizer-0.7.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

opik_optimizer-0.7.0/setup.py

@@ -0,0 +1,38 @@
+from setuptools import setup, find_packages
+
+setup(
+    name="opik_optimizer",
+    version="0.7.0",
+    description="Agent optimization with Opik",
+    author="Comet ML",
+    author_email="info@comet.ml",
+    url="https://github.com/comet-ml/opik",
+    packages=find_packages(where="src"),
+    package_dir={"": "src"},
+    python_requires=">=3.9",
+    install_requires=[
+        "opik>=1.7.17",
+        "dspy>=2.6.18,<3",
+        "litellm",
+        "tqdm",
+        "datasets",
+        "optuna",
+        "pydantic",
+        "pandas",
+        "hf_xet",
+    ],
+    # dev requirements
+    extras_require={
+        "dev": [
+            "adalflow",
+            "pytest",
+            "pytest-conv"
+        ],
+    },
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "Intended Audience :: Developers",
+        "Programming Language :: Python :: 3",
+        "Programming Language :: Python :: 3.10",
+    ],
+)

opik_optimizer-0.7.0/src/opik_optimizer/__init__.py

@@ -0,0 +1,65 @@
+import importlib.metadata
+import logging
+from .logging_config import setup_logging
+
+__version__ = importlib.metadata.version("opik_optimizer")
+
+# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
+setup_logging(level=logging.WARNING)
+
+
+# Lazy imports to avoid circular dependencies
+def __getattr__(name):
+    if name == "MiproOptimizer":
+        from .mipro_optimizer import MiproOptimizer
+
+        return MiproOptimizer
+    elif name == "BaseOptimizer":
+        from .base_optimizer import BaseOptimizer
+
+        return BaseOptimizer
+    elif name == "MetaPromptOptimizer":
+        from .meta_prompt_optimizer import MetaPromptOptimizer
+
+        return MetaPromptOptimizer
+    elif name == "FewShotBayesianOptimizer":
+        from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+
+        return FewShotBayesianOptimizer
+    elif name in ["MetricConfig", "OptimizationConfig", "TaskConfig"]:
+        from .optimization_config.configs import (
+            MetricConfig,
+            OptimizationConfig,
+            TaskConfig,
+        )
+
+        return locals()[name]
+    elif name in ["from_dataset_field", "from_llm_response_text"]:
+        from .optimization_config.mappers import (
+            from_dataset_field,
+            from_llm_response_text,
+        )
+
+        return locals()[name]
+    raise AttributeError(f"module 'opik_optimizer' has no attribute '{name}'")
+
+
+from opik.evaluation.models.litellm import warning_filters
+
+warning_filters.add_warning_filters()
+
+from .optimization_result import OptimizationResult
+
+__all__ = [
+    "BaseOptimizer",
+    "FewShotBayesianOptimizer",
+    "MetaPromptOptimizer",
+    "MiproOptimizer",
+    "MetricConfig",
+    "OptimizationConfig",
+    "TaskConfig",
+    "from_dataset_field",
+    "from_llm_response_text",
+    "OptimizationResult",
+    "setup_logging",
+]

opik_optimizer-0.7.0/src/opik_optimizer/_throttle.py

@@ -0,0 +1,43 @@
+import threading
+import time
+import queue
+from functools import wraps
+
+class RateLimiter:
+    """
+    Rate limiter that enforces a maximum number of calls across all threads.
+    """
+    def __init__(self, max_calls_per_second):
+        self.max_calls_per_second = max_calls_per_second
+        self.interval = 1.0 / max_calls_per_second  # Time between allowed calls
+        self.last_call_time = 0
+        self.lock = threading.Lock()
+        
+    def acquire(self):
+        """
+        Wait until a call is allowed according to the global rate limit.
+        Returns immediately if the call is allowed, otherwise blocks until it's time.
+        """
+        with self.lock:
+            current_time = time.time()
+            time_since_last = current_time - self.last_call_time
+            
+            # If we haven't waited long enough since the last call
+            if time_since_last < self.interval:
+                # Calculate how much longer we need to wait
+                sleep_time = self.interval - time_since_last
+                time.sleep(sleep_time)
+                
+            # Update the last call time (after potential sleep)
+            self.last_call_time = time.time()
+
+def rate_limited(limiter):
+    """Decorator to rate limit a function using the provided limiter"""
+    def decorator(func):
+        @wraps(func)
+        def wrapper(*args, **kwargs):
+            limiter.acquire()
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+

opik_optimizer-0.7.0/src/opik_optimizer/base_optimizer.py

@@ -0,0 +1,240 @@
+from typing import Optional, Union, List, Dict, Any
+import opik
+import logging
+import time
+
+import litellm
+from opik.evaluation import metrics
+from opik.opik_context import get_current_span_data
+from opik.rest_api.core import ApiError
+
+from pydantic import BaseModel
+from ._throttle import RateLimiter, rate_limited
+from .cache_config import initialize_cache
+from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
+from .optimization_config.configs import TaskConfig, MetricConfig
+
+limiter = RateLimiter(max_calls_per_second=15)
+
+# Don't use unsupported params:
+litellm.drop_params = True
+
+# Set up logging:
+logger = logging.getLogger(__name__)
+
+
+class OptimizationRound(BaseModel):
+    round_number: int
+    current_prompt: str
+    current_score: float
+    generated_prompts: List[Dict[str, Any]]
+    best_prompt: str
+    best_score: float
+    improvement: float
+
+
+class BaseOptimizer:
+    def __init__(self, model: str, project_name: Optional[str] = None, **model_kwargs):
+        """
+        Base class for optimizers.
+
+        Args:
+           model: LiteLLM model name
+           project_name: Opik project name
+           model_kwargs: additional args for model (eg, temperature)
+        """
+        self.model = model
+        self.reasoning_model = model
+        self.model_kwargs = model_kwargs
+        self.project_name = project_name
+        self._history = []
+        self.experiment_config = None
+        self.llm_call_counter = 0
+
+        # Initialize shared cache
+        initialize_cache()
+
+    def optimize_prompt(
+        self,
+        dataset: Union[str, opik.Dataset],
+        metric_config: MetricConfig,
+        task_config: TaskConfig,
+        prompt: str,
+        input_key: str,
+        output_key: str,
+        experiment_config: Optional[Dict] = None,
+        **kwargs,
+    ):
+        """
+        Optimize a prompt.
+
+        Args:
+           dataset: Opik dataset name, or Opik dataset
+           metric_config: instance of a MetricConfig
+           task_config: instance of a TaskConfig
+           prompt: the prompt to optimize
+           input_key: input field of dataset
+           output_key: output field of dataset
+           experiment_config: Optional configuration for the experiment
+           **kwargs: Additional arguments for optimization
+        """
+        self.dataset = dataset
+        self.metric = metric
+        self.prompt = prompt
+        self.input_key = input_key
+        self.output_key = output_key
+        self.experiment_config = experiment_config
+
+    def evaluate_prompt(
+        self,
+        dataset: Union[str, opik.Dataset],
+        metric_config: MetricConfig,
+        prompt: str,
+        input_key: str,
+        output_key: str,
+        n_samples: int = 10,
+        task_config: Optional[TaskConfig] = None,
+        dataset_item_ids: Optional[List[str]] = None,
+        experiment_config: Optional[Dict] = None,
+        **kwargs,
+    ) -> float:
+        """
+        Evaluate a prompt.
+
+        Args:
+           dataset: Opik dataset name, or Opik dataset
+           metric_config: instance of a MetricConfig
+           task_config: instance of a TaskConfig
+           prompt: the prompt to evaluate
+           input_key: input field of dataset
+           output_key: output field of dataset
+           n_samples: number of items to test in the dataset
+           dataset_item_ids: Optional list of dataset item IDs to evaluate
+           experiment_config: Optional configuration for the experiment
+           **kwargs: Additional arguments for evaluation
+
+        Returns:
+            float: The evaluation score
+        """
+        self.dataset = dataset
+        self.metric_config = metric_config
+        self.task_config = task_config
+        self.prompt = prompt
+        self.input_key = input_key
+        self.output_key = output_key
+        self.experiment_config = experiment_config
+        return 0.0  # Base implementation returns 0
+
+    def get_history(self) -> List[Dict[str, Any]]:
+        """
+        Get the optimization history.
+
+        Returns:
+            List[Dict[str, Any]]: List of optimization rounds with their details
+        """
+        return self._history
+
+    def _add_to_history(self, round_data: Dict[str, Any]):
+        """
+        Add a round to the optimization history.
+
+        Args:
+            round_data: Dictionary containing round details
+        """
+        self._history.append(round_data)
+
+    @rate_limited(limiter)
+    def _call_model(
+        self,
+        prompt: str,
+        system_prompt: Optional[str] = None,
+        is_reasoning: bool = False,
+    ) -> str:
+        """Call the model to get suggestions based on the meta-prompt."""
+        model = self.reasoning_model if is_reasoning else self.model
+        messages = []
+
+        if system_prompt:
+            messages.append({"role": "system", "content": system_prompt})
+            logger.debug(f"Using custom system prompt: {system_prompt[:100]}...")
+        else:
+            messages.append(
+                {"role": "system", "content": "You are a helpful assistant."}
+            )
+
+        messages.append({"role": "user", "content": prompt})
+        logger.debug(f"Calling model {model} with prompt: {prompt[:100]}...")
+
+        api_params = self.model_kwargs.copy()
+        api_params.update(
+            {
+                "model": model,
+                "messages": messages,
+                # Ensure required params like 'temperature', 'max_tokens' are present
+                # Defaults added here for safety, though usually set in __init__ kwargs
+                "temperature": api_params.get("temperature", 0.3),
+                "max_tokens": api_params.get("max_tokens", 1000),
+            }
+        )
+
+        # Attempt to add Opik monitoring if available
+        try:
+            # Assuming opik_litellm_monitor is imported and configured elsewhere
+            api_params = opik_litellm_monitor.try_add_opik_monitoring_to_params(
+                api_params
+            )
+            logger.debug("Opik monitoring hooks added to LiteLLM params.")
+        except Exception as e:
+            logger.warning(f"Could not add Opik monitoring to LiteLLM params: {e}")
+
+        logger.debug(
+            f"Final API params (excluding messages): { {k:v for k,v in api_params.items() if k != 'messages'} }"
+        )
+
+        # Increment Counter
+        self.llm_call_counter += 1
+        logger.debug(f"LLM Call Count: {self.llm_call_counter}")
+
+        try:
+            response = litellm.completion(**api_params)
+            model_output = response.choices[0].message.content.strip()
+            logger.debug(f"Model response from {model_to_use}: {model_output[:100]}...")
+            return model_output
+        except litellm.exceptions.RateLimitError as e:
+            logger.error(f"LiteLLM Rate Limit Error for model {model_to_use}: {e}")
+            # Consider adding retry logic here with tenacity
+            raise
+        except litellm.exceptions.APIConnectionError as e:
+            logger.error(f"LiteLLM API Connection Error for model {model_to_use}: {e}")
+            # Consider adding retry logic here
+            raise
+        except litellm.exceptions.ContextWindowExceededError as e:
+            logger.error(
+                f"LiteLLM Context Window Exceeded Error for model {model_to_use}. Prompt length: {len(prompt)}. Details: {e}"
+            )
+            raise
+        except litellm.exceptions.APIError as e:  # Catch broader API errors
+            logger.error(f"LiteLLM API Error for model {model_to_use}: {e}")
+            raise
+        except Exception as e:
+            # Catch any other unexpected errors
+            logger.error(
+                f"Unexpected error during model call to {model_to_use}: {type(e).__name__} - {e}"
+            )
+            raise
+
+    def update_optimization(self, optimization, status: str) -> None:
+        """
+        Update the optimization status
+        """
+        # FIXME: remove when a solution is added to opik's optimization.update method
+        count = 0
+        while count < 3:
+            try:
+                optimization.update(status="completed")
+                break
+            except ApiError:
+                count += 1
+                time.sleep(5)
+        if count == 3:
+            logger.warning("Unable to update optimization status; continuing...")

opik_optimizer-0.7.0/src/opik_optimizer/cache_config.py

@@ -0,0 +1,24 @@
+import os
+from pathlib import Path
+import litellm
+from litellm.caching import Cache
+
+# Configure cache directory
+CACHE_DIR = os.path.expanduser("~/.litellm_cache")
+Path(CACHE_DIR).mkdir(parents=True, exist_ok=True)
+
+# Configure cache settings
+CACHE_CONFIG = {
+    "type": "disk",
+    "disk_cache_dir": CACHE_DIR,
+}
+
+def initialize_cache():
+    """Initialize the LiteLLM cache with custom configuration."""
+    litellm.cache = Cache(**CACHE_CONFIG)
+    return litellm.cache
+
+def clear_cache():
+    """Clear the LiteLLM cache."""
+    if litellm.cache:
+        litellm.cache.clear() 

opik_optimizer-0.7.0/src/opik_optimizer/demo/__init__.py

@@ -0,0 +1,7 @@
+from .datasets import get_or_create_dataset
+from .cache import get_litellm_cache
+
+__all__ = [
+    "get_or_create_dataset",
+    "get_litellm_cache",
+]