opik-optimizer 0.7.1__tar.gz → 0.7.3__tar.gz

opik_optimizer-0.7.3/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.7.3
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: support@comet.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Requires-Dist: pyrate-limiter
+Provides-Extra: dev
+Requires-Dist: adalflow; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Opik Optimizer
+
+The Opik Opitmizer can refine your prompts to get better performance
+from your LLMs. You can use a variety of algorithms, including:
+
+* FewShotBayesianOptimizer
+* MiproOptimizer
+* MetaPromptOptimizer
+
+## Quickstart
+
+
+[Open Quickstart Notebook in Colab](https://colab.research.google.com/github/comet-ml/opik/blob/main/sdks/opik_optimizer/notebooks/OpikOptimizerIntro.ipynb)
+
+
+## Setup
+
+1. Configure Opik:
+   ```bash
+   # Install Comet ML CLI
+   pip install opik
+
+   # Configure your API key
+   opik configure
+   # When prompted, enter your Opik API key
+   ```
+
+2. Set up your environment variables:
+   ```bash
+   # OpenAI API key for LLM access
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. Install the package:
+   ```bash
+   pip install opik-optimizer
+   ```
+
+You'll need:
+
+1. An LLM model name
+2. An Opik Dataset (or Opik Dataset name)
+3. An Opik Metric (possibly a custom one)
+4. A starting prompt (string)
+
+## Example
+
+We have prepared some sample datasets for testing:
+
+* "tiny-test"
+* "halu-eval-300"
+* "hotpot-300"
+
+You can see how to use those below:
+
+```python
+from opik.evaluation.metrics import LevenshteinRatio
+from opik_optimizer import FewShotBayesianOptimizer
+from opik_optimizer.demo import get_or_create_dataset
+
+from opik_optimizer import (
+    MetricConfig,
+    TaskConfig,
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+hot_pot_dataset = get_or_create_dataset("hotpot-300")
+
+# For chat prompts instruction doesn't need to contain input parameters from dataset examples.
+prompt_instruction = """
+Answer the question.
+"""
+project_name = "optimize-few-shot-bayesian-hotpot"
+
+optimizer = FewShotBayesianOptimizer(
+    model="gpt-4o-mini",
+    project_name=project_name,
+    min_examples=3,
+    max_examples=8,
+    n_threads=16,
+    seed=42,
+)
+
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name),
+    inputs={
+        "output": from_llm_response_text(),
+        "reference": from_dataset_field(name="answer"),
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"],
+    output_dataset_field="answer",
+    use_chat_prompt=True,
+)
+
+result = optimizer.optimize_prompt(
+    dataset=hot_pot_dataset,
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,
+    n_samples=150,
+)
+
+result.display()
+```
+
+More examples can be found in the `scripts` folder.
+
+## Installation
+
+```bash
+pip install opik-optimizer
+```
+
+## Development
+
+To use the Opik Optimizer from source:
+
+```bash
+git clone git clone git@github.com:comet-ml/opik
+cd sdks/opik_optimizer
+pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+ < 3.13
+- Opik API key
+- OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/README.md

@@ -33,7 +33,7 @@ from your LLMs. You can use a variety of algorithms, including:
 
 3. Install the package:
    ```bash
-   pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+   pip install opik-optimizer
    ```
 
 You'll need:
@@ -55,11 +55,10 @@ You can see how to use those below:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer.few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from opik_optimizer import FewShotBayesianOptimizer
 from opik_optimizer.demo import get_or_create_dataset
 
 from opik_optimizer import (
-    OptimizationConfig,
     MetricConfig,
     TaskConfig,
     from_dataset_field,
@@ -72,40 +71,41 @@ hot_pot_dataset = get_or_create_dataset("hotpot-300")
 prompt_instruction = """
 Answer the question.
 """
-
-initial_prompt_no_examples = [
-    {"role": "system", "content": prompt_instruction},
-    {"role": "user", "content": "{{question}}"},
-]
+project_name = "optimize-few-shot-bayesian-hotpot"
 
 optimizer = FewShotBayesianOptimizer(
     model="gpt-4o-mini",
-    project_name="optimize-few-shot-bayesian-hotpot",
+    project_name=project_name,
     min_examples=3,
     max_examples=8,
     n_threads=16,
     seed=42,
 )
 
-optimization_config = OptimizationConfig(
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name),
+    inputs={
+        "output": from_llm_response_text(),
+        "reference": from_dataset_field(name="answer"),
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"],
+    output_dataset_field="answer",
+    use_chat_prompt=True,
+)
+
+result = optimizer.optimize_prompt(
     dataset=hot_pot_dataset,
-    objective=MetricConfig(
-        metric=LevenshteinRatio(),
-        inputs={
-            "output": from_llm_response_text(),
-            "reference": from_dataset_field(name="answer"),
-        },
-    ),
-    task=TaskConfig(
-        instruction_prompt=prompt_instruction,
-        input_dataset_fields=["question"],
-        output_dataset_field="answer",
-        use_chat_prompt=True,
-    ),
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,
+    n_samples=150,
 )
 
-result = optimizer.optimize_prompt(optimization_config, n_trials=10)
-print(result)
+result.display()
 ```
 
 More examples can be found in the `scripts` folder.
@@ -113,7 +113,7 @@ More examples can be found in the `scripts` folder.
 ## Installation
 
 ```bash
-pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+pip install opik-optimizer
 ```
 
 ## Development
@@ -128,6 +128,6 @@ pip install -e .
 
 ## Requirements
 
-- Python 3.10+
+- Python 3.10+ < 3.13
 - Opik API key
 - OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/setup.py

@@ -2,14 +2,16 @@ from setuptools import setup, find_packages
 
 setup(
     name="opik_optimizer",
-    version="0.7.1",
+    version="0.7.3",
     description="Agent optimization with Opik",
     author="Comet ML",
-    author_email="info@comet.ml",
+    author_email="support@comet.com",
+    long_description=open("README.md", encoding="utf-8").read(),
+    long_description_content_type='text/markdown',
     url="https://github.com/comet-ml/opik",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
-    python_requires=">=3.9",
+    python_requires=">=3.9,<3.13",
     install_requires=[
         "opik>=1.7.17",
         "dspy>=2.6.18,<3",
@@ -20,6 +22,7 @@ setup(
         "pydantic",
         "pandas",
         "hf_xet",
+        "pyrate-limiter",
     ],
     # dev requirements
     extras_require={

opik_optimizer-0.7.3/src/opik_optimizer/__init__.py

@@ -0,0 +1,43 @@
+import importlib.metadata
+import logging
+from .logging_config import setup_logging
+
+__version__ = importlib.metadata.version("opik_optimizer")
+
+# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
+setup_logging(level=logging.WARNING)
+
+# Regular imports
+from .mipro_optimizer import MiproOptimizer
+from .base_optimizer import BaseOptimizer
+from .meta_prompt_optimizer import MetaPromptOptimizer
+from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from .optimization_config.configs import (
+    MetricConfig,
+    OptimizationConfig,
+    TaskConfig,
+)
+from .optimization_config.mappers import (
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+from opik.evaluation.models.litellm import warning_filters
+
+warning_filters.add_warning_filters()
+
+from .optimization_result import OptimizationResult
+
+__all__ = [
+    "BaseOptimizer",
+    "FewShotBayesianOptimizer",
+    "MetaPromptOptimizer",
+    "MiproOptimizer",
+    "MetricConfig",
+    "OptimizationConfig",
+    "TaskConfig",
+    "from_dataset_field",
+    "from_llm_response_text",
+    "OptimizationResult",
+    "setup_logging",
+]

opik_optimizer-0.7.3/src/opik_optimizer/_throttle.py

@@ -0,0 +1,43 @@
+import functools
+import pyrate_limiter
+import time
+import opik.config
+
+from typing import Callable, Any
+
+
+class RateLimiter:
+    """
+    Rate limiter that enforces a maximum number of calls across all threads using pyrate_limiter.
+    """
+    def __init__(self, max_calls_per_second: int):
+        self.max_calls_per_second = max_calls_per_second
+        rate = pyrate_limiter.Rate(max_calls_per_second, pyrate_limiter.Duration.SECOND)
+
+        self.limiter = pyrate_limiter.Limiter(rate, raise_when_fail=False)
+        self.bucket_key = "global_rate_limit"
+        
+    def acquire(self) -> None:
+        while not self.limiter.try_acquire(self.bucket_key):
+            time.sleep(0.01)
+
+def rate_limited(limiter: RateLimiter) -> Callable[[Callable], Callable]:
+    """Decorator to rate limit a function using the provided limiter"""
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            limiter.acquire()
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def get_rate_limiter_for_current_opik_installation() -> RateLimiter:
+    opik_config = opik.config.OpikConfig()
+    max_calls_per_second = (
+        10
+        if opik_config.is_cloud_installation
+        else 50
+    )
+    return RateLimiter(max_calls_per_second=max_calls_per_second)

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer/base_optimizer.py

@@ -4,8 +4,6 @@ import logging
 import time
 
 import litellm
-from opik.evaluation import metrics
-from opik.opik_context import get_current_span_data
 from opik.rest_api.core import ApiError
 
 from pydantic import BaseModel
@@ -14,7 +12,7 @@ from .cache_config import initialize_cache
 from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
 from .optimization_config.configs import TaskConfig, MetricConfig
 
-limiter = RateLimiter(max_calls_per_second=15)
+limiter = RateLimiter(max_calls_per_second=8)
 
 # Don't use unsupported params:
 litellm.drop_params = True
@@ -143,85 +141,6 @@ class BaseOptimizer:
         """
         self._history.append(round_data)
 
-    @rate_limited(limiter)
-    def _call_model(
-        self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        is_reasoning: bool = False,
-    ) -> str:
-        """Call the model to get suggestions based on the meta-prompt."""
-        model = self.reasoning_model if is_reasoning else self.model
-        messages = []
-
-        if system_prompt:
-            messages.append({"role": "system", "content": system_prompt})
-            logger.debug(f"Using custom system prompt: {system_prompt[:100]}...")
-        else:
-            messages.append(
-                {"role": "system", "content": "You are a helpful assistant."}
-            )
-
-        messages.append({"role": "user", "content": prompt})
-        logger.debug(f"Calling model {model} with prompt: {prompt[:100]}...")
-
-        api_params = self.model_kwargs.copy()
-        api_params.update(
-            {
-                "model": model,
-                "messages": messages,
-                # Ensure required params like 'temperature', 'max_tokens' are present
-                # Defaults added here for safety, though usually set in __init__ kwargs
-                "temperature": api_params.get("temperature", 0.3),
-                "max_tokens": api_params.get("max_tokens", 1000),
-            }
-        )
-
-        # Attempt to add Opik monitoring if available
-        try:
-            # Assuming opik_litellm_monitor is imported and configured elsewhere
-            api_params = opik_litellm_monitor.try_add_opik_monitoring_to_params(
-                api_params
-            )
-            logger.debug("Opik monitoring hooks added to LiteLLM params.")
-        except Exception as e:
-            logger.warning(f"Could not add Opik monitoring to LiteLLM params: {e}")
-
-        logger.debug(
-            f"Final API params (excluding messages): { {k:v for k,v in api_params.items() if k != 'messages'} }"
-        )
-
-        # Increment Counter
-        self.llm_call_counter += 1
-        logger.debug(f"LLM Call Count: {self.llm_call_counter}")
-
-        try:
-            response = litellm.completion(**api_params)
-            model_output = response.choices[0].message.content.strip()
-            logger.debug(f"Model response from {model_to_use}: {model_output[:100]}...")
-            return model_output
-        except litellm.exceptions.RateLimitError as e:
-            logger.error(f"LiteLLM Rate Limit Error for model {model_to_use}: {e}")
-            # Consider adding retry logic here with tenacity
-            raise
-        except litellm.exceptions.APIConnectionError as e:
-            logger.error(f"LiteLLM API Connection Error for model {model_to_use}: {e}")
-            # Consider adding retry logic here
-            raise
-        except litellm.exceptions.ContextWindowExceededError as e:
-            logger.error(
-                f"LiteLLM Context Window Exceeded Error for model {model_to_use}. Prompt length: {len(prompt)}. Details: {e}"
-            )
-            raise
-        except litellm.exceptions.APIError as e:  # Catch broader API errors
-            logger.error(f"LiteLLM API Error for model {model_to_use}: {e}")
-            raise
-        except Exception as e:
-            # Catch any other unexpected errors
-            logger.error(
-                f"Unexpected error during model call to {model_to_use}: {type(e).__name__} - {e}"
-            )
-            raise
 
     def update_optimization(self, optimization, status: str) -> None:
         """

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer/demo/cache.py

@@ -9,6 +9,7 @@ import requests
 NAMED_CACHES = {
     "test": "https://drive.google.com/file/d/1RifNtpN-pl0DW49daRaAMJwW7MCsOh6y/view?usp=sharing",
     "test2": "https://drive.google.com/uc?id=1RifNtpN-pl0DW49daRaAMJwW7MCsOh6y&export=download",
+    "opik-workshop": "https://drive.google.com/file/d/1l0aK6KhDPs2bFsQTkfzvOvfacJlhdmHr/view?usp=sharing",
 }
 CACHE_DIR = os.path.expanduser("~/.litellm_cache")
 

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py

@@ -1,8 +1,8 @@
 import random
-from typing import Any, Dict, List, Tuple, Union, Optional, Callable, Literal
-import openai
+from typing import Any, Dict, List, Tuple, Union, Optional, Literal
 import opik
 import optuna
+import optuna.samplers
 import logging
 import json
 
@@ -14,18 +14,18 @@ from opik_optimizer import base_optimizer
 
 from . import prompt_parameter
 from . import prompt_templates
-from .._throttle import RateLimiter, rate_limited
+from .. import _throttle
 from .. import optimization_result, task_evaluator
 
 import litellm
 
 from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
 
-limiter = RateLimiter(max_calls_per_second=15)
+_limiter = _throttle.get_rate_limiter_for_current_opik_installation()
 
 logger = logging.getLogger(__name__)
 
-@rate_limited(limiter)
+@_throttle.rate_limited(_limiter)
 def _call_model(model, messages, seed, model_kwargs):
     model_kwargs = opik_litellm_monitor.try_add_opik_monitoring_to_params(model_kwargs)
 
@@ -59,7 +59,6 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         self.n_threads = n_threads
         self.n_initial_prompts = n_initial_prompts
         self.n_iterations = n_iterations
-
         self._opik_client = opik.Opik()
         logger.debug(f"Initialized FewShotBayesianOptimizer with model: {model}")
 
@@ -85,7 +84,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         split_idx = int(len(dataset) * train_ratio)
         return dataset[:split_idx], dataset[split_idx:]
 
-    def _optimize_prompt(
+    def _optimize_prompt(   
         self,
         dataset: Union[str, Dataset],
         metric_config: MetricConfig,
@@ -171,8 +170,10 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             n_examples = trial.suggest_int(
                 "n_examples", self.min_examples, self.max_examples
             )
-            available_indices = list(range(len(dataset_items)))
-            example_indices = random.sample(available_indices, n_examples)
+            example_indices = [
+                trial.suggest_categorical(f"example_{i}", list(range(len(dataset_items))))
+                for i in range(n_examples)
+            ]
             trial.set_user_attr("example_indices", example_indices)
 
             instruction = task_config.instruction_prompt
@@ -238,7 +239,8 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         except Exception as e:
             logger.warning(f"Could not configure Optuna logging within optimizer: {e}")
 
-        study = optuna.create_study(direction="maximize")
+        sampler = optuna.samplers.TPESampler(seed=self.seed)
+        study = optuna.create_study(direction="maximize", sampler=sampler)
         study.optimize(optimization_objective, n_trials=n_trials)
         logger.info("Optuna study finished.")
 

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer/meta_prompt_optimizer.py

@@ -16,6 +16,7 @@ from opik_optimizer import task_evaluator
 from opik.api_objects import opik_client
 from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
 from opik.environment import get_tqdm_for_current_environment
+from . import _throttle
 
 tqdm = get_tqdm_for_current_environment()
 
@@ -26,6 +27,7 @@ litellm.cache = Cache(type="disk", disk_cache_dir=disk_cache_dir)
 # Set up logging
 logger = logging.getLogger(__name__)  # Gets logger configured by setup_logging
 
+_rate_limiter = _throttle.get_rate_limiter_for_current_opik_installation()
 
 class MetaPromptOptimizer(BaseOptimizer):
     """Optimizer that uses meta-prompting to improve prompts based on examples and performance."""
@@ -176,6 +178,7 @@ class MetaPromptOptimizer(BaseOptimizer):
             optimization_id=optimization_id,
         )
 
+    @_throttle.rate_limited(_rate_limiter)
     def _call_model(
         self,
         prompt: str,

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer/mipro_optimizer/_lm.py

@@ -22,11 +22,11 @@ from dspy.dsp.utils.settings import settings
 from dspy.utils.callback import BaseCallback, with_callbacks
 from dspy.clients.base_lm import BaseLM
 
-from .._throttle import RateLimiter, rate_limited
+from .._throttle import RateLimiter, rate_limited, get_rate_limiter_for_current_opik_installation
 
 logger = logging.getLogger(__name__)
 # Limit how fast an LLM can be called:
-limiter = RateLimiter(max_calls_per_second=15)
+limiter = get_rate_limiter_for_current_opik_installation()
 
 class LM(BaseLM):
     """

opik_optimizer-0.7.3/src/opik_optimizer.egg-info/PKG-INFO

@@ -0,0 +1,173 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.7.3
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: support@comet.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Requires-Dist: pyrate-limiter
+Provides-Extra: dev
+Requires-Dist: adalflow; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Opik Optimizer
+
+The Opik Opitmizer can refine your prompts to get better performance
+from your LLMs. You can use a variety of algorithms, including:
+
+* FewShotBayesianOptimizer
+* MiproOptimizer
+* MetaPromptOptimizer
+
+## Quickstart
+
+
+[Open Quickstart Notebook in Colab](https://colab.research.google.com/github/comet-ml/opik/blob/main/sdks/opik_optimizer/notebooks/OpikOptimizerIntro.ipynb)
+
+
+## Setup
+
+1. Configure Opik:
+   ```bash
+   # Install Comet ML CLI
+   pip install opik
+
+   # Configure your API key
+   opik configure
+   # When prompted, enter your Opik API key
+   ```
+
+2. Set up your environment variables:
+   ```bash
+   # OpenAI API key for LLM access
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. Install the package:
+   ```bash
+   pip install opik-optimizer
+   ```
+
+You'll need:
+
+1. An LLM model name
+2. An Opik Dataset (or Opik Dataset name)
+3. An Opik Metric (possibly a custom one)
+4. A starting prompt (string)
+
+## Example
+
+We have prepared some sample datasets for testing:
+
+* "tiny-test"
+* "halu-eval-300"
+* "hotpot-300"
+
+You can see how to use those below:
+
+```python
+from opik.evaluation.metrics import LevenshteinRatio
+from opik_optimizer import FewShotBayesianOptimizer
+from opik_optimizer.demo import get_or_create_dataset
+
+from opik_optimizer import (
+    MetricConfig,
+    TaskConfig,
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+hot_pot_dataset = get_or_create_dataset("hotpot-300")
+
+# For chat prompts instruction doesn't need to contain input parameters from dataset examples.
+prompt_instruction = """
+Answer the question.
+"""
+project_name = "optimize-few-shot-bayesian-hotpot"
+
+optimizer = FewShotBayesianOptimizer(
+    model="gpt-4o-mini",
+    project_name=project_name,
+    min_examples=3,
+    max_examples=8,
+    n_threads=16,
+    seed=42,
+)
+
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name),
+    inputs={
+        "output": from_llm_response_text(),
+        "reference": from_dataset_field(name="answer"),
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"],
+    output_dataset_field="answer",
+    use_chat_prompt=True,
+)
+
+result = optimizer.optimize_prompt(
+    dataset=hot_pot_dataset,
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,
+    n_samples=150,
+)
+
+result.display()
+```
+
+More examples can be found in the `scripts` folder.
+
+## Installation
+
+```bash
+pip install opik-optimizer
+```
+
+## Development
+
+To use the Opik Optimizer from source:
+
+```bash
+git clone git clone git@github.com:comet-ml/opik
+cd sdks/opik_optimizer
+pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+ < 3.13
+- Opik API key
+- OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.1 → opik_optimizer-0.7.3}/src/opik_optimizer.egg-info/requires.txt

@@ -7,6 +7,7 @@ optuna
 pydantic
 pandas
 hf_xet
+pyrate-limiter
 
 [dev]
 adalflow

opik_optimizer-0.7.1/PKG-INFO

@@ -1,35 +0,0 @@
-Metadata-Version: 2.4
-Name: opik_optimizer
-Version: 0.7.1
-Summary: Agent optimization with Opik
-Home-page: https://github.com/comet-ml/opik
-Author: Comet ML
-Author-email: info@comet.ml
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.9
-License-File: LICENSE
-Requires-Dist: opik>=1.7.17
-Requires-Dist: dspy<3,>=2.6.18
-Requires-Dist: litellm
-Requires-Dist: tqdm
-Requires-Dist: datasets
-Requires-Dist: optuna
-Requires-Dist: pydantic
-Requires-Dist: pandas
-Requires-Dist: hf_xet
-Provides-Extra: dev
-Requires-Dist: adalflow; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-conv; extra == "dev"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary

opik_optimizer-0.7.1/src/opik_optimizer/__init__.py

@@ -1,65 +0,0 @@
-import importlib.metadata
-import logging
-from .logging_config import setup_logging
-
-__version__ = importlib.metadata.version("opik_optimizer")
-
-# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
-setup_logging(level=logging.WARNING)
-
-
-# Lazy imports to avoid circular dependencies
-def __getattr__(name):
-    if name == "MiproOptimizer":
-        from .mipro_optimizer import MiproOptimizer
-
-        return MiproOptimizer
-    elif name == "BaseOptimizer":
-        from .base_optimizer import BaseOptimizer
-
-        return BaseOptimizer
-    elif name == "MetaPromptOptimizer":
-        from .meta_prompt_optimizer import MetaPromptOptimizer
-
-        return MetaPromptOptimizer
-    elif name == "FewShotBayesianOptimizer":
-        from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
-
-        return FewShotBayesianOptimizer
-    elif name in ["MetricConfig", "OptimizationConfig", "TaskConfig"]:
-        from .optimization_config.configs import (
-            MetricConfig,
-            OptimizationConfig,
-            TaskConfig,
-        )
-
-        return locals()[name]
-    elif name in ["from_dataset_field", "from_llm_response_text"]:
-        from .optimization_config.mappers import (
-            from_dataset_field,
-            from_llm_response_text,
-        )
-
-        return locals()[name]
-    raise AttributeError(f"module 'opik_optimizer' has no attribute '{name}'")
-
-
-from opik.evaluation.models.litellm import warning_filters
-
-warning_filters.add_warning_filters()
-
-from .optimization_result import OptimizationResult
-
-__all__ = [
-    "BaseOptimizer",
-    "FewShotBayesianOptimizer",
-    "MetaPromptOptimizer",
-    "MiproOptimizer",
-    "MetricConfig",
-    "OptimizationConfig",
-    "TaskConfig",
-    "from_dataset_field",
-    "from_llm_response_text",
-    "OptimizationResult",
-    "setup_logging",
-]

opik_optimizer-0.7.1/src/opik_optimizer/_throttle.py

@@ -1,43 +0,0 @@
-import threading
-import time
-import queue
-from functools import wraps
-
-class RateLimiter:
-    """
-    Rate limiter that enforces a maximum number of calls across all threads.
-    """
-    def __init__(self, max_calls_per_second):
-        self.max_calls_per_second = max_calls_per_second
-        self.interval = 1.0 / max_calls_per_second  # Time between allowed calls
-        self.last_call_time = 0
-        self.lock = threading.Lock()
-        
-    def acquire(self):
-        """
-        Wait until a call is allowed according to the global rate limit.
-        Returns immediately if the call is allowed, otherwise blocks until it's time.
-        """
-        with self.lock:
-            current_time = time.time()
-            time_since_last = current_time - self.last_call_time
-            
-            # If we haven't waited long enough since the last call
-            if time_since_last < self.interval:
-                # Calculate how much longer we need to wait
-                sleep_time = self.interval - time_since_last
-                time.sleep(sleep_time)
-                
-            # Update the last call time (after potential sleep)
-            self.last_call_time = time.time()
-
-def rate_limited(limiter):
-    """Decorator to rate limit a function using the provided limiter"""
-    def decorator(func):
-        @wraps(func)
-        def wrapper(*args, **kwargs):
-            limiter.acquire()
-            return func(*args, **kwargs)
-        return wrapper
-    return decorator
-

opik_optimizer-0.7.1/src/opik_optimizer.egg-info/PKG-INFO

@@ -1,35 +0,0 @@
-Metadata-Version: 2.4
-Name: opik_optimizer
-Version: 0.7.1
-Summary: Agent optimization with Opik
-Home-page: https://github.com/comet-ml/opik
-Author: Comet ML
-Author-email: info@comet.ml
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.9
-License-File: LICENSE
-Requires-Dist: opik>=1.7.17
-Requires-Dist: dspy<3,>=2.6.18
-Requires-Dist: litellm
-Requires-Dist: tqdm
-Requires-Dist: datasets
-Requires-Dist: optuna
-Requires-Dist: pydantic
-Requires-Dist: pandas
-Requires-Dist: hf_xet
-Provides-Extra: dev
-Requires-Dist: adalflow; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-conv; extra == "dev"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary