opik-optimizer 0.7.2__tar.gz → 0.7.4__tar.gz

{opik_optimizer-0.7.2/src/opik_optimizer.egg-info → opik_optimizer-0.7.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.7.2
+Version: 0.7.4
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML
@@ -21,6 +21,8 @@ Requires-Dist: optuna
 Requires-Dist: pydantic
 Requires-Dist: pandas
 Requires-Dist: hf_xet
+Requires-Dist: pyrate-limiter
+Requires-Dist: deap>=1.4.3
 Provides-Extra: dev
 Requires-Dist: adalflow; extra == "dev"
 Requires-Dist: pytest; extra == "dev"
@@ -72,7 +74,7 @@ from your LLMs. You can use a variety of algorithms, including:
 
 3. Install the package:
    ```bash
-   pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+   pip install opik-optimizer
    ```
 
 You'll need:
@@ -94,11 +96,10 @@ You can see how to use those below:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer.few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from opik_optimizer import FewShotBayesianOptimizer
 from opik_optimizer.demo import get_or_create_dataset
 
 from opik_optimizer import (
-    OptimizationConfig,
     MetricConfig,
     TaskConfig,
     from_dataset_field,
@@ -111,40 +112,41 @@ hot_pot_dataset = get_or_create_dataset("hotpot-300")
 prompt_instruction = """
 Answer the question.
 """
-
-initial_prompt_no_examples = [
-    {"role": "system", "content": prompt_instruction},
-    {"role": "user", "content": "{{question}}"},
-]
+project_name = "optimize-few-shot-bayesian-hotpot"
 
 optimizer = FewShotBayesianOptimizer(
     model="gpt-4o-mini",
-    project_name="optimize-few-shot-bayesian-hotpot",
+    project_name=project_name,
     min_examples=3,
     max_examples=8,
     n_threads=16,
     seed=42,
 )
 
-optimization_config = OptimizationConfig(
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name),
+    inputs={
+        "output": from_llm_response_text(),
+        "reference": from_dataset_field(name="answer"),
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"],
+    output_dataset_field="answer",
+    use_chat_prompt=True,
+)
+
+result = optimizer.optimize_prompt(
     dataset=hot_pot_dataset,
-    objective=MetricConfig(
-        metric=LevenshteinRatio(),
-        inputs={
-            "output": from_llm_response_text(),
-            "reference": from_dataset_field(name="answer"),
-        },
-    ),
-    task=TaskConfig(
-        instruction_prompt=prompt_instruction,
-        input_dataset_fields=["question"],
-        output_dataset_field="answer",
-        use_chat_prompt=True,
-    ),
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,
+    n_samples=150,
 )
 
-result = optimizer.optimize_prompt(optimization_config, n_trials=10)
-print(result)
+result.display()
 ```
 
 More examples can be found in the `scripts` folder.
@@ -152,7 +154,7 @@ More examples can be found in the `scripts` folder.
 ## Installation
 
 ```bash
-pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+pip install opik-optimizer
 ```
 
 ## Development
@@ -167,6 +169,6 @@ pip install -e .
 
 ## Requirements
 
-- Python 3.10+
+- Python 3.10+ < 3.13
 - Opik API key
 - OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.2 → opik_optimizer-0.7.4}/README.md

@@ -33,7 +33,7 @@ from your LLMs. You can use a variety of algorithms, including:
 
 3. Install the package:
    ```bash
-   pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+   pip install opik-optimizer
    ```
 
 You'll need:
@@ -55,11 +55,10 @@ You can see how to use those below:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer.few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from opik_optimizer import FewShotBayesianOptimizer
 from opik_optimizer.demo import get_or_create_dataset
 
 from opik_optimizer import (
-    OptimizationConfig,
     MetricConfig,
     TaskConfig,
     from_dataset_field,
@@ -72,40 +71,41 @@ hot_pot_dataset = get_or_create_dataset("hotpot-300")
 prompt_instruction = """
 Answer the question.
 """
-
-initial_prompt_no_examples = [
-    {"role": "system", "content": prompt_instruction},
-    {"role": "user", "content": "{{question}}"},
-]
+project_name = "optimize-few-shot-bayesian-hotpot"
 
 optimizer = FewShotBayesianOptimizer(
     model="gpt-4o-mini",
-    project_name="optimize-few-shot-bayesian-hotpot",
+    project_name=project_name,
     min_examples=3,
     max_examples=8,
     n_threads=16,
     seed=42,
 )
 
-optimization_config = OptimizationConfig(
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name),
+    inputs={
+        "output": from_llm_response_text(),
+        "reference": from_dataset_field(name="answer"),
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"],
+    output_dataset_field="answer",
+    use_chat_prompt=True,
+)
+
+result = optimizer.optimize_prompt(
     dataset=hot_pot_dataset,
-    objective=MetricConfig(
-        metric=LevenshteinRatio(),
-        inputs={
-            "output": from_llm_response_text(),
-            "reference": from_dataset_field(name="answer"),
-        },
-    ),
-    task=TaskConfig(
-        instruction_prompt=prompt_instruction,
-        input_dataset_fields=["question"],
-        output_dataset_field="answer",
-        use_chat_prompt=True,
-    ),
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,
+    n_samples=150,
 )
 
-result = optimizer.optimize_prompt(optimization_config, n_trials=10)
-print(result)
+result.display()
 ```
 
 More examples can be found in the `scripts` folder.
@@ -113,7 +113,7 @@ More examples can be found in the `scripts` folder.
 ## Installation
 
 ```bash
-pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+pip install opik-optimizer
 ```
 
 ## Development
@@ -128,6 +128,6 @@ pip install -e .
 
 ## Requirements
 
-- Python 3.10+
+- Python 3.10+ < 3.13
 - Opik API key
 - OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.2 → opik_optimizer-0.7.4}/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="opik_optimizer",
-    version="0.7.2",
+    version="0.7.4",
     description="Agent optimization with Opik",
     author="Comet ML",
     author_email="support@comet.com",
@@ -11,6 +11,9 @@ setup(
     url="https://github.com/comet-ml/opik",
     packages=find_packages(where="src"),
     package_dir={"": "src"},
+    package_data={
+        'opik_optimizer': ['data/*.json'],
+    },
     python_requires=">=3.9,<3.13",
     install_requires=[
         "opik>=1.7.17",
@@ -22,6 +25,8 @@ setup(
         "pydantic",
         "pandas",
         "hf_xet",
+        "pyrate-limiter",
+        "deap>=1.4.3",
     ],
     # dev requirements
     extras_require={

{opik_optimizer-0.7.2 → opik_optimizer-0.7.4}/src/opik_optimizer/__init__.py

@@ -27,12 +27,14 @@ from opik.evaluation.models.litellm import warning_filters
 warning_filters.add_warning_filters()
 
 from .optimization_result import OptimizationResult
+from opik_optimizer.evolutionary_optimizer.evolutionary_optimizer import EvolutionaryOptimizer
 
 __all__ = [
     "BaseOptimizer",
     "FewShotBayesianOptimizer",
     "MetaPromptOptimizer",
     "MiproOptimizer",
+    "EvolutionaryOptimizer",
     "MetricConfig",
     "OptimizationConfig",
     "TaskConfig",

opik_optimizer-0.7.4/src/opik_optimizer/_throttle.py

@@ -0,0 +1,43 @@
+import functools
+import pyrate_limiter
+import time
+import opik.config
+
+from typing import Callable, Any
+
+
+class RateLimiter:
+    """
+    Rate limiter that enforces a maximum number of calls across all threads using pyrate_limiter.
+    """
+    def __init__(self, max_calls_per_second: int):
+        self.max_calls_per_second = max_calls_per_second
+        rate = pyrate_limiter.Rate(max_calls_per_second, pyrate_limiter.Duration.SECOND)
+
+        self.limiter = pyrate_limiter.Limiter(rate, raise_when_fail=False)
+        self.bucket_key = "global_rate_limit"
+        
+    def acquire(self) -> None:
+        while not self.limiter.try_acquire(self.bucket_key):
+            time.sleep(0.01)
+
+def rate_limited(limiter: RateLimiter) -> Callable[[Callable], Callable]:
+    """Decorator to rate limit a function using the provided limiter"""
+
+    def decorator(func: Callable) -> Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs) -> Any:
+            limiter.acquire()
+            return func(*args, **kwargs)
+        return wrapper
+    return decorator
+
+
+def get_rate_limiter_for_current_opik_installation() -> RateLimiter:
+    opik_config = opik.config.OpikConfig()
+    max_calls_per_second = (
+        10
+        if opik_config.is_cloud_installation
+        else 50
+    )
+    return RateLimiter(max_calls_per_second=max_calls_per_second)

{opik_optimizer-0.7.2 → opik_optimizer-0.7.4}/src/opik_optimizer/base_optimizer.py

@@ -12,7 +12,7 @@ from .cache_config import initialize_cache
 from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
 from .optimization_config.configs import TaskConfig, MetricConfig
 
-limiter = RateLimiter(max_calls_per_second=15)
+limiter = RateLimiter(max_calls_per_second=8)
 
 # Don't use unsupported params:
 litellm.drop_params = True
@@ -141,85 +141,6 @@ class BaseOptimizer:
         """
         self._history.append(round_data)
 
-    @rate_limited(limiter)
-    def _call_model(
-        self,
-        prompt: str,
-        system_prompt: Optional[str] = None,
-        is_reasoning: bool = False,
-    ) -> str:
-        """Call the model to get suggestions based on the meta-prompt."""
-        model = self.reasoning_model if is_reasoning else self.model
-        messages = []
-
-        if system_prompt:
-            messages.append({"role": "system", "content": system_prompt})
-            logger.debug(f"Using custom system prompt: {system_prompt[:100]}...")
-        else:
-            messages.append(
-                {"role": "system", "content": "You are a helpful assistant."}
-            )
-
-        messages.append({"role": "user", "content": prompt})
-        logger.debug(f"Calling model {model} with prompt: {prompt[:100]}...")
-
-        api_params = self.model_kwargs.copy()
-        api_params.update(
-            {
-                "model": model,
-                "messages": messages,
-                # Ensure required params like 'temperature', 'max_tokens' are present
-                # Defaults added here for safety, though usually set in __init__ kwargs
-                "temperature": api_params.get("temperature", 0.3),
-                "max_tokens": api_params.get("max_tokens", 1000),
-            }
-        )
-
-        # Attempt to add Opik monitoring if available
-        try:
-            # Assuming opik_litellm_monitor is imported and configured elsewhere
-            api_params = opik_litellm_monitor.try_add_opik_monitoring_to_params(
-                api_params
-            )
-            logger.debug("Opik monitoring hooks added to LiteLLM params.")
-        except Exception as e:
-            logger.warning(f"Could not add Opik monitoring to LiteLLM params: {e}")
-
-        logger.debug(
-            f"Final API params (excluding messages): { {k:v for k,v in api_params.items() if k != 'messages'} }"
-        )
-
-        # Increment Counter
-        self.llm_call_counter += 1
-        logger.debug(f"LLM Call Count: {self.llm_call_counter}")
-
-        try:
-            response = litellm.completion(**api_params)
-            model_output = response.choices[0].message.content.strip()
-            logger.debug(f"Model response from {model_to_use}: {model_output[:100]}...")
-            return model_output
-        except litellm.exceptions.RateLimitError as e:
-            logger.error(f"LiteLLM Rate Limit Error for model {model_to_use}: {e}")
-            # Consider adding retry logic here with tenacity
-            raise
-        except litellm.exceptions.APIConnectionError as e:
-            logger.error(f"LiteLLM API Connection Error for model {model_to_use}: {e}")
-            # Consider adding retry logic here
-            raise
-        except litellm.exceptions.ContextWindowExceededError as e:
-            logger.error(
-                f"LiteLLM Context Window Exceeded Error for model {model_to_use}. Prompt length: {len(prompt)}. Details: {e}"
-            )
-            raise
-        except litellm.exceptions.APIError as e:  # Catch broader API errors
-            logger.error(f"LiteLLM API Error for model {model_to_use}: {e}")
-            raise
-        except Exception as e:
-            # Catch any other unexpected errors
-            logger.error(
-                f"Unexpected error during model call to {model_to_use}: {type(e).__name__} - {e}"
-            )
-            raise
 
     def update_optimization(self, optimization, status: str) -> None:
         """