opik-optimizer 0.8.1__tar.gz → 0.9.0__tar.gz

{opik_optimizer-0.8.1/src/opik_optimizer.egg-info → opik_optimizer-0.9.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.8.1
+Version: 0.9.0
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML
@@ -13,7 +13,7 @@ Requires-Python: >=3.9,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: opik>=1.7.17
-Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: dspy<=2.6.24,>=2.6.18
 Requires-Dist: litellm
 Requires-Dist: tqdm
 Requires-Dist: datasets
@@ -113,24 +113,20 @@ Available sample datasets for testing:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer import FewShotBayesianOptimizer
-from opik_optimizer.demo import get_or_create_dataset
-
-from opik_optimizer import (
-    MetricConfig,
-    TaskConfig,
-    from_dataset_field,
-    from_llm_response_text,
-)
+from opik_optimizer import FewShotBayesianOptimizer, ChatPrompt
+from opik_optimizer.datasets import hotpot_300
 
 # Load a sample dataset
-hot_pot_dataset = get_or_create_dataset("hotpot-300")
+hot_pot_dataset = hotpot_300()
 
 # Define the instruction for your chat prompt.
 # Input parameters from dataset examples will be interpolated into the full prompt.
-prompt_instruction = """
-Answer the question based on the provided context.
-"""
+prompt = ChatPrompt(
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "{question}"}
+    ]
+)
 project_name = "optimize-few-shot-bayesian-hotpot" # For Comet logging
 
 optimizer = FewShotBayesianOptimizer(
@@ -142,26 +138,14 @@ optimizer = FewShotBayesianOptimizer(
     seed=42,
 )
 
-metric_config = MetricConfig(
-    metric=LevenshteinRatio(project_name=project_name), # Metric for evaluation
-    inputs={
-        "output": from_llm_response_text(), # Get output from LLM
-        "reference": from_dataset_field(name="answer"), # Get reference from dataset
-    },
-)
-
-task_config = TaskConfig(
-    instruction_prompt=prompt_instruction,
-    input_dataset_fields=["question"], # Fields from dataset to use as input
-    output_dataset_field="answer",     # Field in dataset for reference answer
-    use_chat_prompt=True,              # Use chat-style prompting
-)
+def levenshtein_ratio(dataset_item, llm_output):
+    return LevenshteinRatio().score(reference=dataset_item["answer"], output=llm_output)
 
 # Run the optimization
 result = optimizer.optimize_prompt(
+    prompt=prompt,
     dataset=hot_pot_dataset,
-    metric_config=metric_config,
-    task_config=task_config,
+    metric=levenshtein_ratio,
     n_trials=10,   # Number of optimization trials
     n_samples=150, # Number of dataset samples for evaluation per trial
 )

{opik_optimizer-0.8.1 → opik_optimizer-0.9.0}/README.md

@@ -73,24 +73,20 @@ Available sample datasets for testing:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer import FewShotBayesianOptimizer
-from opik_optimizer.demo import get_or_create_dataset
-
-from opik_optimizer import (
-    MetricConfig,
-    TaskConfig,
-    from_dataset_field,
-    from_llm_response_text,
-)
+from opik_optimizer import FewShotBayesianOptimizer, ChatPrompt
+from opik_optimizer.datasets import hotpot_300
 
 # Load a sample dataset
-hot_pot_dataset = get_or_create_dataset("hotpot-300")
+hot_pot_dataset = hotpot_300()
 
 # Define the instruction for your chat prompt.
 # Input parameters from dataset examples will be interpolated into the full prompt.
-prompt_instruction = """
-Answer the question based on the provided context.
-"""
+prompt = ChatPrompt(
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "{question}"}
+    ]
+)
 project_name = "optimize-few-shot-bayesian-hotpot" # For Comet logging
 
 optimizer = FewShotBayesianOptimizer(
@@ -102,26 +98,14 @@ optimizer = FewShotBayesianOptimizer(
     seed=42,
 )
 
-metric_config = MetricConfig(
-    metric=LevenshteinRatio(project_name=project_name), # Metric for evaluation
-    inputs={
-        "output": from_llm_response_text(), # Get output from LLM
-        "reference": from_dataset_field(name="answer"), # Get reference from dataset
-    },
-)
-
-task_config = TaskConfig(
-    instruction_prompt=prompt_instruction,
-    input_dataset_fields=["question"], # Fields from dataset to use as input
-    output_dataset_field="answer",     # Field in dataset for reference answer
-    use_chat_prompt=True,              # Use chat-style prompting
-)
+def levenshtein_ratio(dataset_item, llm_output):
+    return LevenshteinRatio().score(reference=dataset_item["answer"], output=llm_output)
 
 # Run the optimization
 result = optimizer.optimize_prompt(
+    prompt=prompt,
     dataset=hot_pot_dataset,
-    metric_config=metric_config,
-    task_config=task_config,
+    metric=levenshtein_ratio,
     n_trials=10,   # Number of optimization trials
     n_samples=150, # Number of dataset samples for evaluation per trial
 )
@@ -153,4 +137,4 @@ To contribute or use the Opik Optimizer from source:
 
 - Python `>=3.9,<3.13`
 - Opik API key (recommended for full functionality, configure via `opik configure`)
-- API key for your chosen LLM provider (e.g., OpenAI, Anthropic, Gemini), configured as per LiteLLM guidelines.
+- API key for your chosen LLM provider (e.g., OpenAI, Anthropic, Gemini), configured as per LiteLLM guidelines.

opik_optimizer-0.9.0/pyproject.toml

@@ -0,0 +1,3 @@
+[tool.ruff]
+line-length = 100
+extend-select = ["I"]

{opik_optimizer-0.8.1 → opik_optimizer-0.9.0}/setup.py

@@ -1,8 +1,8 @@
-from setuptools import setup, find_packages
+from setuptools import find_packages, setup
 
 setup(
     name="opik_optimizer",
-    version="0.8.1",
+    version="0.9.0",
     description="Agent optimization with Opik",
     author="Comet ML",
     author_email="support@comet.com",
@@ -17,7 +17,7 @@ setup(
     python_requires=">=3.9,<3.13",
     install_requires=[
         "opik>=1.7.17",
-        "dspy>=2.6.18,<3",
+        "dspy>=2.6.18,<=2.6.24",
         "litellm",
         "tqdm",
         "datasets",

{opik_optimizer-0.8.1 → opik_optimizer-0.9.0}/src/opik_optimizer/__init__.py

@@ -1,47 +1,36 @@
 import importlib.metadata
 import logging
-from .logging_config import setup_logging
 
-__version__ = importlib.metadata.version("opik_optimizer")
+from opik.evaluation.models.litellm import warning_filters
 
-# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
-setup_logging(level=logging.WARNING)
+from opik_optimizer.evolutionary_optimizer.evolutionary_optimizer import EvolutionaryOptimizer
 
-# Regular imports
-from .mipro_optimizer import MiproOptimizer
+from . import datasets
 from .base_optimizer import BaseOptimizer
-from .meta_prompt_optimizer import MetaPromptOptimizer
 from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
-from .optimization_config.configs import (
-    MetricConfig,
-    OptimizationConfig,
-    TaskConfig,
-)
-from .optimization_config.mappers import (
-    from_dataset_field,
-    from_llm_response_text,
-)
+from .logging_config import setup_logging
+from .meta_prompt_optimizer import MetaPromptOptimizer
+from .mipro_optimizer import MiproOptimizer
+from .optimization_config.chat_prompt import ChatPrompt
+from .optimization_config.configs import TaskConfig
+from .optimization_result import OptimizationResult
 
-from opik.evaluation.models.litellm import warning_filters
-from . import datasets
+__version__ = importlib.metadata.version("opik_optimizer")
 
-warning_filters.add_warning_filters()
+# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
+setup_logging(level=logging.WARNING)
 
-from .optimization_result import OptimizationResult
-from opik_optimizer.evolutionary_optimizer.evolutionary_optimizer import EvolutionaryOptimizer
+warning_filters.add_warning_filters()
 
 __all__ = [
     "BaseOptimizer",
+    "ChatPrompt",
     "FewShotBayesianOptimizer",
     "MetaPromptOptimizer",
     "MiproOptimizer",
     "EvolutionaryOptimizer",
-    "MetricConfig",
-    "OptimizationConfig",
-    "TaskConfig",
-    "from_dataset_field",
-    "from_llm_response_text",
     "OptimizationResult",
     "setup_logging",
     "datasets",
+    "TaskConfig"
 ]

{opik_optimizer-0.8.1 → opik_optimizer-0.9.0}/src/opik_optimizer/base_optimizer.py

@@ -1,16 +1,16 @@
-from typing import Optional, Union, List, Dict, Any
-import opik
 import logging
 import time
+from abc import abstractmethod
+from typing import Any, Callable, Dict, List, Optional
 
 import litellm
-from . import _throttle
+import opik
 from opik.rest_api.core import ApiError
-
 from pydantic import BaseModel
+
+from . import _throttle, optimization_result
 from .cache_config import initialize_cache
-from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
-from .optimization_config.configs import TaskConfig, MetricConfig
+from .optimization_config import chat_prompt
 
 _limiter = _throttle.get_rate_limiter_for_current_opik_installation()
 
@@ -22,11 +22,13 @@ logger = logging.getLogger(__name__)
 
 
 class OptimizationRound(BaseModel):
+    model_config = {"arbitrary_types_allowed": True}
+
     round_number: int
-    current_prompt: str
+    current_prompt: "chat_prompt.ChatPrompt"
     current_score: float
-    generated_prompts: List[Dict[str, Any]]
-    best_prompt: str
+    generated_prompts: Any
+    best_prompt: "chat_prompt.ChatPrompt"
     best_score: float
     improvement: float
 
@@ -54,46 +56,37 @@ class BaseOptimizer:
         # Initialize shared cache
         initialize_cache()
 
+    @abstractmethod
     def optimize_prompt(
         self,
-        dataset: Union[str, opik.Dataset],
-        metric_config: MetricConfig,
-        task_config: TaskConfig,
-        prompt: str,
-        input_key: str,
-        output_key: str,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: opik.Dataset,
+        metrics: List[Callable],
         experiment_config: Optional[Dict] = None,
         **kwargs,
-    ):
+    ) -> optimization_result.OptimizationResult:
         """
         Optimize a prompt.
 
         Args:
            dataset: Opik dataset name, or Opik dataset
-           metric_config: instance of a MetricConfig
-           task_config: instance of a TaskConfig
+           metrics: A list of metric functions, these functions should have two arguments:
+               dataset_item and llm_output
            prompt: the prompt to optimize
            input_key: input field of dataset
            output_key: output field of dataset
            experiment_config: Optional configuration for the experiment
            **kwargs: Additional arguments for optimization
         """
-        self.dataset = dataset
-        self.metric = metric
-        self.prompt = prompt
-        self.input_key = input_key
-        self.output_key = output_key
-        self.experiment_config = experiment_config
+        pass
 
+    @abstractmethod
     def evaluate_prompt(
         self,
-        dataset: Union[str, opik.Dataset],
-        metric_config: MetricConfig,
-        prompt: str,
-        input_key: str,
-        output_key: str,
-        n_samples: int = 10,
-        task_config: Optional[TaskConfig] = None,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: opik.Dataset,
+        metrics: List[Callable],
+        n_samples: Optional[int] = None,
         dataset_item_ids: Optional[List[str]] = None,
         experiment_config: Optional[Dict] = None,
         **kwargs,
@@ -102,12 +95,10 @@ class BaseOptimizer:
         Evaluate a prompt.
 
         Args:
-           dataset: Opik dataset name, or Opik dataset
-           metric_config: instance of a MetricConfig
-           task_config: instance of a TaskConfig
            prompt: the prompt to evaluate
-           input_key: input field of dataset
-           output_key: output field of dataset
+           dataset: Opik dataset name, or Opik dataset
+           metrics: A list of metric functions, these functions should have two arguments:
+               dataset_item and llm_output
            n_samples: number of items to test in the dataset
            dataset_item_ids: Optional list of dataset item IDs to evaluate
            experiment_config: Optional configuration for the experiment
@@ -116,14 +107,7 @@ class BaseOptimizer:
         Returns:
             float: The evaluation score
         """
-        self.dataset = dataset
-        self.metric_config = metric_config
-        self.task_config = task_config
-        self.prompt = prompt
-        self.input_key = input_key
-        self.output_key = output_key
-        self.experiment_config = experiment_config
-        return 0.0  # Base implementation returns 0
+        pass
 
     def get_history(self) -> List[Dict[str, Any]]:
         """

{opik_optimizer-0.8.1 → opik_optimizer-0.9.0}/src/opik_optimizer/datasets/__init__.py

@@ -1,15 +1,14 @@
-from .hotpot_qa import hotpot_300, hotpot_500
-from .halu_eval import halu_eval_300
-from .tiny_test import tiny_test
-from .gsm8k import gsm8k
 from .ai2_arc import ai2_arc
-from .truthful_qa import truthful_qa
 from .cnn_dailymail import cnn_dailymail
-from .ragbench import ragbench_sentence_relevance
 from .election_questions import election_questions
+from .gsm8k import gsm8k
+from .halu_eval import halu_eval_300
+from .hotpot_qa import hotpot_300, hotpot_500
 from .medhallu import medhallu
 from .rag_hallucinations import rag_hallucinations
-
+from .ragbench import ragbench_sentence_relevance
+from .tiny_test import tiny_test
+from .truthful_qa import truthful_qa
 
 __all__ = [
     "hotpot_300",