opik-optimizer 0.8.1__py3-none-any.whl → 0.9.0__py3-none-any.whl

opik_optimizer/__init__.py

@@ -1,47 +1,36 @@
 import importlib.metadata
 import logging
-from .logging_config import setup_logging
 
-__version__ = importlib.metadata.version("opik_optimizer")
+from opik.evaluation.models.litellm import warning_filters
 
-# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
-setup_logging(level=logging.WARNING)
+from opik_optimizer.evolutionary_optimizer.evolutionary_optimizer import EvolutionaryOptimizer
 
-# Regular imports
-from .mipro_optimizer import MiproOptimizer
+from . import datasets
 from .base_optimizer import BaseOptimizer
-from .meta_prompt_optimizer import MetaPromptOptimizer
 from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
-from .optimization_config.configs import (
-    MetricConfig,
-    OptimizationConfig,
-    TaskConfig,
-)
-from .optimization_config.mappers import (
-    from_dataset_field,
-    from_llm_response_text,
-)
+from .logging_config import setup_logging
+from .meta_prompt_optimizer import MetaPromptOptimizer
+from .mipro_optimizer import MiproOptimizer
+from .optimization_config.chat_prompt import ChatPrompt
+from .optimization_config.configs import TaskConfig
+from .optimization_result import OptimizationResult
 
-from opik.evaluation.models.litellm import warning_filters
-from . import datasets
+__version__ = importlib.metadata.version("opik_optimizer")
 
-warning_filters.add_warning_filters()
+# Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
+setup_logging(level=logging.WARNING)
 
-from .optimization_result import OptimizationResult
-from opik_optimizer.evolutionary_optimizer.evolutionary_optimizer import EvolutionaryOptimizer
+warning_filters.add_warning_filters()
 
 __all__ = [
     "BaseOptimizer",
+    "ChatPrompt",
     "FewShotBayesianOptimizer",
     "MetaPromptOptimizer",
     "MiproOptimizer",
     "EvolutionaryOptimizer",
-    "MetricConfig",
-    "OptimizationConfig",
-    "TaskConfig",
-    "from_dataset_field",
-    "from_llm_response_text",
     "OptimizationResult",
     "setup_logging",
     "datasets",
+    "TaskConfig"
 ]

opik_optimizer/base_optimizer.py

@@ -1,16 +1,16 @@
-from typing import Optional, Union, List, Dict, Any
-import opik
 import logging
 import time
+from abc import abstractmethod
+from typing import Any, Callable, Dict, List, Optional
 
 import litellm
-from . import _throttle
+import opik
 from opik.rest_api.core import ApiError
-
 from pydantic import BaseModel
+
+from . import _throttle, optimization_result
 from .cache_config import initialize_cache
-from opik.evaluation.models.litellm import opik_monitor as opik_litellm_monitor
-from .optimization_config.configs import TaskConfig, MetricConfig
+from .optimization_config import chat_prompt
 
 _limiter = _throttle.get_rate_limiter_for_current_opik_installation()
 
@@ -22,11 +22,13 @@ logger = logging.getLogger(__name__)
 
 
 class OptimizationRound(BaseModel):
+    model_config = {"arbitrary_types_allowed": True}
+
     round_number: int
-    current_prompt: str
+    current_prompt: "chat_prompt.ChatPrompt"
     current_score: float
-    generated_prompts: List[Dict[str, Any]]
-    best_prompt: str
+    generated_prompts: Any
+    best_prompt: "chat_prompt.ChatPrompt"
     best_score: float
     improvement: float
 
@@ -54,46 +56,37 @@ class BaseOptimizer:
         # Initialize shared cache
         initialize_cache()
 
+    @abstractmethod
     def optimize_prompt(
         self,
-        dataset: Union[str, opik.Dataset],
-        metric_config: MetricConfig,
-        task_config: TaskConfig,
-        prompt: str,
-        input_key: str,
-        output_key: str,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: opik.Dataset,
+        metrics: List[Callable],
         experiment_config: Optional[Dict] = None,
         **kwargs,
-    ):
+    ) -> optimization_result.OptimizationResult:
         """
         Optimize a prompt.
 
         Args:
            dataset: Opik dataset name, or Opik dataset
-           metric_config: instance of a MetricConfig
-           task_config: instance of a TaskConfig
+           metrics: A list of metric functions, these functions should have two arguments:
+               dataset_item and llm_output
            prompt: the prompt to optimize
            input_key: input field of dataset
            output_key: output field of dataset
            experiment_config: Optional configuration for the experiment
            **kwargs: Additional arguments for optimization
         """
-        self.dataset = dataset
-        self.metric = metric
-        self.prompt = prompt
-        self.input_key = input_key
-        self.output_key = output_key
-        self.experiment_config = experiment_config
+        pass
 
+    @abstractmethod
     def evaluate_prompt(
         self,
-        dataset: Union[str, opik.Dataset],
-        metric_config: MetricConfig,
-        prompt: str,
-        input_key: str,
-        output_key: str,
-        n_samples: int = 10,
-        task_config: Optional[TaskConfig] = None,
+        prompt: chat_prompt.ChatPrompt,
+        dataset: opik.Dataset,
+        metrics: List[Callable],
+        n_samples: Optional[int] = None,
         dataset_item_ids: Optional[List[str]] = None,
         experiment_config: Optional[Dict] = None,
         **kwargs,
@@ -102,12 +95,10 @@ class BaseOptimizer:
         Evaluate a prompt.
 
         Args:
-           dataset: Opik dataset name, or Opik dataset
-           metric_config: instance of a MetricConfig
-           task_config: instance of a TaskConfig
            prompt: the prompt to evaluate
-           input_key: input field of dataset
-           output_key: output field of dataset
+           dataset: Opik dataset name, or Opik dataset
+           metrics: A list of metric functions, these functions should have two arguments:
+               dataset_item and llm_output
            n_samples: number of items to test in the dataset
            dataset_item_ids: Optional list of dataset item IDs to evaluate
            experiment_config: Optional configuration for the experiment
@@ -116,14 +107,7 @@ class BaseOptimizer:
         Returns:
             float: The evaluation score
         """
-        self.dataset = dataset
-        self.metric_config = metric_config
-        self.task_config = task_config
-        self.prompt = prompt
-        self.input_key = input_key
-        self.output_key = output_key
-        self.experiment_config = experiment_config
-        return 0.0  # Base implementation returns 0
+        pass
 
     def get_history(self) -> List[Dict[str, Any]]:
         """

opik_optimizer/datasets/__init__.py

@@ -1,15 +1,14 @@
-from .hotpot_qa import hotpot_300, hotpot_500
-from .halu_eval import halu_eval_300
-from .tiny_test import tiny_test
-from .gsm8k import gsm8k
 from .ai2_arc import ai2_arc
-from .truthful_qa import truthful_qa
 from .cnn_dailymail import cnn_dailymail
-from .ragbench import ragbench_sentence_relevance
 from .election_questions import election_questions
+from .gsm8k import gsm8k
+from .halu_eval import halu_eval_300
+from .hotpot_qa import hotpot_300, hotpot_500
 from .medhallu import medhallu
 from .rag_hallucinations import rag_hallucinations
-
+from .ragbench import ragbench_sentence_relevance
+from .tiny_test import tiny_test
+from .truthful_qa import truthful_qa
 
 __all__ = [
     "hotpot_300",