opik-optimizer 0.9.0rc0__tar.gz → 0.9.2__tar.gz

{opik_optimizer-0.9.0rc0/src/opik_optimizer.egg-info → opik_optimizer-0.9.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.9.0rc0
+Version: 0.9.2
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/setup.py

@@ -2,7 +2,7 @@ from setuptools import find_packages, setup
 
 setup(
     name="opik_optimizer",
-    version="0.9.0rc0",
+    version="0.9.2",
     description="Agent optimization with Opik",
     author="Comet ML",
     author_email="support@comet.com",

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py

@@ -828,7 +828,35 @@ Return only the new prompt list object.
             auto_continue: Whether to automatically continue optimization
             **kwargs: Additional keyword arguments
         """
-        reporting.display_header(self.__class__.__name__, verbose=self.verbose)
+        if not isinstance(prompt, chat_prompt.ChatPrompt):
+            raise ValueError("Prompt must be a ChatPrompt object")
+        
+        if not isinstance(dataset, opik.Dataset):
+            raise ValueError("Dataset must be a Dataset object")
+        
+        if not isinstance(metric, Callable):
+            raise ValueError("Metric must be a function that takes `dataset_item` and `llm_output` as arguments.")
+
+        # Step 0. Start Opik optimization run
+        opik_optimization_run: Optional[optimization.Optimization] = None
+        try:
+            opik_optimization_run: optimization.Optimization = self._opik_client.create_optimization(
+                dataset_name=dataset.name,
+                objective_name=metric.__name__,
+                metadata={"optimizer": self.__class__.__name__},
+            )
+            self._current_optimization_id = opik_optimization_run.id
+        except Exception as e:
+            logger.warning(f"Opik server error: {e}. Continuing without Opik tracking.")
+            self._current_optimization_id = None
+
+        reporting.display_header(
+            algorithm=self.__class__.__name__,
+            optimization_id=self._current_optimization_id,
+            dataset_id=dataset.id,
+            verbose=self.verbose
+        )
+
         reporting.display_configuration(
             prompt.formatted_messages,
             {
@@ -841,9 +869,9 @@ Return only the new prompt list object.
             verbose=self.verbose
         )
 
+        # Step 1. Step variables and define fitness function
         self.llm_call_counter = 0
         self._history = []
-        self._current_optimization_id = None
         self._current_generation = 0
         self._best_fitness_history = []
         self._generations_without_improvement = 0
@@ -851,7 +879,6 @@ Return only the new prompt list object.
         self._current_population = []
         self._generations_without_overall_improvement = 0
         
-        # Step 0. Define fitness function
         if self.enable_moo:
             def _deap_evaluate_individual_fitness(
                     messages: List[Dict[str, str]]
@@ -884,19 +911,6 @@ Return only the new prompt list object.
                 return (fitness_score,)
         self.toolbox.register("evaluate", _deap_evaluate_individual_fitness)
 
-        # Step 1. Start Opik optimization run
-        opik_optimization_run: Optional[optimization.Optimization] = None
-        try:
-            opik_optimization_run: optimization.Optimization = self._opik_client.create_optimization(
-                dataset_name=dataset.name,
-                objective_name=metric.__name__,
-                metadata={"optimizer": self.__class__.__name__},
-            )
-            self._current_optimization_id = opik_optimization_run.id
-            logger.info(f"Created Opik Optimization run with ID: {self._current_optimization_id}")
-        except Exception as e:
-            logger.warning(f"Opik server error: {e}. Continuing without Opik tracking.")
-
         # Step 2. Compute the initial performance of the prompt
         with reporting.baseline_performance(verbose=self.verbose) as report_baseline_performance:
             initial_eval_result: Tuple[float, float] | Tuple[float, ] = _deap_evaluate_individual_fitness(prompt.formatted_messages)
@@ -976,7 +990,7 @@ Return only the new prompt list object.
                 best_prompt=best_prompt_overall,
                 best_score=best_primary_score_overall,
                 improvement=0.0
-            ).dict()
+            ).model_dump()
             self._add_to_history(initial_round_data)
 
         with reporting.start_evolutionary_algo(verbose=self.verbose) as report_evolutionary_algo:
@@ -1035,7 +1049,7 @@ Return only the new prompt list object.
                     best_prompt=best_prompt_overall,
                     best_score=best_primary_score_overall,
                     improvement=(best_primary_score_overall - initial_primary_score) / abs(initial_primary_score) if initial_primary_score and initial_primary_score != 0 else (1.0 if best_primary_score_overall > 0 else 0.0)
-                ).dict()
+                ).model_dump()
                 self._add_to_history(gen_round_data)
 
         stopped_early_flag = self._generations_without_overall_improvement >= self.DEFAULT_EARLY_STOPPING_GENERATIONS
@@ -1101,6 +1115,7 @@ Return only the new prompt list object.
         # Add final details
         final_details.update({
             "total_generations_run": generation_idx + 1,
+            "num_generations": self.num_generations,
             "population_size": self.population_size,
             "mutation_probability": self.mutation_rate,
             "crossover_probability": self.crossover_rate,
@@ -1132,7 +1147,9 @@ Return only the new prompt list object.
         return OptimizationResult(
             optimizer=self.__class__.__name__,
             prompt=final_best_prompt.formatted_messages, 
-            score=final_primary_score, 
+            score=final_primary_score,
+            initial_prompt=prompt.formatted_messages,
+            initial_score=initial_primary_score,
             metric_name=metric.__name__,
             details=final_details,
             history=self.get_history(),
@@ -1186,6 +1203,7 @@ Return only the new prompt list object.
             response = litellm.completion(
                 model=self.model, messages=messages, **final_call_params
             )
+            self.llm_call_counter += 1
 
             logger.debug(f"Response: {response}")
             return response.choices[0].message.content

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/evolutionary_optimizer/reporting.py

@@ -2,8 +2,6 @@ from contextlib import contextmanager
 from io import StringIO
 from typing import List
 
-import rich
-from rich.console import Console
 from rich.panel import Panel
 from rich.text import Text
 

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py

@@ -1,3 +1,4 @@
+import copy
 import json
 import logging
 import random
@@ -39,8 +40,9 @@ Your task:
     - Add a section title in XML or markdown format. The examples will be provided as `example_1\nexample_2\n...` with each example following the example template.
 - Analyze the examples to infer a consistent structure, and create a single string few_shot_example_template using the Python .format() style. Make sure to follow the following instructions:
     - Unless absolutely relevant, do not return an object but instead a string that can be inserted as part of {FEW_SHOT_EXAMPLE_PLACEHOLDER}
-    - Make sure to include the variables as part of this string so we can before string formatting with actual examples. Only variables available in the examples can be used. Do not use anything else, do not apply any transformations to the variables either.
-    - The few shot examples should include the expected response as the goal is to provide examples of the expected output format.
+    - Make sure to include the variables as part of this string so we can before string formatting with actual examples. Only variables available in the examples can be used.
+    - Do not apply any transformations to the variables either, only the variable name should be included in the format `{{<variable_name>}}`
+    - The few shot examples should include the expected response as the goal is to provide examples of the response.
     - Ensure the format of the few shot examples are consistent with how the model will be called
 
 Return your output as a JSON object with:
@@ -193,6 +195,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
 
     def _run_optimization(
         self,
+        initial_prompt: chat_prompt.ChatPrompt,
         fewshot_prompt_template: FewShotPromptTemplate,
         dataset: Dataset,
         metric: Callable,
@@ -248,13 +251,14 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                 for key, value in example.items():
                     processed_example[key] = str(value)
 
-                try:
-                    processed_demo_examples.append(
-                        fewshot_prompt_template.example_template.format(**processed_example)
-                    )
-                except Exception as e:
-                    logger.error(f"Failed to format fewshot prompt template {fewshot_prompt_template} with example: {processed_example} ")
-                    raise
+                processed_demo_example=fewshot_prompt_template.example_template
+                for key, value in processed_example.items():
+                    try:
+                        processed_demo_example=processed_demo_example.replace(f"{{{key}}}", str(value))
+                    except Exception:
+                        logger.error(f"Failed to format fewshot prompt template {fewshot_prompt_template} with example: {processed_example} ")
+                        raise
+                processed_demo_examples.append(processed_demo_example)
             few_shot_examples = "\n\n".join(processed_demo_examples)
             
             llm_task = self._build_task_from_messages(
@@ -301,7 +305,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             trial_config = {
                 "demo_examples": demo_examples,
                 "message_list_with_placeholder": fewshot_prompt_template.message_list_with_placeholder,
-                "message_list": messages
+                "message_list": messages_for_reporting
             }
             trial.set_user_attr("score", score)
             trial.set_user_attr("config", trial_config)
@@ -363,6 +367,12 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         best_score = best_trial.value
         best_example_indices = best_trial.user_attrs.get("example_indices", [])
 
+        if best_score <= baseline_score:
+            best_score = baseline_score
+            best_prompt = initial_prompt.formatted_messages
+        else:
+            best_prompt = best_trial.user_attrs["config"]["message_list"]
+
         reporting.display_result(
             initial_score=baseline_score,
             best_score=best_score,
@@ -373,9 +383,12 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         return optimization_result.OptimizationResult(
             optimizer=self.__class__.__name__,
             prompt=best_trial.user_attrs["config"]["message_list"],
+            initial_prompt=initial_prompt.formatted_messages,
+            initial_score=baseline_score,
             score=best_score,
             metric_name=metric.__name__,
             details={
+                "initial_score": baseline_score,
                 "chat_messages": best_trial.user_attrs["config"]["message_list"],
                 "prompt_parameter": best_trial.user_attrs["config"],
                 #"n_examples": best_n_examples,
@@ -413,6 +426,16 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         Returns:
             OptimizationResult: Result of the optimization
         """
+        if not isinstance(prompt, chat_prompt.ChatPrompt):
+            raise ValueError("Prompt must be a ChatPrompt object")
+        
+        if not isinstance(dataset, Dataset):
+            raise ValueError("Dataset must be a Dataset object")
+        
+        if not isinstance(metric, Callable):
+            raise ValueError("Metric must be a function that takes `dataset_item` and `llm_output` as arguments.")
+
+
         optimization = None
         try:
             optimization = self._opik_client.create_optimization(
@@ -420,15 +443,22 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                 objective_name=metric.__name__,
                 metadata={"optimizer": self.__class__.__name__},
             )
+            optimization_run_id = optimization.id
         except Exception:
             logger.warning(
                 "Opik server does not support optimizations. Please upgrade opik."
             )
             optimization = None
+            optimization_run_id = None
 
         try:
             # Start experiment reporting
-            reporting.display_header("Few-Shot Bayesian Optimizer", verbose=self.verbose)
+            reporting.display_header(
+                algorithm=self.__class__.__name__,
+                optimization_id=optimization_run_id,
+                dataset_id=dataset.id,
+                verbose=self.verbose
+            )
             reporting.display_configuration(
                 prompt.formatted_messages,
                 optimizer_config={
@@ -467,6 +497,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
 
             # Step 3. Start the optimization process
             result = self._run_optimization(
+                initial_prompt=prompt,
                 fewshot_prompt_template=fewshot_template,
                 dataset=dataset,
                 metric=metric,
@@ -562,17 +593,15 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         self, messages: List[Dict[str, str]], few_shot_examples: Optional[str] = None
     ):
         def llm_task(dataset_item: Dict[str, Any]) -> Dict[str, Any]:
-            prompt_ = [{
-                "role": item["role"],
-                "content": item["content"].format(**dataset_item)
-            } for item in messages]
+            prompt_ = copy.deepcopy(messages)
+            for key, value in dataset_item.items():
+                for item in prompt_:
+                    item["content"] = item["content"].replace("{" + key + "}", str(value))
 
             if few_shot_examples:
-                prompt_ = [{
-                    "role": item["role"],
-                    "content": item["content"].replace(FEW_SHOT_EXAMPLE_PLACEHOLDER, few_shot_examples)
-                } for item in prompt_]
-                
+                for item in prompt_:
+                    item["content"] = item["content"].replace(FEW_SHOT_EXAMPLE_PLACEHOLDER, few_shot_examples)
+            
             response = self._call_model(
                 model=self.model,
                 messages=prompt_,
@@ -584,4 +613,4 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                 mappers.EVALUATED_LLM_TASK_OUTPUT: response.choices[0].message.content
             }
 
-        return llm_task, messages
+        return llm_task

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py

@@ -1,4 +1,5 @@
 import json
+import copy
 import logging
 import os
 from typing import Any, Callable, Dict, List, Optional, overload
@@ -255,7 +256,7 @@ class MetaPromptOptimizer(BaseOptimizer):
             **experiment_config,
             **{
                 "optimizer": self.__class__.__name__,
-                "metric": metric.__name__,
+                "metric": getattr(metric, '__name__', str(metric)),
                 "dataset": dataset.name,
                 "configuration": {
                     "prompt": prompt.formatted_messages,
@@ -300,7 +301,7 @@ class MetaPromptOptimizer(BaseOptimizer):
 
         # Use dataset's get_items with limit for sampling
         logger.debug(
-            f"Starting evaluation with {subset_size if subset_size else 'all'} samples for metric: {metric.__name__}"
+            f"Starting evaluation with {subset_size if subset_size else 'all'} samples for metric: {getattr(metric, '__name__', str(metric))}"
         )
         score = task_evaluator.evaluate(
             dataset=dataset,
@@ -341,8 +342,15 @@ class MetaPromptOptimizer(BaseOptimizer):
         Returns:
             OptimizationResult: Structured result containing optimization details
         """
-        reporting.display_header(self.__class__.__name__, verbose=self.verbose)
+        if not isinstance(prompt, chat_prompt.ChatPrompt):
+            raise ValueError("Prompt must be a ChatPrompt object")
         
+        if not isinstance(dataset, Dataset):
+            raise ValueError("Dataset must be a Dataset object")
+        
+        if not isinstance(metric, Callable):
+            raise ValueError("Metric must be a function that takes `dataset_item` and `llm_output` as arguments.")
+
         total_items = len(dataset.get_items())
         if n_samples is not None and n_samples > total_items:
             logger.warning(
@@ -350,21 +358,12 @@ class MetaPromptOptimizer(BaseOptimizer):
             )
             n_samples = None
 
-        reporting.display_configuration(
-            messages=prompt.formatted_messages,
-            optimizer_config={
-                "optimizer": self.__class__.__name__,
-                "n_samples": n_samples,
-                "auto_continue": auto_continue
-            },
-            verbose=self.verbose
-        )
-
+        
         optimization = None
         try:
             optimization = self._opik_client.create_optimization(
                 dataset_name=dataset.name,
-                objective_name=metric.__name__,
+                objective_name=getattr(metric, '__name__', str(metric)),
                 metadata={"optimizer": self.__class__.__name__},
             )
             logger.debug(f"Created optimization with ID: {optimization.id}")
@@ -374,6 +373,22 @@ class MetaPromptOptimizer(BaseOptimizer):
             )
             optimization = None
 
+        reporting.display_header(
+            algorithm=self.__class__.__name__,
+            optimization_id=optimization.id if optimization is not None else None,
+            dataset_id=dataset.id,
+            verbose=self.verbose
+        )
+        reporting.display_configuration(
+            messages=prompt.formatted_messages,
+            optimizer_config={
+                "optimizer": self.__class__.__name__,
+                "n_samples": n_samples,
+                "auto_continue": auto_continue
+            },
+            verbose=self.verbose
+        )
+
         try:
             result = self._optimize_prompt(
                 optimization_id=optimization.id if optimization is not None else None,
@@ -411,6 +426,7 @@ class MetaPromptOptimizer(BaseOptimizer):
         self.dataset = dataset
         self.prompt = prompt
         self.llm_call_counter = 0 # Reset counter for run
+        initial_prompt: List[Dict[str, str]] = prompt.formatted_messages
 
         current_prompt = prompt.formatted_messages
         experiment_config = experiment_config or {}
@@ -418,7 +434,7 @@ class MetaPromptOptimizer(BaseOptimizer):
             **experiment_config,
             **{
                 "optimizer": self.__class__.__name__,
-                "metric": metric.__name__,
+                "metric": getattr(metric, '__name__', str(metric)),
                 "dataset": self.dataset.name,
                 "configuration": {
                     "prompt": current_prompt,
@@ -527,11 +543,11 @@ class MetaPromptOptimizer(BaseOptimizer):
 
         return self._create_result(
             metric,
-            prompt,
-            best_prompt,
-            best_score,
-            initial_score,
-            rounds,
+            initial_prompt=initial_prompt,
+            best_prompt=best_prompt,
+            best_score=best_score,
+            initial_score=initial_score,
+            rounds=rounds,
         )
 
     def _calculate_improvement(
@@ -581,21 +597,19 @@ class MetaPromptOptimizer(BaseOptimizer):
     def _create_result(
         self,
         metric: Callable,
-        prompt: chat_prompt.ChatPrompt,
-        best_prompt: str,
+        initial_prompt: List[Dict[str, str]],
+        best_prompt: List[Dict[str, str]],
         best_score: float,
         initial_score: float,
         rounds: List[OptimizationRound],
     ) -> OptimizationResult:
         """Create the final OptimizationResult object."""
         details = {
-            "initial_prompt": prompt,
-            "initial_score": initial_score,
             "final_prompt": best_prompt,
             "final_score": best_score,
             "rounds": rounds,
             "total_rounds": len(rounds),
-            "metric_name": metric.__name__,
+            "metric_name": getattr(metric, '__name__', str(metric)),
             "model": self.model,
             "temperature": self.model_kwargs.get("temperature"),
         }
@@ -604,7 +618,9 @@ class MetaPromptOptimizer(BaseOptimizer):
             optimizer=self.__class__.__name__,
             prompt=best_prompt,
             score=best_score,
-            metric_name=metric.__name__,
+            initial_prompt=initial_prompt,
+            initial_score=initial_score,
+            metric_name=getattr(metric, '__name__', str(metric)),
             details=details,
             llm_calls=self.llm_call_counter
         )

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/meta_prompt_optimizer/reporting.py

@@ -1,6 +1,5 @@
 from contextlib import contextmanager
 
-import rich
 from rich.text import Text
 
 from ..reporting_utils import (

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/mipro_optimizer/_lm.py

@@ -82,6 +82,7 @@ class LM(BaseLM):
         self.finetuning_model = finetuning_model
         self.launch_kwargs = launch_kwargs or {}
         self.train_kwargs = train_kwargs or {}
+        self.llm_call_counter = 0
 
         # Handle model-specific configuration for different model families
         model_family = model.split("/")[-1].lower() if "/" in model else model.lower()
@@ -129,6 +130,7 @@ class LM(BaseLM):
         if not getattr(results, "cache_hit", False) and dspy.settings.usage_tracker and hasattr(results, "usage"):
             settings.usage_tracker.add_usage(self.model, dict(results.usage))
 
+        self.llm_call_counter += 1
         return results
 
     def launch(self, launch_kwargs: Optional[Dict[str, Any]] = None):
@@ -323,6 +325,7 @@ def litellm_completion(request: Dict[str, Any], num_retries: int, cache={"no-cac
             **retry_kwargs,
             **request,
         )
+
         chunks = []
         async for chunk in response:
             if caller_predict_id:

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/mipro_optimizer/mipro_optimizer.py

@@ -42,11 +42,10 @@ class MiproOptimizer(BaseOptimizer):
         self.tools = []
         self.num_threads = self.model_kwargs.pop("num_threads", 6)
         self.model_kwargs["model"] = self.model
-        self.llm_call_counter = 0
         # FIXME: add mipro_optimizer=True - It does not count the LLM calls made internally by DSPy during MiproOptimizer.optimizer.compile().
-        lm = LM(**self.model_kwargs)
+        self.lm = LM(**self.model_kwargs)
         opik_callback = OpikCallback(project_name=self.project_name, log_graph=True)
-        dspy.configure(lm=lm, callbacks=[opik_callback])
+        dspy.configure(lm=self.lm, callbacks=[opik_callback])
         logger.debug(f"Initialized MiproOptimizer with model: {model}")
 
     def evaluate_prompt(
@@ -54,7 +53,7 @@ class MiproOptimizer(BaseOptimizer):
         dataset: Union[str, Dataset],
         metric: Callable,
         task_config: TaskConfig,
-        prompt: Union[str, dspy.Module, OptimizationResult] = None,
+        prompt: Optional[Union[str, dspy.Module, OptimizationResult]] = None,
         n_samples: int = 10,
         dataset_item_ids: Optional[List[str]] = None,
         experiment_config: Optional[Dict] = None,
@@ -463,7 +462,7 @@ class MiproOptimizer(BaseOptimizer):
                 metric_name=self.opik_metric.__name__ if hasattr(self, 'opik_metric') else "unknown_metric",
                 details={"error": "No candidate programs generated by MIPRO"},
                 history=mipro_history_processed,
-                llm_calls=self.llm_call_counter
+                llm_calls=self.lm.llm_call_counter
             )
 
         self.module = self.get_best().details["program"]
@@ -488,7 +487,7 @@ class MiproOptimizer(BaseOptimizer):
             demonstrations=best_program_details.demonstrations,
             details=best_program_details.details,
             history=mipro_history_processed,
-            llm_calls=self.llm_call_counter
+            llm_calls=self.lm.llm_call_counter
         )
 
     def get_best(self, position: int = 0) -> OptimizationResult:
@@ -501,7 +500,7 @@ class MiproOptimizer(BaseOptimizer):
                 metric_name=getattr(self, 'opik_metric', None).name if hasattr(self, 'opik_metric') and self.opik_metric else "unknown_metric",
                 details={"error": "No programs generated or compile failed"},
                 history=[],
-                llm_calls=self.llm_call_counter
+                llm_calls=self.lm.llm_call_counter
             )
             
         score = self.best_programs[position]["score"]
@@ -528,5 +527,5 @@ class MiproOptimizer(BaseOptimizer):
             metric_name=self.opik_metric.__name__,
             demonstrations=demos,
             details={"program": program_module},
-            llm_calls=self.llm_call_counter
+            llm_calls=self.lm.llm_call_counter
         )

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/optimization_config/chat_prompt.py

@@ -94,8 +94,8 @@ class ChatPrompt:
         }
 
     @classmethod
-    def model_validate(cls, obj: Any, *, strict: bool | None = None, from_attributes: bool | None = None, 
-                      context: Any | None = None, by_alias: bool | None = None, by_name: bool | None = None) -> 'ChatPrompt':
+    def model_validate(cls, obj: Any, *, strict: Optional[bool] = None, from_attributes: Optional[bool] = None, 
+                      context: Optional[Any] = None, by_alias: Optional[bool] = None, by_name: Optional[bool] = None) -> 'ChatPrompt':
         """Custom validation method to handle nested objects during deserialization."""
         return ChatPrompt(
             system=obj.get('system', None),

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/optimization_result.py

@@ -17,6 +17,10 @@ class OptimizationResult(pydantic.BaseModel):
     score: float
     metric_name: str
     
+    # Initial score
+    initial_prompt: Optional[List[Dict[Literal["role", "content"], str]]] = None
+    initial_score: Optional[float] = None
+    
     details: Dict[str, Any] = pydantic.Field(default_factory=dict)
     history: List[Dict[str, Any]] = []
     llm_calls: Optional[int] = None
@@ -33,7 +37,7 @@ class OptimizationResult(pydantic.BaseModel):
 
     def _calculate_improvement_str(self) -> str:
         """Helper to calculate improvement percentage string."""
-        initial_s = self.details.get("initial_score")
+        initial_s = self.initial_score
         final_s = self.score
 
         # Check if initial score exists and is a number
@@ -60,7 +64,7 @@ class OptimizationResult(pydantic.BaseModel):
         """Provides a clean, well-formatted plain-text summary."""
         separator = "=" * 80
         rounds_ran = len(self.details.get("rounds", []))
-        initial_score = self.details.get("initial_score")
+        initial_score = self.initial_score
         initial_score_str = (
             f"{initial_score:.4f}" if isinstance(initial_score, (int, float)) else "N/A"
         )
@@ -74,7 +78,6 @@ class OptimizationResult(pydantic.BaseModel):
             .replace("[dim]", "")
             .replace("[/dim]", "")
         )
-        stopped_early = self.details.get("stopped_early", "N/A")
 
         model_name = self.details.get("model", "N/A")
         temp = self.details.get("temperature")
@@ -101,7 +104,6 @@ class OptimizationResult(pydantic.BaseModel):
             f"Final Best Score: {final_score_str}",
             f"Total Improvement:{improvement_str.rjust(max(0, 18 - len('Total Improvement:')))}",
             f"Rounds Completed: {rounds_ran}",
-            f"Stopped Early:    {stopped_early}",
             "\nFINAL OPTIMIZED PROMPT / STRUCTURE:",
             "--------------------------------------------------------------------------------",
             f"{final_prompt_display}",
@@ -114,7 +116,7 @@ class OptimizationResult(pydantic.BaseModel):
         """Provides a rich, formatted output for terminals supporting Rich."""
         improvement_str = self._calculate_improvement_str()
         rounds_ran = len(self.details.get("rounds", []))
-        initial_score = self.details.get("initial_score")
+        initial_score = self.initial_score
         initial_score_str = (
             f"{initial_score:.4f}"
             if isinstance(initial_score, (int, float))
@@ -124,8 +126,6 @@ class OptimizationResult(pydantic.BaseModel):
         stopped_early = self.details.get("stopped_early", "N/A")
 
         model_name = self.details.get("model", "[dim]N/A[/dim]")
-        temp = self.details.get("temperature")
-        temp_str = f"{temp:.1f}" if isinstance(temp, (int, float)) else "[dim]N/A[/dim]"
 
         table = rich.table.Table.grid(padding=(0, 1))
         table.add_column(style="dim")
@@ -135,13 +135,12 @@ class OptimizationResult(pydantic.BaseModel):
             "Optimizer:",
             f"[bold]{self.optimizer}[/bold]",
         )
-        table.add_row("Model Used:", f"{model_name} ([dim]Temp:[/dim] {temp_str})")
+        table.add_row("Model Used:", f"{model_name}")
         table.add_row("Metric Evaluated:", f"[bold]{self.metric_name}[/bold]")
         table.add_row("Initial Score:", initial_score_str)
         table.add_row("Final Best Score:", f"[bold cyan]{final_score_str}[/bold cyan]")
         table.add_row("Total Improvement:", improvement_str)
         table.add_row("Rounds Completed:", str(rounds_ran))
-        table.add_row("Stopped Early:", str(stopped_early))
 
         # Display Chat Structure if available
         panel_title = "[bold]Final Optimized Prompt[/bold]"

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/reporting_utils.py

@@ -2,13 +2,14 @@ import logging
 from contextlib import contextmanager
 from typing import Dict, List, Optional
 
-import rich
 from rich import box
 from rich.console import Console, Group
 from rich.panel import Panel
 from rich.progress import track
 from rich.text import Text
 
+from .utils import get_optimization_run_url_by_id
+
 PANEL_WIDTH = 70
 
 def get_console(*args, **kwargs):
@@ -21,10 +22,8 @@ def convert_tqdm_to_rich(description: Optional[str] = None, verbose: int = 1):
     """Context manager to convert tqdm to rich."""
     import opik.evaluation.engine.evaluation_tasks_executor
     
-    optimizer_logger = logging.getLogger('opik_optimizer')
-
     def _tqdm_to_track(iterable, desc, disable, total):
-        disable = verbose == 0 or optimizer_logger.level > logging.INFO
+        disable = verbose == 0
         return track(
             iterable,
             description=description or desc,
@@ -91,16 +90,36 @@ def display_messages(messages: List[Dict[str, str]], prefix: str = ""):
         for line in rendered_panel.splitlines():
             console.print(Text(prefix) + Text.from_ansi(line))
 
-def display_header(algorithm: str, verbose: int = 1):
+def display_header(
+    algorithm: str,
+    optimization_id: Optional[str]=None,
+    dataset_id: Optional[str]=None,
+    verbose: int = 1
+):
     if verbose < 1:
         return
 
+    if optimization_id is not None and dataset_id is not None:    
+        optimization_url = get_optimization_run_url_by_id(
+            optimization_id=optimization_id,
+            dataset_id=dataset_id
+        )
+        
+        # Create a visually appealing panel with an icon and ensure link doesn't wrap
+        
+        link_text = Text("-> View optimization details in your Opik dashboard")
+        link_text.stylize(f"link {optimization_url}", 28, len(link_text))
+    else:
+        link_text = Text("No optimization run link available", style="dim")
+
     content = Text.assemble(
         ("● ", "green"),  
         "Running Opik Evaluation - ",
-        (algorithm, "blue")
-    )
+        (algorithm, "blue"),
+        "\n\n"
+    ).append(link_text)
 
+    
     panel = Panel(
         content,
         box=box.ROUNDED,
@@ -126,7 +145,7 @@ def display_result(initial_score, best_score, best_prompt, verbose: int = 1):
             perc_change = (best_score - initial_score) / initial_score
             content = [Text(f"Prompt was optimized and improved from {initial_score:.4f} to {best_score:.4f} ({perc_change:.2%})", style="bold green")]
     else:
-        content = [Text("Optimization trial did not find a better prompt than the initial one.", style="bold red")]
+        content = [Text(f"Optimization run did not find a better prompt than the initial one.\nScore: {best_score:.4f}", style="dim bold red")]
     
     content.append(Text("\nOptimized prompt:"))
     for i, msg in enumerate(best_prompt):

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/task_evaluator.py

@@ -11,16 +11,23 @@ def _create_metric_class(metric: Callable):
     class MetricClass(base_metric.BaseMetric):
         def __init__(self):
             self.name = metric.__name__
-        
+
         def score(self, llm_output, **kwargs) -> score_result.ScoreResult:
             try:
                 metric_val = metric(dataset_item=kwargs, llm_output=llm_output)
                 if isinstance(metric_val , score_result.ScoreResult):
-                    return metric_val
+                    return score_result.ScoreResult(
+                        name = self.name,
+                        value = metric_val.value,
+                        scoring_failed=metric_val.scoring_failed,
+                        metadata=metric_val.metadata,
+                        reason=metric_val.reason
+                    )
                 else:
                     return score_result.ScoreResult(
                         name = self.name,
-                        value = metric_val
+                        value = metric_val,
+                        scoring_failed=False
                     )
             except Exception:
                 return score_result.ScoreResult(
@@ -71,7 +78,7 @@ def evaluate(
         items = [item for item in items if item.get("id") in dataset_item_ids]
 
     eval_metrics = [_create_metric_class(metric)]
-
+    
     if optimization_id is not None:
         result = opik_evaluator.evaluate_optimization_trial(
             optimization_id=optimization_id,

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/utils.py

@@ -1,23 +1,17 @@
 """Utility functions and constants for the optimizer package."""
 
-from typing import Dict, Any, Optional, TYPE_CHECKING, Type, Literal, Final
-from types import TracebackType
-
-import opik
-from opik.api_objects.opik_client import Opik
-from opik.api_objects.optimization import Optimization
-
+import base64
 import json
 import logging
 import random
 import string
-import base64
 import urllib.parse
-from rich import console
+from types import TracebackType
+from typing import Any, Dict, Final, Literal, Optional, Type
 
-# Type hint for OptimizationResult without circular import
-if TYPE_CHECKING:
-    from .optimization_result import OptimizationResult
+import opik
+from opik.api_objects.opik_client import Opik
+from opik.api_objects.optimization import Optimization
 
 ALLOWED_URL_CHARACTERS: Final[str] = ":/&?="
 logger = logging.getLogger(__name__)
@@ -63,6 +57,7 @@ class OptimizationContextManager:
                 name=self.name,
                 metadata=self.metadata,
             )
+            
             if self.optimization:
                 return self.optimization
             else:
@@ -238,8 +233,10 @@ def ensure_ending_slash(url: str) -> str:
 
 
 def get_optimization_run_url_by_id(
-    dataset_id: str, optimization_id: str, url_override: str
+    dataset_id: str, optimization_id: str
 ) -> str:
+    opik_config = opik.config.get_from_user_inputs()
+    url_override = opik_config.url_override
     encoded_opik_url = base64.b64encode(url_override.encode("utf-8")).decode("utf-8")
 
     run_path = urllib.parse.quote(
@@ -247,18 +244,3 @@ def get_optimization_run_url_by_id(
         safe=ALLOWED_URL_CHARACTERS,
     )
     return urllib.parse.urljoin(ensure_ending_slash(url_override), run_path)
-
-
-def display_optimization_run_link(
-    optimization_id: str, dataset_id: str, url_override: str
-) -> None:
-    console_container = console.Console()
-
-    optimization_url = get_optimization_run_url_by_id(
-        optimization_id=optimization_id,
-        dataset_id=dataset_id,
-        url_override=url_override,
-    )
-    console_container.print(
-        f"View the optimization run [link={optimization_url}]in your Opik dashboard[/link]."
-    )

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2/src/opik_optimizer.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.9.0rc0
+Version: 0.9.2
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML

{opik_optimizer-0.9.0rc0 → opik_optimizer-0.9.2}/src/opik_optimizer/__init__.py

@@ -12,8 +12,8 @@ from .logging_config import setup_logging
 from .meta_prompt_optimizer import MetaPromptOptimizer
 from .mipro_optimizer import MiproOptimizer
 from .optimization_config.chat_prompt import ChatPrompt
-from .optimization_result import OptimizationResult
 from .optimization_config.configs import TaskConfig
+from .optimization_result import OptimizationResult
 
 __version__ = importlib.metadata.version("opik_optimizer")