opik-optimizer 1.0.5__py3-none-any.whl → 1.1.0__py3-none-any.whl

opik_optimizer/evolutionary_optimizer/prompts.py

@@ -0,0 +1,305 @@
+# Centralized prompt templates used by EvolutionaryOptimizer. This file contains
+# only string builders and constants; it has no side effects.
+
+
+INFER_STYLE_SYSTEM_PROMPT = """You are an expert in linguistic analysis and prompt engineering. Your task is to analyze a few input-output examples from a dataset and provide a concise, actionable description of the desired output style. This description will be used to guide other LLMs in generating and refining prompts.
+
+Focus on characteristics like:
+- **Length**: (e.g., single word, short phrase, one sentence, multiple sentences, a paragraph)
+- **Tone**: (e.g., factual, formal, informal, conversational, academic)
+- **Structure**: (e.g., direct answer first, explanation then answer, list, yes/no then explanation)
+- **Content Details**: (e.g., includes only the answer, includes reasoning, provides examples, avoids pleasantries)
+- **Keywords/Phrasing**: Any recurring keywords or phrasing patterns in the outputs.
+
+Provide a single string that summarizes this style. This summary should be directly usable as an instruction for another LLM.
+For example: 'Outputs should be a single, concise proper noun.' OR 'Outputs should be a short paragraph explaining the reasoning, followed by a direct answer, avoiding conversational pleasantries.' OR 'Outputs are typically 1-2 sentences, providing a direct factual answer.'
+Return ONLY this descriptive string, with no preamble or extra formatting.
+"""
+
+
+def style_inference_user_prompt(examples_str: str) -> str:
+    return f"""Please analyze the following examples from a dataset and provide a concise, actionable description of the REQUIRED output style for the target LLM. Before describing the output style, make sure to understand the dataset content and structure as it can include input, output and metadata fields. This description will be used to guide other LLMs in generating and refining prompts.
+
+{examples_str}
+
+Based on these examples, what is the desired output style description?
+Remember to focus on aspects like length, tone, structure, content details, and any recurring keywords or phrasing patterns in the outputs.
+The description should be a single string that can be directly used as an instruction for another LLM.
+Return ONLY this descriptive string.
+"""
+
+
+def semantic_mutation_system_prompt(output_style_guidance: str | None) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return (
+        "You are a prompt engineering expert. Your goal is to modify prompts to improve their "
+        f"effectiveness in eliciting specific types of answers, particularly matching the style: '{style}'. "
+        "Follow the specific modification instruction provided."
+    )
+
+
+def synonyms_system_prompt() -> str:
+    return (
+        "You are a helpful assistant that provides synonyms. Return only the synonym word, "
+        "no explanation or additional text."
+    )
+
+
+def rephrase_system_prompt() -> str:
+    return (
+        "You are a helpful assistant that rephrases text. Return only the modified phrase, "
+        "no explanation or additional text."
+    )
+
+
+def fresh_start_system_prompt(output_style_guidance: str | None) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return (
+        "You are an expert prompt engineer. Your task is to generate novel, effective prompts from scratch "
+        "based on a task description, specifically aiming for prompts that elicit answers in the style: "
+        f"'{style}'. Output ONLY a raw JSON list of strings."
+    )
+
+
+def variation_system_prompt(output_style_guidance: str | None) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""You are an expert prompt engineer specializing in creating diverse and effective prompts. Given an initial prompt, your task is to generate a diverse set of alternative prompts.
+
+For each prompt variation, consider:
+1. Different levels of specificity and detail, including significantly more detailed and longer versions.
+2. Various ways to structure the instruction, exploring more complex sentence structures and phrasings.
+3. Alternative phrasings that maintain the core intent but vary in style and complexity.
+4. Different emphasis on key components, potentially elaborating on them.
+5. Various ways to express constraints or requirements.
+6. Different approaches to clarity and conciseness, but also explore more verbose and explanatory styles.
+7. Alternative ways to guide the model's response format.
+8. Consider variations that are substantially longer and more descriptive than the original.
+
+The generated prompts should guide a target LLM to produce outputs in the following style: '{style}'
+
+Return a JSON array of prompts with the following structure:
+{{
+    "prompts": [
+        {{
+            "prompt": "alternative prompt 1",
+            "strategy": "brief description of the variation strategy used, e.g., 'focused on eliciting specific output style'"
+        }},
+        {{
+            "prompt": "alternative prompt 2",
+            "strategy": "brief description of the variation strategy used"
+        }}
+    ]
+}}
+Each prompt variation should aim to get the target LLM to produce answers matching the desired style: '{style}'.
+"""
+
+
+def llm_crossover_system_prompt(output_style_guidance: str | None) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""You are an expert prompt engineer specializing in creating novel prompts by intelligently blending existing ones.
+Given two parent prompts, your task is to generate one or two new child prompts that effectively combine the strengths, styles, or core ideas of both parents.
+The children should be coherent and aim to explore a potentially more effective region of the prompt design space, with a key goal of eliciting responses from the target language model in the following style: '{style}'.
+
+Consider the following when generating children:
+- Identify the key instructions, constraints, and desired output formats in each parent, paying attention to any hints about desired output style.
+- Explore ways to merge these elements such that the resulting prompt strongly guides the target LLM towards the desired output style.
+- You can create a child that is a direct blend, or one that takes a primary structure from one parent and incorporates specific elements from the other, always optimizing for clear instruction towards the desired output style.
+- If generating two children, try to make them distinct from each other and from the parents, perhaps by emphasizing different aspects of the parental combination that could lead to the desired output style.
+
+All generated prompts must aim for eliciting answers in the style: '{style}'.
+
+Return a JSON object that is a list of both child prompts. Each child prompt is a list of LLM messages. Example:
+[
+    {{"role": "<role>", "content": "<content>"}},
+    {{"role": "<role>", "content": "<content>"}}
+]
+
+
+"""
+
+
+def radical_innovation_system_prompt(output_style_guidance: str | None) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""You are an expert prompt engineer and a creative problem solver.
+Given a task description and an existing prompt for that task (which might be underperforming), your goal is to generate a new, significantly improved, and potentially very different prompt.
+Do not just make minor edits. Think about alternative approaches, structures, and phrasings that could lead to better performance.
+Consider clarity, specificity, constraints, and how to best guide the language model for the described task TO PRODUCE OUTPUTS IN THE FOLLOWING STYLE: '{style}'.
+Return only the new prompt string, with no preamble or explanation.
+"""
+
+
+def llm_crossover_user_prompt(
+    parent1_messages: list[dict[str, str]],
+    parent2_messages: list[dict[str, str]],
+    output_style_guidance: str | None,
+) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""Parent Prompt 1:
+'''{parent1_messages}'''
+
+Parent Prompt 2:
+'''{parent2_messages}'''
+
+Desired output style from target LLM for children prompts: '{style}'
+
+Please generate TWO child prompts by intelligently blending the ideas, styles, or structures from these two parents, ensuring the children aim to elicit the desired output style.
+Follow the instructions provided in the system prompt regarding the JSON output format:
+[
+    {{"role": "<role>", "content": "<content>"}}, {{"role": "<role>", "content": "<content>"}}
+]
+"""
+
+
+def mutation_strategy_prompts(output_style_guidance: str | None) -> dict[str, str]:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return {
+        "rephrase": (
+            "Create a different way to express the same instruction, possibly with a different "
+            "length or structure, ensuring it still aims for an answer from the target LLM in the style of: "
+            f"'{style}'."
+        ),
+        "simplify": (
+            "Simplify the instruction while maintaining its core meaning, potentially making it more concise, "
+            "to elicit an answer in the style of: "
+            f"'{style}'."
+        ),
+        "elaborate": (
+            "Add more relevant detail and specificity to the instruction, potentially increasing its length, "
+            "but only if it helps achieve a more accurate answer from the target LLM in the style of: "
+            f"'{style}'."
+        ),
+        "restructure": (
+            "Change the structure of the instruction (e.g., reorder sentences, combine/split ideas) while keeping its intent, ensuring the new structure strongly guides towards an output in the style of: "
+            f"'{style}'."
+        ),
+        "focus": (
+            "Emphasize the key aspects of the instruction, perhaps by rephrasing or adding clarifying statements, "
+            "to better elicit an answer in the style of: "
+            f"'{style}'."
+        ),
+        "increase_complexity_and_detail": (
+            "Significantly elaborate on this instruction. Add more details, examples, context, or constraints to make it more comprehensive. "
+            "The goal of this elaboration is to make the prompt itself more detailed, so that it VERY CLEARLY guides the target LLM to produce a highly accurate final answer in the style of: "
+            f"'{style}'. The prompt can be long if needed to achieve this output style."
+        ),
+    }
+
+
+def semantic_mutation_user_prompt(
+    prompt_messages: list[dict[str, str]],
+    task_description: str,
+    output_style_guidance: str | None,
+    strategy_instruction: str,
+) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""Given this prompt: '{prompt_messages}'
+Task context: {task_description}
+Desired output style from target LLM: '{style}'
+Instruction for this modification: {strategy_instruction}.
+Return only the modified prompt message list, nothing else. Make sure to return a valid JSON object.
+"""
+
+
+def radical_innovation_user_prompt(
+    task_description: str,
+    output_style_guidance: str | None,
+    existing_prompt_messages: list[dict[str, str]],
+) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""Task Context:
+{task_description}
+Desired output style from target LLM: '{style}'
+
+Existing Prompt (which may be underperforming):
+'''{existing_prompt_messages}'''
+
+Please generate a new, significantly improved, and potentially very different prompt for this task.
+Focus on alternative approaches, better clarity, or more effective guidance for the language model, aiming for the desired output style.
+Return only the new prompt list object.
+"""
+
+
+def fresh_start_user_prompt(
+    task_description: str,
+    output_style_guidance: str | None,
+    num_to_generate: int,
+) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""Here is a description of a task: ```{task_description}```
+
+The goal is to generate prompts that will make a target LLM produce responses in the following style: ```{style}```.
+
+Please generate {num_to_generate} diverse and effective prompt(s) for a language model to accomplish this task, ensuring they guide towards this specific output style.
+Focus on clarity, completeness, and guiding the model effectively towards the desired style. Explore different structural approaches.
+
+Example of valid response: [
+    ["role": "<role>", "content": "<Prompt targeting specified style.>"],
+    ["role": "<role>", "content": "<Another prompt designed for the output style.>"]
+]
+
+Your response MUST be a valid JSON list of AI messages. Do NOT include any other text, explanations, or Markdown formatting like ```json ... ``` around the list.
+"""
+
+
+def variation_user_prompt(
+    initial_prompt_messages: list[dict[str, str]],
+    task_description: str,
+    output_style_guidance: str | None,
+    num_variations: int,
+) -> str:
+    style = (
+        output_style_guidance
+        or "Produce clear, effective, and high-quality responses suitable for the task."
+    )
+    return f"""Initial prompt:'''{initial_prompt_messages}'''
+Task context: ```{task_description}```
+Desired output style from target LLM: '{style}'
+
+Generate {num_variations} diverse alternative prompts based on the initial prompt above, keeping the task context and desired output style in mind.
+All generated prompt variations should strongly aim to elicit answers from the target LLM matching the style: '{style}'.
+For each variation, consider how to best achieve this style, e.g., by adjusting specificity, structure, phrasing, constraints, or by explicitly requesting it.
+
+Return a JSON array of prompts with the following structure:
+{{
+    "prompts": [
+        {{
+            "prompt": [{{"role": "<role>", "content": "<content>"}}],
+            "strategy": "brief description of the variation strategy used, e.g., 'direct instruction for target style'"
+        }}
+        // ... more prompts if num_variations > 1
+    ]
+}}
+Ensure a good mix of variations, all targeting the specified output style from the end LLM.
+
+Return a valid JSON object that is correctly escaped. Return nothing else, do not include any additional text or Markdown formatting.
+"""

opik_optimizer/evolutionary_optimizer/reporting.py

@@ -1,6 +1,6 @@
 from contextlib import contextmanager
 from io import StringIO
-from typing import Any, List
+from typing import Any
 
 from rich.panel import Panel
 from rich.text import Text
@@ -117,6 +117,16 @@ def initializing_population(verbose: int = 1) -> Any:
                     f"│    Generating {num_fresh_starts} fresh prompts based on the task description."
                 )
 
+        def failed_fresh_prompts(self, num_fresh_starts: int, error: str) -> None:
+            if verbose >= 1:
+                console.print(
+                    Text(
+                        f"│       Failed to generate {num_fresh_starts} fresh prompts: {error}",
+                        style="dim red",
+                    )
+                )
+                console.print("│")
+
         def success_fresh_prompts(self, num_fresh_starts: int) -> None:
             if verbose >= 1:
                 console.print(
@@ -155,7 +165,7 @@ def initializing_population(verbose: int = 1) -> Any:
                 )
                 console.print("│")
 
-        def end(self, population_prompts: List[chat_prompt.ChatPrompt]) -> None:
+        def end(self, population_prompts: list[chat_prompt.ChatPrompt]) -> None:
             if verbose >= 1:
                 console.print(
                     f"│ Successfully initialized population with {len(population_prompts)} prompts."
@@ -205,11 +215,13 @@ def evaluate_initial_population(verbose: int = 1) -> Any:
             if verbose >= 1:
                 if score >= baseline_score:
                     console.print(
-                        Text(f"\r  Prompt {index+1} score was: {score}.", style="green")
+                        Text(
+                            f"\r  Prompt {index + 1} score was: {score}.", style="green"
+                        )
                     )
                 else:
                     console.print(
-                        Text(f"\r  Prompt {index+1} score was: {score}.", style="dim")
+                        Text(f"\r  Prompt {index + 1} score was: {score}.", style="dim")
                     )
 
     # Use our log suppression context manager and yield the reporter

opik_optimizer/evolutionary_optimizer/style_ops.py

@@ -0,0 +1,86 @@
+from typing import TYPE_CHECKING, Any
+
+import logging
+
+import opik
+
+from . import prompts as evo_prompts
+from . import reporting
+from ..optimization_config import chat_prompt
+
+
+logger = logging.getLogger(__name__)
+
+
+class StyleOps:
+    if TYPE_CHECKING:
+        verbose: int
+        _call_model: Any
+
+    def _infer_output_style_from_dataset(
+        self, dataset: opik.Dataset, prompt: chat_prompt.ChatPrompt, n_examples: int = 5
+    ) -> str | None:
+        """Analyzes dataset examples to infer the desired output style using the LLM."""
+        with reporting.infer_output_style(
+            verbose=self.verbose
+        ) as report_infer_output_style:
+            report_infer_output_style.start_style_inference()
+
+            try:
+                items_to_process = dataset.get_items(n_examples)
+            except Exception as e:
+                report_infer_output_style.error(
+                    f"Failed to get items from dataset '{dataset.name}': {e}"
+                )
+                return None
+
+            if not items_to_process:
+                report_infer_output_style.error(
+                    f"Dataset '{dataset.name}' is empty. Cannot infer output style."
+                )
+                return None
+
+            if len(items_to_process) < min(n_examples, 2):
+                report_infer_output_style.error(
+                    f"Not enough dataset items (found {len(items_to_process)}) to reliably infer output style. Need at least {min(n_examples, 2)}."
+                )
+                return None
+
+            examples_str = ""
+            for i, item_content in enumerate(items_to_process):
+                filtered_content: dict[str, str] = {
+                    x: y for x, y in item_content.items() if x != "id"
+                }
+                examples_str += (
+                    f"Example {i + 1}:\nDataset Item:\n{filtered_content}\n---\n"
+                )
+
+            user_prompt_for_style_inference = evo_prompts.style_inference_user_prompt(
+                examples_str
+            )
+
+            try:
+                inferred_style = self._call_model(
+                    messages=[
+                        {
+                            "role": "system",
+                            "content": evo_prompts.INFER_STYLE_SYSTEM_PROMPT,
+                        },
+                        {"role": "user", "content": user_prompt_for_style_inference},
+                    ],
+                    is_reasoning=True,
+                )
+                inferred_style = inferred_style.strip()
+                if inferred_style:
+                    report_infer_output_style.success(inferred_style)
+                    return inferred_style
+                else:
+                    report_infer_output_style.error(
+                        "LLM returned empty string for inferred output style."
+                    )
+                    return None
+            except Exception as e:
+                report_infer_output_style.error(
+                    f"Error during output style inference: {e}"
+                )
+                return None

opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py

@@ -1,4 +1,5 @@
-from typing import Any, Callable, Dict, List, Optional, Tuple, Type
+from typing import Any
+from collections.abc import Callable
 
 import json
 import logging
@@ -56,7 +57,7 @@ Respond only with the JSON object. Do not include any explanation or extra text.
 
 
 class FewShotPromptTemplate(BaseModel):
-    message_list_with_placeholder: List[Dict[str, str]]
+    message_list_with_placeholder: list[dict[str, str]]
     example_template: str
 
 
@@ -119,10 +120,10 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
     def _call_model(
         self,
         model: str,
-        messages: List[Dict[str, str]],
+        messages: list[dict[str, str]],
         seed: int,
-        model_kwargs: Dict[str, Any],
-    ) -> Dict[str, Any]:
+        model_kwargs: dict[str, Any],
+    ) -> dict[str, Any]:
         """
         Args:
             model: The model to use for the call
@@ -159,8 +160,8 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         return response
 
     def _split_dataset(
-        self, dataset: List[Dict[str, Any]], train_ratio: float
-    ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+        self, dataset: list[dict[str, Any]], train_ratio: float
+    ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]:
         """
         Split the dataset into training and validation sets.
 
@@ -194,7 +195,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         self,
         model: str,
         prompt: chat_prompt.ChatPrompt,
-        few_shot_examples: List[Dict[str, Any]],
+        few_shot_examples: list[dict[str, Any]],
     ) -> FewShotPromptTemplate:
         """
         Generate a few-shot prompt template that can be used to insert examples into the prompt.
@@ -215,7 +216,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             "examples": few_shot_examples,
         }
 
-        messages: List[Dict[str, str]] = [
+        messages: list[dict[str, str]] = [
             {"role": "system", "content": SYSTEM_PROMPT_TEMPLATE},
             {"role": "user", "content": json.dumps(user_message)},
         ]
@@ -244,9 +245,9 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         metric: Callable,
         baseline_score: float,
         n_trials: int = 10,
-        optimization_id: Optional[str] = None,
-        experiment_config: Optional[Dict] = None,
-        n_samples: Optional[int] = None,
+        optimization_id: str | None = None,
+        experiment_config: dict | None = None,
+        n_samples: int | None = None,
     ) -> optimization_result.OptimizationResult:
         reporting.start_optimization_run(verbose=self.verbose)
 
@@ -450,6 +451,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             best_score=best_score,
             best_prompt=best_prompt,
             verbose=self.verbose,
+            tools=getattr(prompt, "tools", None),
         )
 
         return optimization_result.OptimizationResult(
@@ -489,9 +491,9 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         dataset: Dataset,
         metric: Callable,
         n_trials: int = 10,
-        agent_class: Optional[Type[OptimizableAgent]] = None,
-        experiment_config: Optional[Dict] = None,
-        n_samples: Optional[int] = None,
+        agent_class: type[OptimizableAgent] | None = None,
+        experiment_config: dict | None = None,
+        n_samples: int | None = None,
     ) -> optimization_result.OptimizationResult:
         """
         Args:
@@ -557,6 +559,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                 "n_samples": n_samples,
             },
             verbose=self.verbose,
+            tools=getattr(prompt, "tools", None),
         )
 
         utils.disable_experiment_reporting()
@@ -614,10 +617,10 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         prompt: chat_prompt.ChatPrompt,
         dataset: opik.Dataset,
         metric: Callable,
-        n_samples: Optional[int] = None,
-        dataset_item_ids: Optional[List[str]] = None,
-        experiment_config: Optional[Dict] = None,
-        optimization_id: Optional[str] = None,
+        n_samples: int | None = None,
+        dataset_item_ids: list[str] | None = None,
+        experiment_config: dict | None = None,
+        optimization_id: str | None = None,
         **kwargs: Any,
     ) -> float:
         """
@@ -673,14 +676,14 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
     def _build_task_from_messages(
         self,
         prompt: chat_prompt.ChatPrompt,
-        messages: List[Dict[str, str]],
-        few_shot_examples: Optional[str] = None,
-    ) -> Callable[[Dict[str, Any]], Dict[str, Any]]:
+        messages: list[dict[str, str]],
+        few_shot_examples: str | None = None,
+    ) -> Callable[[dict[str, Any]], dict[str, Any]]:
         new_prompt = prompt.copy()
         new_prompt.set_messages(messages)
         agent = self.agent_class(new_prompt)
 
-        def llm_task(dataset_item: Dict[str, Any]) -> Dict[str, Any]:
+        def llm_task(dataset_item: dict[str, Any]) -> dict[str, Any]:
             """
             Process a single dataset item through the LLM task.
 

opik_optimizer/few_shot_bayesian_optimizer/reporting.py

@@ -1,6 +1,6 @@
 from contextlib import contextmanager
 from io import StringIO
-from typing import Any, Dict, List, Optional, TYPE_CHECKING
+from typing import Any, Optional, TYPE_CHECKING
 
 from rich.panel import Panel
 from rich.text import Text
@@ -46,9 +46,16 @@ def display_evaluation(
                 yield Reporter()
             finally:
                 if verbose >= 1:
-                    console.print(
-                        Text(f"\r  Baseline score was: {score:.4f}.\n", style="green")
-                    )
+                    if score is not None:
+                        console.print(
+                            Text(
+                                f"\r  Baseline score was: {score:.4f}.\n", style="green"
+                            )
+                        )
+                    else:
+                        console.print(
+                            Text("\r  Baseline score was: None\n", style="red")
+                        )
 
 
 @contextmanager
@@ -121,7 +128,7 @@ def start_optimization_trial(
 
     # Create a simple object with a method to set the score
     class Reporter:
-        def start_trial(self, messages: List[Dict[str, str]]) -> None:
+        def start_trial(self, messages: list[dict[str, str]]) -> None:
             if verbose >= 1:
                 console.print(
                     Text(

opik_optimizer/gepa_optimizer/__init__.py

@@ -0,0 +1,3 @@
+from .gepa_optimizer import GepaOptimizer
+
+__all__ = ["GepaOptimizer"]