opik-optimizer 0.7.0__py3-none-any.whl

opik_optimizer/optimization_result.py

@@ -0,0 +1,211 @@
+"""Module containing the OptimizationResult class."""
+
+from typing import Dict, List, Any, Optional, Union, Literal
+import pydantic
+from opik.evaluation.metrics import BaseMetric
+from pydantic import BaseModel, Field
+from .base_optimizer import OptimizationRound  # Adjust import as necessary
+import rich
+
+
+class OptimizationResult(pydantic.BaseModel):
+    """Result of an optimization run."""
+
+    prompt: Union[str, List[Dict[Literal["role", "content"], str]]]
+    score: float
+    metric_name: str
+    metadata: Dict[str, Any] = pydantic.Field(
+        default_factory=dict
+    )  # Default empty dict
+    details: Dict[str, Any] = pydantic.Field(default_factory=dict)  # Default empty dict
+    best_prompt: Optional[str] = None
+    best_score: Optional[float] = None
+    best_metric_name: Optional[str] = None
+    best_details: Optional[Dict[str, Any]] = None
+    all_results: Optional[List[Dict[str, Any]]] = None
+    history: List[Dict[str, Any]] = []
+    metric: Optional[BaseMetric] = None
+    demonstrations: Optional[List[Dict[str, Any]]] = None
+    optimizer: str = "Optimizer"
+    tool_prompts: Optional[Dict[str, str]] = None
+
+    model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
+
+    def _calculate_improvement_str(self) -> str:
+        """Helper to calculate improvement percentage string."""
+        initial_s = self.details.get("initial_score")
+        final_s = self.score
+
+        # Check if initial score exists and is a number
+        if not isinstance(initial_s, (int, float)):
+            return "[dim]N/A (no initial score)[/dim]"
+
+        # Proceed with calculation only if initial_s is valid
+        if initial_s != 0:
+            improvement_pct = (final_s - initial_s) / abs(initial_s)
+            # Basic coloring for rich, plain for str
+            color_start = ""
+            color_end = ""
+            if improvement_pct > 0:
+                color_start, color_end = "[bold green]", "[/bold green]"
+            elif improvement_pct < 0:
+                color_start, color_end = "[bold red]", "[/bold red]"
+            return f"{color_start}{improvement_pct:.2%}{color_end}"
+        elif final_s > 0:
+            return "[bold green]infinite[/bold green] (initial score was 0)"
+        else:
+            return "0.00% (no improvement from 0)"
+
+    def __str__(self) -> str:
+        """Provides a clean, well-formatted plain-text summary."""
+        separator = "=" * 80
+        rounds_ran = len(self.details.get("rounds", []))
+        initial_score = self.details.get("initial_score")
+        initial_score_str = (
+            f"{initial_score:.4f}" if isinstance(initial_score, (int, float)) else "N/A"
+        )
+        final_score_str = f"{self.score:.4f}"
+        improvement_str = (
+            self._calculate_improvement_str()
+            .replace("[bold green]", "")
+            .replace("[/bold green]", "")
+            .replace("[bold red]", "")
+            .replace("[/bold red]", "")
+            .replace("[dim]", "")
+            .replace("[/dim]", "")
+        )
+        stopped_early = self.details.get("stopped_early", "N/A")
+
+        model_name = self.details.get("model", "N/A")
+        temp = self.details.get("temperature")
+        temp_str = f"{temp:.1f}" if isinstance(temp, (int, float)) else "N/A"
+
+        final_prompt_display = self.prompt
+        if self.details.get("prompt_type") == "chat" and self.details.get(
+            "chat_messages"
+        ):
+            try:
+                chat_display = "\n".join(
+                    [
+                        f"  {msg.get('role', 'unknown')}: {str(msg.get('content', ''))[:150]}..."
+                        for msg in self.details["chat_messages"]
+                    ]
+                )
+                final_prompt_display = f"Instruction:\n  {self.prompt}\nFew-Shot Examples (Chat Structure):\n{chat_display}"
+            except Exception:
+                pass
+
+        output = [
+            f"\n{separator}",
+            f"OPTIMIZATION COMPLETE",
+            f"{separator}",
+            f"Optimizer:        {self.details.get('optimizer', type(self).__name__)}",
+            f"Model Used:       {model_name} (Temp: {temp_str})",
+            f"Metric Evaluated: {self.metric_name}",
+            f"Initial Score:    {initial_score_str}",
+            f"Final Best Score: {final_score_str}",
+            f"Total Improvement:{improvement_str.rjust(max(0, 18 - len('Total Improvement:')))}",
+            f"Rounds Completed: {rounds_ran}",
+            f"Stopped Early:    {stopped_early}",
+            f"\nFINAL OPTIMIZED PROMPT / STRUCTURE:",
+            f"--------------------------------------------------------------------------------",
+            f"{final_prompt_display}",
+            f"--------------------------------------------------------------------------------",
+            f"{separator}",
+        ]
+        return "\n".join(output)
+
+    def __rich__(self) -> rich.panel.Panel:
+        """Provides a rich, formatted output for terminals supporting Rich."""
+        improvement_str = self._calculate_improvement_str()
+        rounds_ran = len(self.details.get("rounds", []))
+        initial_score = self.details.get("initial_score")
+        initial_score_str = (
+            f"{initial_score:.4f}"
+            if isinstance(initial_score, (int, float))
+            else "[dim]N/A[/dim]"
+        )
+        final_score_str = f"{self.score:.4f}"
+        stopped_early = self.details.get("stopped_early", "N/A")
+
+        model_name = self.details.get("model", "[dim]N/A[/dim]")
+        temp = self.details.get("temperature")
+        temp_str = f"{temp:.1f}" if isinstance(temp, (int, float)) else "[dim]N/A[/dim]"
+
+        table = rich.table.Table.grid(padding=(0, 1))
+        table.add_column(style="dim")
+        table.add_column()
+
+        table.add_row(
+            "Optimizer:",
+            f"[bold]{self.details.get('optimizer', type(self).__name__)}[/bold]",
+        )
+        table.add_row("Model Used:", f"{model_name} ([dim]Temp:[/dim] {temp_str})")
+        table.add_row("Metric Evaluated:", f"[bold]{self.metric_name}[/bold]")
+        table.add_row("Initial Score:", initial_score_str)
+        table.add_row("Final Best Score:", f"[bold cyan]{final_score_str}[/bold cyan]")
+        table.add_row("Total Improvement:", improvement_str)
+        table.add_row("Rounds Completed:", str(rounds_ran))
+        table.add_row("Stopped Early:", str(stopped_early))
+
+        # Display Chat Structure if available
+        prompt_renderable: Any = rich.text.Text(
+            self.prompt or "", overflow="fold"
+        )  # Default to text
+        panel_title = "[bold]Final Optimized Prompt (Instruction)[/bold]"
+
+        if self.details.get("prompt_type") == "chat" and self.details.get(
+            "chat_messages"
+        ):
+            panel_title = "[bold]Final Optimized Prompt (Chat Structure)[/bold]"
+            try:
+                chat_group_items = [
+                    f"[dim]Instruction:[/dim] [i]{self.prompt}[/i]\n---"
+                ]
+                for msg in self.details["chat_messages"]:
+                    role = msg.get("role", "unknown")
+                    content = str(msg.get("content", ""))
+                    role_style = (
+                        "bold green"
+                        if role == "user"
+                        else (
+                            "bold blue"
+                            if role == "assistant"
+                            else ("bold magenta" if role == "system" else "")
+                        )
+                    )
+                    chat_group_items.append(
+                        f"[{role_style}]{role.capitalize()}:[/] {content}"
+                    )
+                    chat_group_items.append("---")  # Separator
+                prompt_renderable = rich.console.Group(*chat_group_items)
+
+            except Exception:
+                # Fallback to simple text prompt
+                prompt_renderable = rich.text.Text(self.prompt or "", overflow="fold")
+                panel_title = (
+                    "[bold]Final Optimized Prompt (Instruction - fallback)[/bold]"
+                )
+
+        prompt_panel = rich.panel.Panel(
+            prompt_renderable, title=panel_title, border_style="blue", padding=(1, 2)
+        )
+
+        content_group = rich.console.Group(table, "\n", prompt_panel)
+
+        return rich.panel.Panel(
+            content_group,
+            title="[bold yellow]Optimization Complete[/bold yellow]",
+            border_style="yellow",
+            box=rich.box.DOUBLE_EDGE,
+            padding=1,
+        )
+
+    def model_dump(self) -> Dict[str, Any]:
+        return super().model_dump()
+
+    def display(self) -> None:
+        """
+        Displays the OptimizationResult using rich formatting
+        """
+        rich.print(self)

opik_optimizer/task_evaluator.py

@@ -0,0 +1,102 @@
+import opik
+
+from typing import Any, Callable, Dict, List, Optional
+from opik_optimizer.optimization_config.configs import MetricConfig
+from opik.evaluation.metrics import score_result
+
+from opik.evaluation import evaluator as opik_evaluator
+
+
+def evaluate(
+    dataset: opik.Dataset,
+    evaluated_task: Callable[[Dict[str, Any]], Dict[str, Any]],
+    metric_config: MetricConfig,
+    num_threads: int,
+    optimization_id: Optional[str] = None,
+    dataset_item_ids: Optional[List[str]] = None,
+    project_name: Optional[str] = None,
+    n_samples: Optional[int] = None,
+    experiment_config: Optional[Dict[str, Any]] = None,
+) -> float:
+    """
+    Evaluate a task on a dataset.
+
+    Args:
+        dataset: A list of dictionaries representing the dataset.
+        metric_config: The metric configuration to use for evaluation.
+        evaluated_task: A function that takes a dataset item dict as input and returns a dictionary with output(s).
+        dataset_item_ids: Optional list of dataset item IDs to evaluate.
+        project_name: Optional project name for evaluation.
+        n_samples: Optional number of test examples to perform the evaluation and then stop.
+        num_threads: Number of threads to use for evaluation.
+        experiment_config: The dictionary with parameters that describe experiment
+        optimization_id: Optional optimization ID for the experiment.
+
+    Returns:
+        float: The average score of the evaluated task.
+    """
+    items = dataset.get_items(dataset_item_ids)
+    if not items:
+        print("[DEBUG] Empty dataset, returning 0.0")
+        return 0.0
+
+    if dataset_item_ids:
+        items = [item for item in items if item.get("id") in dataset_item_ids]
+
+    if n_samples:
+        items = items[:n_samples]
+
+    # TODO: move to debug logger
+    # print(f"[DEBUG] Starting evaluation with task: {evaluated_task}")
+    # print(f"[DEBUG] Items to evaluate: {items}")
+    # print(f"[DEBUG] Metric config inputs: {metric_config.inputs}")
+    # print(f"[DEBUG] Number of threads: {num_threads}")
+    # print(f"[DEBUG] Project name: {project_name}")
+
+    scoring_key_mapping = {
+        key: value if isinstance(value, str) else value.__name__
+        for key, value in metric_config.inputs.items()
+    }
+    scoring_key_mapping["output"] = "_llm_task_output"
+
+    if optimization_id is not None:
+        result = opik_evaluator.evaluate_optimization_trial(
+            optimization_id=optimization_id,
+            dataset=dataset,
+            task=evaluated_task,
+            project_name=project_name,
+            scoring_key_mapping=scoring_key_mapping,
+            dataset_item_ids=dataset_item_ids,
+            scoring_metrics=[metric_config.metric],
+            task_threads=num_threads,
+            nb_samples=n_samples,
+            experiment_config=experiment_config,
+        )
+    else:
+        result = opik_evaluator.evaluate(
+            dataset=dataset,
+            task=evaluated_task,
+            project_name=project_name,
+            scoring_key_mapping=scoring_key_mapping,
+            dataset_item_ids=dataset_item_ids,
+            scoring_metrics=[metric_config.metric],
+            task_threads=num_threads,
+            nb_samples=n_samples,
+            experiment_config=experiment_config,
+        )
+
+    if not result.test_results:
+        return 0.0
+
+    # We may allow score aggregation customization.
+    score_results: List[score_result.ScoreResult] = [
+        test_result.score_results[0] for test_result in result.test_results
+    ]
+    if not score_results:
+        return 0.0
+
+    avg_score = sum([score_result_.value for score_result_ in score_results]) / len(
+        score_results
+    )
+
+    return avg_score

opik_optimizer/utils.py

@@ -0,0 +1,132 @@
+"""Utility functions and constants for the optimizer package."""
+
+import opik
+import logging
+import random
+import string
+from opik.api_objects.opik_client import Opik
+
+from typing import List, Dict, Any, Optional, Callable, TYPE_CHECKING
+
+# Test dataset name for optimizer examples
+TEST_DATASET_NAME = "tiny-test-optimizer"
+
+# Type hint for OptimizationResult without circular import
+if TYPE_CHECKING:
+    from .optimization_result import OptimizationResult
+
+logger = logging.getLogger(__name__)
+
+
+def format_prompt(prompt: str, **kwargs: Any) -> str:
+    """
+    Format a prompt string with the given keyword arguments.
+
+    Args:
+        prompt: The prompt string to format
+        **kwargs: Keyword arguments to format into the prompt
+
+    Returns:
+        str: The formatted prompt string
+
+    Raises:
+        ValueError: If any required keys are missing from kwargs
+    """
+    try:
+        return prompt.format(**kwargs)
+    except KeyError as e:
+        raise ValueError(f"Missing required key in prompt: {e}")
+
+
+def validate_prompt(prompt: str) -> bool:
+    """
+    Validate a prompt string.
+
+    Args:
+        prompt: The prompt string to validate
+
+    Returns:
+        bool: True if the prompt is valid, False otherwise
+    """
+    if not prompt or not prompt.strip():
+        return False
+    return True
+
+
+def setup_logging(log_level: str = "INFO") -> None:
+    """
+    Setup logging configuration.
+
+    Args:
+        log_level: The log level to use (default: INFO)
+    """
+    valid_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
+    if log_level not in valid_levels:
+        raise ValueError(f"Invalid log level. Must be one of {valid_levels}")
+
+    numeric_level = getattr(logging, log_level.upper())
+    logging.basicConfig(level=numeric_level)
+
+
+def get_random_seed() -> int:
+    """
+    Get a random seed for reproducibility.
+
+    Returns:
+        int: A random seed
+    """
+    import random
+
+    return random.randint(0, 2**32 - 1)
+
+
+def get_or_create_dataset(
+    dataset_name: str,
+    description: str,
+    data_loader: Callable[[], List[Dict[str, Any]]],
+    project_name: Optional[str] = None,
+) -> opik.Dataset:
+    """
+    Get an existing dataset or create a new one if it doesn't exist.
+
+    Args:
+        dataset_name: Name of the dataset
+        description: Description of the dataset
+        data: Optional data to insert into the dataset
+        project_name: Optional project name
+
+    Returns:
+        opik.Dataset: The dataset object
+    """
+    client = Opik(project_name=project_name)
+
+    try:
+        # Try to get existing dataset
+        dataset = client.get_dataset(dataset_name)
+        # If dataset exists but has no data, delete it
+        if not dataset.get_items():
+            print("Dataset exists but is empty - deleting it...")
+            # Delete all items in the dataset
+            items = dataset.get_items()
+            if items:
+                dataset.delete(items_ids=[item.id for item in items])
+            # Delete the dataset itself
+            client.delete_dataset(dataset_name)
+            raise Exception("Dataset deleted, will create new one")
+    except Exception:
+        # Create new dataset
+        print("Creating new dataset...")
+        dataset = client.create_dataset(name=dataset_name, description=description)
+
+        dataset_items = data_loader()
+        dataset.insert(dataset_items)
+
+        # Verify data was added
+        if not dataset.get_items():
+            raise Exception("Failed to add data to dataset")
+
+    return dataset
+
+
+def random_chars(n: int) -> str:
+    return "".join(random.choice(string.ascii_letters) for _ in range(n))

opik_optimizer-0.7.0.dist-info/METADATA

@@ -0,0 +1,35 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.7.0
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: info@comet.ml
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Provides-Extra: dev
+Requires-Dist: adalflow; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary

opik_optimizer-0.7.0.dist-info/RECORD

@@ -0,0 +1,30 @@
+opik_optimizer/__init__.py,sha256=RQc6N4ca7WS34oXGx6pQQjAh9QVRfjtX6dlVS1oJvLw,1847
+opik_optimizer/_throttle.py,sha256=7vcHoISqXbysymwdb1LPAFJB28tOmih9zzZQWajpH0k,1494
+opik_optimizer/base_optimizer.py,sha256=mHi5b_8Ang6o_kl9m0x8NyXMRYq9OFyaidRfw_wWdEY,8348
+opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
+opik_optimizer/logging_config.py,sha256=ELevhxtflYinTo-jVvyQYZbXG7FgAe_b5dPa9y5uLWw,2774
+opik_optimizer/meta_prompt_optimizer.py,sha256=tRxA4YD2AWvKl7fJNg6Oxxay8iyqvdJ8zXy59-qKyUM,46802
+opik_optimizer/optimization_result.py,sha256=e8isosO43o4YOuVr7yYBE_4m_36wodQ9OJ5WC35Mjzo,8765
+opik_optimizer/task_evaluator.py,sha256=MafDMaLeW0_yGPrumLvYF0HzQUKrnpAlM_0N_TPG8tw,3695
+opik_optimizer/utils.py,sha256=HivUsNzbt7BcuZeEvikdER1DaTPUFLJrpaVQ8raZYD8,3637
+opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
+opik_optimizer/demo/cache.py,sha256=U0ovsBT7s39RSy34Mr-vTacdVk_jERKbRhdlGT6gF_0,3562
+opik_optimizer/demo/datasets.py,sha256=hD6JZAQotEDQb4nK7dbnurquILqQsrFRF7nUwon_iXE,22930
+opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
+opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=zh88ElxKcoIkgkwZyXlQ0rwkxkPAoG8BW0DdvRpPhyQ,15106
+opik_optimizer/few_shot_bayesian_optimizer/prompt_parameter.py,sha256=EDsSIFAUOfiZKWLrOAaBDB7Exk7cmIs4ccI95kVa7JY,3118
+opik_optimizer/few_shot_bayesian_optimizer/prompt_templates.py,sha256=HmvD-UeT3aKiiet5cUtULXe6iFPEOo6hxyDE0pH2LnQ,2424
+opik_optimizer/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opik_optimizer/mipro_optimizer/__init__.py,sha256=CF9TVXjOxTobDO1kAS8CD4eyLVzEozxjfgoKwIO6ZpU,44
+opik_optimizer/mipro_optimizer/_lm.py,sha256=UwSEcTLVIt_a-coQbLACNnm-RTMJIzLEyPS4qLfUosg,16316
+opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=r8FKaqvtZq_R7FwGnXqp1foCLk7M7r6M-CMvWbJtP5c,39512
+opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=5QS7OKqOMKe4CD_8W2FMD_qJNmulkvxmOT_YtJ3BllM,14755
+opik_optimizer/mipro_optimizer/utils.py,sha256=4et1JA1QInX3h6Is-_RqzliFwJqkm6tlA0X5CryG60I,3142
+opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opik_optimizer/optimization_config/configs.py,sha256=MYL9H2UAqeyGBlBGWbOZ-6Snto4ZMuXnypgvVuUSW1Y,1132
+opik_optimizer/optimization_config/mappers.py,sha256=RXgTMxPzTQ1AHGke6Zca6rTcfCI7IkCKhQYciaEGSAo,1698
+opik_optimizer-0.7.0.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
+opik_optimizer-0.7.0.dist-info/METADATA,sha256=2M3tCmbCp4kIYzsfF2E_7u3ntgl5VNC7AGwXPKNsWWc,962
+opik_optimizer-0.7.0.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+opik_optimizer-0.7.0.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-0.7.0.dist-info/RECORD,,

opik_optimizer-0.7.0.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.3.1)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

opik_optimizer-0.7.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Comet
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

opik_optimizer-0.7.0.dist-info/top_level.txt

@@ -0,0 +1 @@
+opik_optimizer