opik-optimizer 0.7.8__py3-none-any.whl → 0.8.1__py3-none-any.whl

opik_optimizer/mipro_optimizer/mipro_optimizer.py

@@ -1,6 +1,7 @@
 from typing import Any, Dict, List, Tuple, Union, Optional, Literal
 import os
 import random
+from datetime import datetime
 
 import opik
 
@@ -37,11 +38,13 @@ logger = logging.getLogger(__name__)  # Inherits config from setup_logging
 
 
 class MiproOptimizer(BaseOptimizer):
-    def __init__(self, model, project_name: Optional[str] = None, **model_kwargs):
-        super().__init__(model, project_name, **model_kwargs)
+    def __init__(self, model, project_name: Optional[str] = None, verbose: int = 1, **model_kwargs):
+        super().__init__(model, project_name, verbose=verbose, **model_kwargs)
         self.tools = []
         self.num_threads = self.model_kwargs.pop("num_threads", 6)
         self.model_kwargs["model"] = self.model
+        self.llm_call_counter = 0
+        # FIXME: add mipro_optimizer=True - It does not count the LLM calls made internally by DSPy during MiproOptimizer.optimizer.compile().
         lm = LM(**self.model_kwargs)
         opik_callback = OpikCallback(project_name=self.project_name, log_graph=True)
         dspy.configure(lm=lm, callbacks=[opik_callback])
@@ -56,6 +59,7 @@ class MiproOptimizer(BaseOptimizer):
         n_samples: int = 10,
         dataset_item_ids: Optional[List[str]] = None,
         experiment_config: Optional[Dict] = None,
+        verbose: int = 1,
         **kwargs,
     ) -> float:
         """
@@ -69,6 +73,7 @@ class MiproOptimizer(BaseOptimizer):
             n_samples: number of items to test in the dataset
             dataset_item_ids: Optional list of dataset item IDs to evaluate
             experiment_config: Optional configuration for the experiment
+            verbose: Verbosity level
             **kwargs: Additional arguments for evaluation
 
         Returns:
@@ -76,10 +81,14 @@ class MiproOptimizer(BaseOptimizer):
         """
         # FIMXE: call super when it is ready
         # FIXME: Intermediate values:
+        self.llm_call_counter += 1
         metric = metric_config.metric
         input_key = task_config.input_dataset_fields[0]  # FIXME: allow all inputs
         output_key = task_config.output_dataset_field
 
+        # Kwargs might contain n_samples, passed from run_benchmark.py
+        n_samples = kwargs.pop("n_samples", None) # Get n_samples from kwargs if present
+
         if isinstance(dataset, str):
             opik_client = opik.Opik(project_name=self.project_name)
             dataset = opik_client.get_dataset(dataset)
@@ -144,12 +153,32 @@ class MiproOptimizer(BaseOptimizer):
 
             return result
 
-        if n_samples is not None:
-            if dataset_item_ids is not None:
-                raise Exception("Can't use n_samples and dataset_item_ids")
+        # Robust n_samples handling for selecting dataset_item_ids
+        dataset_items_for_eval = dataset.get_items()
+        num_total_items = len(dataset_items_for_eval)
+        dataset_item_ids_to_use = dataset_item_ids # Use provided IDs if any
 
-            all_ids = [dataset_item["id"] for dataset_item in dataset.get_items()]
-            dataset_item_ids = random.sample(all_ids, n_samples)
+        if n_samples is not None: # If n_samples is specified by the caller (run_benchmark.py)
+            if dataset_item_ids is not None:
+                # This case should ideally be an error or a clear precedence rule.
+                # For now, let's assume if dataset_item_ids is provided, it takes precedence over n_samples.
+                logger.warning("MiproOptimizer.evaluate_prompt: Both n_samples and dataset_item_ids provided. Using provided dataset_item_ids.")
+                # dataset_item_ids_to_use is already dataset_item_ids
+            elif n_samples > num_total_items:
+                logger.warning(f"MiproOptimizer.evaluate_prompt: n_samples ({n_samples}) > total items ({num_total_items}). Using all {num_total_items} items.")
+                dataset_item_ids_to_use = None # opik.evaluation.evaluate handles None as all items
+            elif n_samples <= 0:
+                logger.warning(f"MiproOptimizer.evaluate_prompt: n_samples ({n_samples}) is <= 0. Using all {num_total_items} items.")
+                dataset_item_ids_to_use = None
+            else:
+                # n_samples is valid and dataset_item_ids was not provided, so sample now.
+                all_ids = [item["id"] for item in dataset_items_for_eval]
+                dataset_item_ids_to_use = random.sample(all_ids, n_samples)
+                logger.info(f"MiproOptimizer.evaluate_prompt: Sampled {n_samples} items for evaluation.")
+        else: # n_samples is None
+            if dataset_item_ids is None:
+                logger.info(f"MiproOptimizer.evaluate_prompt: n_samples is None and dataset_item_ids is None. Using all {num_total_items} items.")
+            # dataset_item_ids_to_use is already dataset_item_ids (which could be None)
 
         experiment_config = experiment_config or {}
         experiment_config = {
@@ -171,9 +200,10 @@ class MiproOptimizer(BaseOptimizer):
             # "reference" needs to match metric
             scoring_key_mapping={"reference": output_key},
             task_threads=self.num_threads,
-            dataset_item_ids=dataset_item_ids,
+            dataset_item_ids=dataset_item_ids_to_use,
             project_name=self.project_name,
             experiment_config=experiment_config,
+            verbose=verbose,
         )
 
         # Calculate average score across all metrics
@@ -207,6 +237,7 @@ class MiproOptimizer(BaseOptimizer):
             optimization = self._opik_client.create_optimization(
                 dataset_name=dataset.name,
                 objective_name=metric_config.metric.name,
+                metadata={"optimizer": self.__class__.__name__},
             )
         except Exception:
             logger.warning(
@@ -284,13 +315,14 @@ class MiproOptimizer(BaseOptimizer):
         **kwargs,
     ) -> None:
         # FIXME: Intermediate values:
+        self.llm_call_counter = 0
         metric = metric_config.metric
         prompt = task_config.instruction_prompt
         input_key = task_config.input_dataset_fields[0]  # FIXME: allow all
         output_key = task_config.output_dataset_field
         self.tools = task_config.tools
         self.num_candidates = num_candidates
-        self.seed = 9
+        self.seed = 42
         self.input_key = input_key
         self.output_key = output_key
         self.prompt = prompt
@@ -347,7 +379,7 @@ class MiproOptimizer(BaseOptimizer):
             metric=self.metric_function,
             auto=self.auto,
             num_threads=self.num_threads,
-            verbose=False,
+            verbose=(self.verbose == 1),
             num_candidates=self.num_candidates,
             seed=self.seed,
             opik_prompt_task_config=task_config,
@@ -373,6 +405,9 @@ class MiproOptimizer(BaseOptimizer):
         """
         Continue to look for optimizations
         """
+        if not hasattr(self, 'optimizer') or not self.optimizer:
+            raise RuntimeError("MiproOptimizer not prepared. Call prepare_optimize_prompt first.")
+
         self.results = self.optimizer.compile(
             student=self.module,
             trainset=self.trainset,
@@ -385,12 +420,114 @@ class MiproOptimizer(BaseOptimizer):
             key=lambda item: item["score"],
             reverse=True,
         )
+
+        mipro_history_processed = []
+        # self.num_candidates is set in prepare_optimize_prompt, defaults to 10
+        # If self.num_candidates is 0 or None, this logic might break or be odd.
+        # Add a safeguard for num_candidates_per_round if self.num_candidates is not usable.
+        num_candidates_per_round = self.num_candidates if hasattr(self, 'num_candidates') and self.num_candidates and self.num_candidates > 0 else 1
+
+        for i, candidate_data in enumerate(self.results.candidate_programs):
+            program_module = candidate_data.get("program")
+            instruction = "N/A"
+            if hasattr(program_module, 'signature') and hasattr(program_module.signature, 'instructions'):
+                instruction = program_module.signature.instructions
+            elif hasattr(program_module, 'extended_signature') and hasattr(program_module.extended_signature, 'instructions'):
+                instruction = program_module.extended_signature.instructions
+            elif hasattr(program_module, 'predictor') and hasattr(program_module.predictor, 'signature') and hasattr(program_module.predictor.signature, 'instructions'):
+                instruction = program_module.predictor.signature.instructions
+
+            # Remove R and C calculation for Mipro as its history is flat
+            # current_round_number = (i // num_candidates_per_round) + 1
+            # current_candidate_in_round = (i % num_candidates_per_round) + 1
+
+            iter_detail = {
+                "iteration": i + 1,
+                # "round_number": current_round_number, # Remove round_number
+                # "candidate_in_round": current_candidate_in_round, # Remove candidate_in_round
+                "timestamp": datetime.now().isoformat(),
+                "prompt_candidate": instruction,
+                "parameters_used": {
+                    "program_summary": str(program_module)[:500]
+                },
+                "scores": [], # Initialize scores list
+                "tokens_used": None, # TODO: add tokens_used
+                "cost": None, # TODO: add cost
+                "duration_seconds": None, # TODO: add duration_seconds
+            }
+
+            current_score = candidate_data.get("score")
+            metric_name_for_history = self.opik_metric.name if hasattr(self, 'opik_metric') and self.opik_metric else "unknown_metric"
+
+            # Unscale if it's a known 0-1 metric that MIPRO might scale to 0-100
+            # For now, specifically targeting Levenshtein-like metrics
+            if isinstance(current_score, (float, int)) and \
+               ("levenshtein" in metric_name_for_history.lower() or "similarity" in metric_name_for_history.lower()):
+                # Assuming scores like 32.4 are 0-1 scores scaled by 100
+                if abs(current_score) > 1.0: # A simple check to see if it looks scaled
+                    logger.debug(f"Mipro history: Unscaling score {current_score} for metric {metric_name_for_history} by dividing by 100.")
+                    current_score /= 100.0
+            
+            iter_detail["scores"].append({
+                "metric_name": metric_name_for_history,
+                "score": current_score,
+                "opik_evaluation_id": None # TODO: add opik_evaluation_id
+            })
+            mipro_history_processed.append(iter_detail)
+
+        if not self.best_programs:
+            logger.warning("MIPRO compile returned no candidate programs.")
+            return OptimizationResult(
+                optimizer="MiproOptimizer",
+                prompt=self.prompt,
+                score=0.0,
+                metric_name=self.opik_metric.name if hasattr(self, 'opik_metric') else "unknown_metric",
+                details={"error": "No candidate programs generated by MIPRO"},
+                history=mipro_history_processed,
+                llm_calls=self.llm_call_counter
+            )
+
         self.module = self.get_best().details["program"]
-        return self.get_best()
+        best_program_details = self.get_best()
+        
+        # Unscale the main score if necessary, similar to history scores
+        final_best_score = best_program_details.score
+        final_metric_name = best_program_details.metric_name
+        if isinstance(final_best_score, (float, int)) and \
+           final_metric_name and \
+           ("levenshtein" in final_metric_name.lower() or "similarity" in final_metric_name.lower()):
+            if abs(final_best_score) > 1.0: # A simple check to see if it looks scaled
+                logger.debug(f"Mipro main result: Unscaling score {final_best_score} for metric {final_metric_name} by dividing by 100.")
+                final_best_score /= 100.0
+
+        return OptimizationResult(
+            optimizer="MiproOptimizer",
+            prompt=best_program_details.prompt,
+            tool_prompts=best_program_details.tool_prompts,
+            score=final_best_score, # Use the potentially unscaled score
+            metric_name=final_metric_name,
+            demonstrations=best_program_details.demonstrations,
+            details=best_program_details.details,
+            history=mipro_history_processed,
+            llm_calls=self.llm_call_counter
+        )
 
     def get_best(self, position: int = 0) -> OptimizationResult:
+        if not hasattr(self, 'best_programs') or not self.best_programs:
+            logger.error("get_best() called but no best_programs found. MIPRO compile might have failed or yielded no results.")
+            return OptimizationResult(
+                optimizer="MiproOptimizer",
+                prompt=getattr(self, 'prompt', "Error: Initial prompt not found"), 
+                score=0.0, 
+                metric_name=getattr(self, 'opik_metric', None).name if hasattr(self, 'opik_metric') and self.opik_metric else "unknown_metric",
+                details={"error": "No programs generated or compile failed"},
+                history=[],
+                llm_calls=self.llm_call_counter
+            )
+            
         score = self.best_programs[position]["score"]
-        state = self.best_programs[position]["program"].dump_state()
+        program_module = self.best_programs[position]["program"]
+        state = program_module.dump_state()
         if self.tools:
             tool_names = [tool.__name__ for tool in self.tools]
             tool_prompts = get_tool_prompts(
@@ -410,5 +547,6 @@ class MiproOptimizer(BaseOptimizer):
             score=score,
             metric_name=self.opik_metric.name,
             demonstrations=demos,
-            details={"program": self.best_programs[position]["program"]},
+            details={"program": program_module},
+            llm_calls=self.llm_call_counter
         )

opik_optimizer/optimization_result.py

@@ -7,6 +7,15 @@ from pydantic import BaseModel, Field
 from .base_optimizer import OptimizationRound  # Adjust import as necessary
 import rich
 
+class OptimizationStep(BaseModel):
+    """Represents a single step or trial in an optimization process."""
+    step: int
+    score: Optional[float] = None
+    prompt: Optional[Union[str, List[Dict[str, str]]]] = None
+    parameters: Optional[Dict[str, Any]] = None
+    timestamp: Optional[str] = None
+    # Add other relevant details per step if needed
+
 
 class OptimizationResult(pydantic.BaseModel):
     """Result of an optimization run."""
@@ -28,6 +37,8 @@ class OptimizationResult(pydantic.BaseModel):
     demonstrations: Optional[List[Dict[str, Any]]] = None
     optimizer: str = "Optimizer"
     tool_prompts: Optional[Dict[str, str]] = None
+    opik_metadata: Optional[Dict[str, Any]] = None
+    llm_calls: Optional[int] = None
 
     model_config = pydantic.ConfigDict(arbitrary_types_allowed=True)
 

opik_optimizer/task_evaluator.py

@@ -1,11 +1,12 @@
 import opik
-
+import logging
 from typing import Any, Callable, Dict, List, Optional
 from opik_optimizer.optimization_config.configs import MetricConfig
 from opik.evaluation.metrics import score_result
 
 from opik.evaluation import evaluator as opik_evaluator
 
+logger = logging.getLogger(__name__)
 
 def evaluate(
     dataset: opik.Dataset,
@@ -17,6 +18,7 @@ def evaluate(
     project_name: Optional[str] = None,
     n_samples: Optional[int] = None,
     experiment_config: Optional[Dict[str, Any]] = None,
+    verbose: int = 1,
 ) -> float:
     """
     Evaluate a task on a dataset.
@@ -31,6 +33,7 @@ def evaluate(
         num_threads: Number of threads to use for evaluation.
         experiment_config: The dictionary with parameters that describe experiment
         optimization_id: Optional optimization ID for the experiment.
+        verbose: Whether to print debug information.
 
     Returns:
         float: The average score of the evaluated task.
@@ -71,6 +74,7 @@ def evaluate(
             task_threads=num_threads,
             nb_samples=n_samples,
             experiment_config=experiment_config,
+            verbose=verbose,
         )
     else:
         result = opik_evaluator.evaluate(
@@ -83,6 +87,7 @@ def evaluate(
             task_threads=num_threads,
             nb_samples=n_samples,
             experiment_config=experiment_config,
+            verbose=verbose,
         )
 
     if not result.test_results:

opik_optimizer/utils.py

@@ -8,9 +8,6 @@ from opik.api_objects.opik_client import Opik
 
 from typing import List, Dict, Any, Optional, Callable, TYPE_CHECKING
 
-# Test dataset name for optimizer examples
-TEST_DATASET_NAME = "tiny-test-optimizer"
-
 # Type hint for OptimizationResult without circular import
 if TYPE_CHECKING:
     from .optimization_result import OptimizationResult
@@ -79,54 +76,5 @@ def get_random_seed() -> int:
 
     return random.randint(0, 2**32 - 1)
 
-
-def get_or_create_dataset(
-    dataset_name: str,
-    description: str,
-    data_loader: Callable[[], List[Dict[str, Any]]],
-    project_name: Optional[str] = None,
-) -> opik.Dataset:
-    """
-    Get an existing dataset or create a new one if it doesn't exist.
-
-    Args:
-        dataset_name: Name of the dataset
-        description: Description of the dataset
-        data: Optional data to insert into the dataset
-        project_name: Optional project name
-
-    Returns:
-        opik.Dataset: The dataset object
-    """
-    client = Opik(project_name=project_name)
-
-    try:
-        # Try to get existing dataset
-        dataset = client.get_dataset(dataset_name)
-        # If dataset exists but has no data, delete it
-        if not dataset.get_items():
-            print("Dataset exists but is empty - deleting it...")
-            # Delete all items in the dataset
-            items = dataset.get_items()
-            if items:
-                dataset.delete(items_ids=[item.id for item in items])
-            # Delete the dataset itself
-            client.delete_dataset(dataset_name)
-            raise Exception("Dataset deleted, will create new one")
-    except Exception:
-        # Create new dataset
-        print("Creating new dataset...")
-        dataset = client.create_dataset(name=dataset_name, description=description)
-
-        dataset_items = data_loader()
-        dataset.insert(dataset_items)
-
-        # Verify data was added
-        if not dataset.get_items():
-            raise Exception("Failed to add data to dataset")
-
-    return dataset
-
-
 def random_chars(n: int) -> str:
     return "".join(random.choice(string.ascii_letters) for _ in range(n))

opik_optimizer-0.8.1.dist-info/METADATA

@@ -0,0 +1,196 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.8.1
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: support@comet.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Requires-Dist: pyrate-limiter
+Requires-Dist: deap>=1.4.3
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Opik Agent Optimizer
+
+[![PyPI version](https://img.shields.io/pypi/v/opik-optimizer.svg)](https://pypi.org/project/opik-optimizer/)
+[![Python versions](https://img.shields.io/pypi/pyversions/opik-optimizer.svg)](https://pypi.org/project/opik-optimizer/)
+[![Downloads](https://static.pepy.tech/badge/opik-optimizer)](https://pepy.tech/project/opik-optimizer)
+[![License](https://img.shields.io/github/license/comet-ml/opik)](https://github.com/comet-ml/opik/blob/main/LICENSE)
+
+The Opik Agent Optimizer refines your prompts to achieve better performance from your Large Language Models (LLMs). It supports a variety of optimization algorithms, including:
+
+* EvolutionaryOptimizer
+* FewShotBayesianOptimizer
+* MetaPromptOptimizer
+* MiproOptimizer
+
+Opik Optimizer is a component of the [Opik platform](https://github.com/comet-ml/opik), an open-source LLM evaluation platform by Comet.
+For more information about the broader Opik ecosystem, visit our [Website](https://www.comet.com/site/products/opik/) or [Documentation](https://www.comet.com/docs/opik/).
+
+## Quickstart
+
+Explore Opik Optimizer's capabilities with our interactive notebook:
+
+<a href="https://colab.research.google.com/github/comet-ml/opik/blob/main/sdks/opik_optimizer/notebooks/OpikOptimizerIntro.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open in Colab"/>
+</a>
+
+## Setup
+
+To get started with Opik Optimizer, follow these steps:
+
+1.  **Install the package:**
+    ```bash
+    # using pip
+    pip install opik-optimizer
+
+    # using uv (faster)
+    uv pip install opik-optimizer
+    ```
+
+2.  **Configure Opik (Optional, for advanced features):**
+    If you plan to log optimization experiments to Comet or use Opik Datasets, you'll need to configure the Opik client:
+    ```bash
+    # Install the main Opik CLI (if not already installed)
+    pip install opik
+
+    # Configure your Comet API key and workspace
+    opik configure
+    # When prompted, enter your Opik API key and workspace details.
+    ```
+    Using Opik with Comet allows you to track your optimization runs, compare results, and manage datasets seamlessly.
+
+3.  **Set up LLM Provider API Keys:**
+    Ensure your environment variables are set for the LLM(s) you intend to use. For example, for OpenAI models:
+    ```bash
+    export OPENAI_API_KEY="your_openai_api_key"
+    ```
+    The optimizer utilizes LiteLLM, so you can configure keys for various providers as per LiteLLM's documentation.
+
+You'll typically need:
+
+*   An LLM model name (e.g., "gpt-4o-mini", "claude-3-haiku-20240307").
+*   An [Opik Dataset](https://www.comet.com/docs/opik/evaluation/manage_datasets/) (or a compatible local dataset/data generator).
+*   An [Opik Metric](https://www.comet.com/docs/opik/evaluation/metrics/overview/) (or a custom evaluation function).
+*   A starting prompt (template string).
+
+## Example
+
+Here's a brief example of how to use the `FewShotBayesianOptimizer`. We'll use a sample dataset provided by Opik.
+
+Available sample datasets for testing:
+*   `"tiny-test"`
+*   `"halu-eval-300"`
+*   `"hotpot-300"`
+
+```python
+from opik.evaluation.metrics import LevenshteinRatio
+from opik_optimizer import FewShotBayesianOptimizer
+from opik_optimizer.demo import get_or_create_dataset
+
+from opik_optimizer import (
+    MetricConfig,
+    TaskConfig,
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+# Load a sample dataset
+hot_pot_dataset = get_or_create_dataset("hotpot-300")
+
+# Define the instruction for your chat prompt.
+# Input parameters from dataset examples will be interpolated into the full prompt.
+prompt_instruction = """
+Answer the question based on the provided context.
+"""
+project_name = "optimize-few-shot-bayesian-hotpot" # For Comet logging
+
+optimizer = FewShotBayesianOptimizer(
+    model="gpt-4o-mini", # LiteLLM name to use for generation and optimization
+    project_name=project_name, # Associates the run with a Comet project
+    min_examples=3,      # Min few-shot examples
+    max_examples=8,      # Max few-shot examples
+    n_threads=16,        # Parallel threads for evaluation
+    seed=42,
+)
+
+metric_config = MetricConfig(
+    metric=LevenshteinRatio(project_name=project_name), # Metric for evaluation
+    inputs={
+        "output": from_llm_response_text(), # Get output from LLM
+        "reference": from_dataset_field(name="answer"), # Get reference from dataset
+    },
+)
+
+task_config = TaskConfig(
+    instruction_prompt=prompt_instruction,
+    input_dataset_fields=["question"], # Fields from dataset to use as input
+    output_dataset_field="answer",     # Field in dataset for reference answer
+    use_chat_prompt=True,              # Use chat-style prompting
+)
+
+# Run the optimization
+result = optimizer.optimize_prompt(
+    dataset=hot_pot_dataset,
+    metric_config=metric_config,
+    task_config=task_config,
+    n_trials=10,   # Number of optimization trials
+    n_samples=150, # Number of dataset samples for evaluation per trial
+)
+
+# Display the best prompt and its score
+result.display()
+```
+The `result` object contains the optimized prompt, evaluation scores, and other details from the optimization process. If `project_name` is provided and Opik is configured, results will also be logged to your Comet workspace.
+
+## Development
+
+To contribute or use the Opik Optimizer from source:
+
+1.  **Clone the Opik repository:**
+    ```bash
+    git clone git@github.com:comet-ml/opik.git
+    ```
+2.  **Navigate to the optimizer's directory:**
+    ```bash
+    cd opik/sdks/opik_optimizer  # Adjust 'opik' if you cloned into a different folder name
+    ```
+3.  **Install in editable mode (with development dependencies):**
+    ```bash
+    pip install -e .[dev]
+    ```
+    The `[dev]` extra installs dependencies useful for development, such as `pytest`.
+
+## Requirements
+
+- Python `>=3.9,<3.13`
+- Opik API key (recommended for full functionality, configure via `opik configure`)
+- API key for your chosen LLM provider (e.g., OpenAI, Anthropic, Gemini), configured as per LiteLLM guidelines.

opik_optimizer-0.8.1.dist-info/RECORD

@@ -0,0 +1,45 @@
+opik_optimizer/__init__.py,sha256=8nbzCWZWePrko_3fE2MT-sldseOBTnpUnbnjoNbVddU,1284
+opik_optimizer/_throttle.py,sha256=ztub8qlwz4u0GVA2TIoLig0D1Cs0hJ7_o_SnT_C7Nmk,1360
+opik_optimizer/base_optimizer.py,sha256=Gp96LSmWBHpC5rOoDkDUunRayvqf-A510TMwjsVhZYk,5018
+opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
+opik_optimizer/logging_config.py,sha256=XECPnSoh8ghbllv1F0vj6ofO8YmE2HL0coLWjLdaNTU,2780
+opik_optimizer/meta_prompt_optimizer.py,sha256=-5fAPz0LsQiQS-xj67hxr3KizvxoOScAA7gS6ACM9PY,49457
+opik_optimizer/optimization_result.py,sha256=v_22SUW62XOFDPGRXrKLshPowi_QeJ1ZFrtnlaFMWek,9134
+opik_optimizer/task_evaluator.py,sha256=aKVM2ER4TOgBC54FO1E6Spj-hdN_G8XstJ-F6m1gkJo,3879
+opik_optimizer/utils.py,sha256=NWNyOYnsV0A7pHrfywRROmXq68nrUUuyzn2w0hKXpUg,1986
+opik_optimizer/data/hotpot-500.json,sha256=YXxCtuvYvxSu5u0y4559a6b1qwgAYsWzT_SUKv_21ew,76862
+opik_optimizer/datasets/__init__.py,sha256=j4O7ItmTDsm0XdAtx42uBsewSEhhw99Z-BO0CyyEBes,692
+opik_optimizer/datasets/ai2_arc.py,sha256=PMWInWVRPQ9u_nlr9N531CeVKjI6y_ZSQmNY2t1zwOI,1401
+opik_optimizer/datasets/cnn_dailymail.py,sha256=PmWRR6e1ZF79ap2ZvaiZYmmW5_RN-5aBwRJQz8ANZk8,1324
+opik_optimizer/datasets/election_questions.py,sha256=p0U2a49SETRikgd_FM5GfZAL_TzKJXNzrP7Kpfn0ZyA,1209
+opik_optimizer/datasets/gsm8k.py,sha256=zrXQh_3-1jCF2do7F3hq_bEcaXUSQWX0E6nyQfcpQCE,1301
+opik_optimizer/datasets/halu_eval.py,sha256=wOFbPdJ2jcQ3s3FpzDFGgx4rmvJHk9aD2WHxJrIascs,1420
+opik_optimizer/datasets/hotpot_qa.py,sha256=fgznrfV6DO1B8BekvL3Hc2hwzBCvph-HiZuEuwTiTqU,2142
+opik_optimizer/datasets/medhallu.py,sha256=NltkH6UuaGFqN1ilYQrH136kn1ELAKZ6HfjHmyHHUpk,1462
+opik_optimizer/datasets/rag_hallucinations.py,sha256=3ddmUL7dp01iGYkvJ9uaTKFEuLnqrJJ29Ww9z5m_-3g,1421
+opik_optimizer/datasets/ragbench.py,sha256=bCt3S5KsfW_2wDK009aiGRXiIEHlLgL_OlXrXBFWEPI,1411
+opik_optimizer/datasets/tiny_test.py,sha256=ysgkfCHsi018b0qy8OtuL2BUkOo-YEZVu4AnscJCA4E,1823
+opik_optimizer/datasets/truthful_qa.py,sha256=xbRjW0UOm7oDN3jAnTZD7HChgDGspwhAhFpHV7zTtag,4166
+opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
+opik_optimizer/demo/cache.py,sha256=5WqK8rSiijzU6s4VHIjLuL1LR5i1yHtY-x5FZTduSus,3669
+opik_optimizer/demo/datasets.py,sha256=MezQlG4Q_cgSH7zQOmJcDwkGU8JV0xKSnZwCJGaj-88,2494
+opik_optimizer/evolutionary_optimizer/__init__.py,sha256=OQ2ART5g-7EVGOISvTGY-AbmEyyDFEJJCsmJBzGJIpw,57
+opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py,sha256=rrSZ8rUeUkVQ8qZwz16gY3TUoOwi0o-rVPZLumNeSWs,76650
+opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
+opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=F4NQdm4YN7BMioxQdhzBZK8qhFEDG3qYhF--M29jfzQ,19334
+opik_optimizer/few_shot_bayesian_optimizer/prompt_parameter.py,sha256=EDsSIFAUOfiZKWLrOAaBDB7Exk7cmIs4ccI95kVa7JY,3118
+opik_optimizer/few_shot_bayesian_optimizer/prompt_templates.py,sha256=HmvD-UeT3aKiiet5cUtULXe6iFPEOo6hxyDE0pH2LnQ,2424
+opik_optimizer/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opik_optimizer/mipro_optimizer/__init__.py,sha256=CF9TVXjOxTobDO1kAS8CD4eyLVzEozxjfgoKwIO6ZpU,44
+opik_optimizer/mipro_optimizer/_lm.py,sha256=bcTy2Y5HjSaFQOATIpUaA86eIp3vKHaMuDI2_RvN2ww,16376
+opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=CiQWe39LCp-81SZmLyMybIw2lc_0RBKcxclLEuSXQgI,39757
+opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=XFXlhDCMBS5wXIVds83F26ztKSkqF3IeEDXYJvL-dZc,23957
+opik_optimizer/mipro_optimizer/utils.py,sha256=wG1koygXfm_pvtA1jR-YaU4NATPbJZoTI7LE7l7df7g,3273
+opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opik_optimizer/optimization_config/configs.py,sha256=MYL9H2UAqeyGBlBGWbOZ-6Snto4ZMuXnypgvVuUSW1Y,1132
+opik_optimizer/optimization_config/mappers.py,sha256=RXgTMxPzTQ1AHGke6Zca6rTcfCI7IkCKhQYciaEGSAo,1698
+opik_optimizer-0.8.1.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
+opik_optimizer-0.8.1.dist-info/METADATA,sha256=HPdTa0hebpLlqgFVY4Ue-VFaqDucUIPb1KrZS8w_rX4,7085
+opik_optimizer-0.8.1.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
+opik_optimizer-0.8.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-0.8.1.dist-info/RECORD,,