opik-optimizer 0.8.0__py3-none-any.whl → 0.9.0rc0__py3-none-any.whl

opik_optimizer/utils.py

@@ -1,20 +1,100 @@
 """Utility functions and constants for the optimizer package."""
 
+from typing import Dict, Any, Optional, TYPE_CHECKING, Type, Literal, Final
+from types import TracebackType
+
 import opik
+from opik.api_objects.opik_client import Opik
+from opik.api_objects.optimization import Optimization
+
+import json
 import logging
 import random
 import string
-from opik.api_objects.opik_client import Opik
-
-from typing import List, Dict, Any, Optional, Callable, TYPE_CHECKING
+import base64
+import urllib.parse
+from rich import console
 
 # Type hint for OptimizationResult without circular import
 if TYPE_CHECKING:
     from .optimization_result import OptimizationResult
 
+ALLOWED_URL_CHARACTERS: Final[str] = ":/&?="
 logger = logging.getLogger(__name__)
 
 
+class OptimizationContextManager:
+    """
+    Context manager for handling optimization lifecycle.
+    Automatically updates optimization status to "completed" or "cancelled" based on context exit.
+    """
+
+    def __init__(
+        self,
+        client: Opik,
+        dataset_name: str,
+        objective_name: str,
+        name: Optional[str] = None,
+        metadata: Optional[Dict[str, Any]] = None,
+    ):
+        """
+        Initialize the optimization context.
+
+        Args:
+            client: The Opik client instance
+            dataset_name: Name of the dataset for optimization
+            objective_name: Name of the optimization objective
+            name: Optional name for the optimization
+            metadata: Optional metadata for the optimization
+        """
+        self.client = client
+        self.dataset_name = dataset_name
+        self.objective_name = objective_name
+        self.name = name
+        self.metadata = metadata
+        self.optimization: Optional[Optimization] = None
+
+    def __enter__(self) -> Optional[Optimization]:
+        """Create and return the optimization."""
+        try:
+            self.optimization = self.client.create_optimization(
+                dataset_name=self.dataset_name,
+                objective_name=self.objective_name,
+                name=self.name,
+                metadata=self.metadata,
+            )
+            if self.optimization:
+                return self.optimization
+            else:
+                return None
+        except Exception:
+            logger.warning(
+                "Opik server does not support optimizations. Please upgrade opik."
+            )
+            logger.warning("Continuing without Opik optimization tracking.")
+            return None
+
+    def __exit__(
+        self,
+        exc_type: Optional[Type[BaseException]],
+        exc_val: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> Literal[False]:
+        """Update optimization status based on context exit."""
+        if self.optimization is None:
+            return False
+
+        try:
+            if exc_type is None:
+                self.optimization.update(status="completed")
+            else:
+                self.optimization.update(status="cancelled")
+        except Exception as e:
+            logger.error(f"Failed to update optimization status: {e}")
+
+        return False
+
+
 def format_prompt(prompt: str, **kwargs: Any) -> str:
     """
     Format a prompt string with the given keyword arguments.
@@ -76,5 +156,109 @@ def get_random_seed() -> int:
 
     return random.randint(0, 2**32 - 1)
 
+
 def random_chars(n: int) -> str:
     return "".join(random.choice(string.ascii_letters) for _ in range(n))
+
+
+def disable_experiment_reporting():
+    import opik.evaluation.report
+    
+    opik.evaluation.report._patch_display_experiment_results = opik.evaluation.report.display_experiment_results
+    opik.evaluation.report._patch_display_experiment_link = opik.evaluation.report.display_experiment_link
+    opik.evaluation.report.display_experiment_results = lambda *args, **kwargs: None
+    opik.evaluation.report.display_experiment_link = lambda *args, **kwargs: None
+
+
+def enable_experiment_reporting():
+    import opik.evaluation.report
+
+    try:
+        opik.evaluation.report.display_experiment_results = opik.evaluation.report._patch_display_experiment_results
+        opik.evaluation.report.display_experiment_link = opik.evaluation.report._patch_display_experiment_link
+    except AttributeError:
+        pass
+
+    
+def json_to_dict(json_str: str) -> Any:
+    cleaned_json_string = json_str.strip()
+
+    try:
+        return json.loads(cleaned_json_string)
+    except json.JSONDecodeError:
+        if cleaned_json_string.startswith("```json"):
+            cleaned_json_string = cleaned_json_string[7:] 
+            if cleaned_json_string.endswith("```"):
+                cleaned_json_string = cleaned_json_string[:-3]
+        elif cleaned_json_string.startswith("```"):
+            cleaned_json_string = cleaned_json_string[3:]
+            if cleaned_json_string.endswith("```"):
+                cleaned_json_string = cleaned_json_string[:-3]
+
+        try:
+            return json.loads(cleaned_json_string)
+        except json.JSONDecodeError as e:
+            print(f"Failed to parse JSON string: {json_str}")
+            logger.debug(f"Failed to parse JSON string: {json_str}")
+            raise e
+
+
+def optimization_context(
+    client: Opik,
+    dataset_name: str,
+    objective_name: str,
+    name: Optional[str] = None,
+    metadata: Optional[Dict[str, Any]] = None,
+) -> OptimizationContextManager:
+    """
+    Create a context manager for handling optimization lifecycle.
+    Automatically updates optimization status to "completed" or "cancelled" based on context exit.
+
+    Args:
+        client: The Opik client instance
+        dataset_name: Name of the dataset for optimization
+        objective_name: Name of the optimization objective
+        name: Optional name for the optimization
+        metadata: Optional metadata for the optimization
+
+    Returns:
+        OptimizationContextManager: A context manager that handles optimization lifecycle
+    """
+    return OptimizationContextManager(
+        client=client,
+        dataset_name=dataset_name,
+        objective_name=objective_name,
+        name=name,
+        metadata=metadata,
+    )
+
+
+def ensure_ending_slash(url: str) -> str:
+    return url.rstrip("/") + "/"
+
+
+def get_optimization_run_url_by_id(
+    dataset_id: str, optimization_id: str, url_override: str
+) -> str:
+    encoded_opik_url = base64.b64encode(url_override.encode("utf-8")).decode("utf-8")
+
+    run_path = urllib.parse.quote(
+        f"v1/session/redirect/optimizations/?optimization_id={optimization_id}&dataset_id={dataset_id}&path={encoded_opik_url}",
+        safe=ALLOWED_URL_CHARACTERS,
+    )
+    return urllib.parse.urljoin(ensure_ending_slash(url_override), run_path)
+
+
+def display_optimization_run_link(
+    optimization_id: str, dataset_id: str, url_override: str
+) -> None:
+    console_container = console.Console()
+
+    optimization_url = get_optimization_run_url_by_id(
+        optimization_id=optimization_id,
+        dataset_id=dataset_id,
+        url_override=url_override,
+    )
+    console_container.print(
+        f"View the optimization run [link={optimization_url}]in your Opik dashboard[/link]."
+    )

{opik_optimizer-0.8.0.dist-info → opik_optimizer-0.9.0rc0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.8.0
+Version: 0.9.0rc0
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML
@@ -13,7 +13,7 @@ Requires-Python: >=3.9,<3.13
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: opik>=1.7.17
-Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: dspy<=2.6.24,>=2.6.18
 Requires-Dist: litellm
 Requires-Dist: tqdm
 Requires-Dist: datasets
@@ -113,24 +113,20 @@ Available sample datasets for testing:
 
 ```python
 from opik.evaluation.metrics import LevenshteinRatio
-from opik_optimizer import FewShotBayesianOptimizer
-from opik_optimizer.demo import get_or_create_dataset
-
-from opik_optimizer import (
-    MetricConfig,
-    TaskConfig,
-    from_dataset_field,
-    from_llm_response_text,
-)
+from opik_optimizer import FewShotBayesianOptimizer, ChatPrompt
+from opik_optimizer.datasets import hotpot_300
 
 # Load a sample dataset
-hot_pot_dataset = get_or_create_dataset("hotpot-300")
+hot_pot_dataset = hotpot_300()
 
 # Define the instruction for your chat prompt.
 # Input parameters from dataset examples will be interpolated into the full prompt.
-prompt_instruction = """
-Answer the question based on the provided context.
-"""
+prompt = ChatPrompt(
+    messages=[
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": "{question}"}
+    ]
+)
 project_name = "optimize-few-shot-bayesian-hotpot" # For Comet logging
 
 optimizer = FewShotBayesianOptimizer(
@@ -142,26 +138,14 @@ optimizer = FewShotBayesianOptimizer(
     seed=42,
 )
 
-metric_config = MetricConfig(
-    metric=LevenshteinRatio(project_name=project_name), # Metric for evaluation
-    inputs={
-        "output": from_llm_response_text(), # Get output from LLM
-        "reference": from_dataset_field(name="answer"), # Get reference from dataset
-    },
-)
-
-task_config = TaskConfig(
-    instruction_prompt=prompt_instruction,
-    input_dataset_fields=["question"], # Fields from dataset to use as input
-    output_dataset_field="answer",     # Field in dataset for reference answer
-    use_chat_prompt=True,              # Use chat-style prompting
-)
+def levenshtein_ratio(dataset_item, llm_output):
+    return LevenshteinRatio().score(reference=dataset_item["answer"], output=llm_output)
 
 # Run the optimization
 result = optimizer.optimize_prompt(
+    prompt=prompt,
     dataset=hot_pot_dataset,
-    metric_config=metric_config,
-    task_config=task_config,
+    metric=levenshtein_ratio,
     n_trials=10,   # Number of optimization trials
     n_samples=150, # Number of dataset samples for evaluation per trial
 )

opik_optimizer-0.9.0rc0.dist-info/RECORD

@@ -0,0 +1,48 @@
+opik_optimizer/__init__.py,sha256=wAeC8Hy0ENiQnLkoqSBeJfkqaw_vrZRfMka8a6qnX6Y,1089
+opik_optimizer/_throttle.py,sha256=ztub8qlwz4u0GVA2TIoLig0D1Cs0hJ7_o_SnT_C7Nmk,1360
+opik_optimizer/base_optimizer.py,sha256=mg5D5m2hIzq3XbVjRhx8c_HuXWZWaRE2J6QtkHnxkqE,4439
+opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
+opik_optimizer/logging_config.py,sha256=XECPnSoh8ghbllv1F0vj6ofO8YmE2HL0coLWjLdaNTU,2780
+opik_optimizer/optimization_result.py,sha256=PeDIoNIHaJsi9WsDoKPgO6sW3CkPKZKK7RvY8tmjUN0,7508
+opik_optimizer/reporting_utils.py,sha256=pRcRhE9w1q6PVdTmqrTyqIlUmeMAknwJJFT99FG0tuk,5523
+opik_optimizer/task_evaluator.py,sha256=xqkrnh72_U23QSW_8X48mh83LaFFFooAIgpgFWol8_c,4040
+opik_optimizer/utils.py,sha256=y7I58vESRphuMLA4cfH3sNiSuntyiqG8hwK5UuwVdt4,8239
+opik_optimizer/data/hotpot-500.json,sha256=YXxCtuvYvxSu5u0y4559a6b1qwgAYsWzT_SUKv_21ew,76862
+opik_optimizer/datasets/__init__.py,sha256=V4LVDOaRjwzaYvhdQ3V6CAwFaeKnxyTV1lp_ES9Z31E,691
+opik_optimizer/datasets/ai2_arc.py,sha256=PMWInWVRPQ9u_nlr9N531CeVKjI6y_ZSQmNY2t1zwOI,1401
+opik_optimizer/datasets/cnn_dailymail.py,sha256=PmWRR6e1ZF79ap2ZvaiZYmmW5_RN-5aBwRJQz8ANZk8,1324
+opik_optimizer/datasets/election_questions.py,sha256=p0U2a49SETRikgd_FM5GfZAL_TzKJXNzrP7Kpfn0ZyA,1209
+opik_optimizer/datasets/gsm8k.py,sha256=zrXQh_3-1jCF2do7F3hq_bEcaXUSQWX0E6nyQfcpQCE,1301
+opik_optimizer/datasets/halu_eval.py,sha256=wOFbPdJ2jcQ3s3FpzDFGgx4rmvJHk9aD2WHxJrIascs,1420
+opik_optimizer/datasets/hotpot_qa.py,sha256=fgznrfV6DO1B8BekvL3Hc2hwzBCvph-HiZuEuwTiTqU,2142
+opik_optimizer/datasets/medhallu.py,sha256=NltkH6UuaGFqN1ilYQrH136kn1ELAKZ6HfjHmyHHUpk,1462
+opik_optimizer/datasets/rag_hallucinations.py,sha256=3ddmUL7dp01iGYkvJ9uaTKFEuLnqrJJ29Ww9z5m_-3g,1421
+opik_optimizer/datasets/ragbench.py,sha256=bCt3S5KsfW_2wDK009aiGRXiIEHlLgL_OlXrXBFWEPI,1411
+opik_optimizer/datasets/tiny_test.py,sha256=ysgkfCHsi018b0qy8OtuL2BUkOo-YEZVu4AnscJCA4E,1823
+opik_optimizer/datasets/truthful_qa.py,sha256=xbRjW0UOm7oDN3jAnTZD7HChgDGspwhAhFpHV7zTtag,4166
+opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
+opik_optimizer/demo/cache.py,sha256=5WqK8rSiijzU6s4VHIjLuL1LR5i1yHtY-x5FZTduSus,3669
+opik_optimizer/demo/datasets.py,sha256=MezQlG4Q_cgSH7zQOmJcDwkGU8JV0xKSnZwCJGaj-88,2494
+opik_optimizer/evolutionary_optimizer/__init__.py,sha256=OQ2ART5g-7EVGOISvTGY-AbmEyyDFEJJCsmJBzGJIpw,57
+opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py,sha256=cnpga8ytSjx5lNq2URLBCyV9s5r1s9_eKD4BU7rWW_g,76259
+opik_optimizer/evolutionary_optimizer/reporting.py,sha256=Gl52sH7XaU4GXUhFt_FcfjYFN3MghnDt1ISULATjbP4,9944
+opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
+opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=Pb0GHUEf5DdnLJaYI5PVnar2F-hhWslEEYwWftnZ0Gs,24332
+opik_optimizer/few_shot_bayesian_optimizer/reporting.py,sha256=j1mNEQyFT7YUVlMU1TxPZxrf5sPwiHZ2nx1fOL4ZIog,4756
+opik_optimizer/meta_prompt_optimizer/__init__.py,sha256=syiN2_fMm5iZDQezZCHYe-ZiGOIPlBkLt49Sa1kuR70,97
+opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py,sha256=LrN8kGoB-qm5Tvncpmcy2qd79vxkcMokei2sMXrv0jw,34404
+opik_optimizer/meta_prompt_optimizer/reporting.py,sha256=4Lju2uxSBkCVYyJ6ZSS-GjDFVnmP14R6XVtr-tEFlL0,5765
+opik_optimizer/mipro_optimizer/__init__.py,sha256=23dqXp1lG00ZiMZvU75FzzLmzaHe_-5krchwdvMhWzE,53
+opik_optimizer/mipro_optimizer/_lm.py,sha256=bcTy2Y5HjSaFQOATIpUaA86eIp3vKHaMuDI2_RvN2ww,16376
+opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=wQP3D5g3X2e0h05vJy_CvW0qDMdGqLYmuUVzdndcScE,39258
+opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=pfD8toZVCpqSDdGwyOUvAeyORyGyYqrua71JFzVw2GA,23305
+opik_optimizer/mipro_optimizer/utils.py,sha256=-d9xOKxmYbKwpNM2aheKQVf3gxCh5B1ENuAvzc38xe8,2509
+opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+opik_optimizer/optimization_config/chat_prompt.py,sha256=1_15vmtIawzp60Eu_lROu7Py2NLjeiRzk0Jhutu7pbU,3485
+opik_optimizer/optimization_config/configs.py,sha256=HzpEP84bnqtDs76dtmPGecDQ-Ux2wIk0JVv7A2gsE3k,496
+opik_optimizer/optimization_config/mappers.py,sha256=RMUWwYvXNCJe6w1jYiT6EX218UYZS1PUMMe12OjNEug,1692
+opik_optimizer-0.9.0rc0.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
+opik_optimizer-0.9.0rc0.dist-info/METADATA,sha256=ir4vFau8I-2ZMDTojCxnIdy-VUBgLTUjPjSGaqX0HlY,6591
+opik_optimizer-0.9.0rc0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+opik_optimizer-0.9.0rc0.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-0.9.0rc0.dist-info/RECORD,,

{opik_optimizer-0.8.0.dist-info → opik_optimizer-0.9.0rc0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.8.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

opik_optimizer/few_shot_bayesian_optimizer/prompt_parameter.py

@@ -1,91 +0,0 @@
-import pydantic
-from typing import List, Dict, Literal
-import json
-
-from . import prompt_templates
-
-ChatItem = Dict[Literal["role", "content"], str]
-
-
-class ChatPromptParameter(pydantic.BaseModel):
-    name: str
-    instruction: str
-    task_input_parameters: List[str]
-    task_output_parameter: str
-    demo_examples: List[Dict[str, str]] = pydantic.Field(default_factory=list)
-
-    _few_shot_system_prompt_intro: str = "You are an intelligent assistant that learns from few-shot examples provided earlier in the conversation. Whenever you respond, carefully follow the structure, tone, and format of previous assistant replies, using them as a guide"
-    
-    def as_template(self) -> prompt_templates.ChatPromptTemplate:
-        if not self.demo_examples:
-            return prompt_templates.ChatPromptTemplate(
-                chat_template=[
-                    {
-                        "role": "system",
-                        "content": self.instruction
-                    },
-                    {
-                        "role": "user",
-                        "content": json.dumps({param: f"{{{{{param}}}}}" for param in self.task_input_parameters})
-                    }
-                ]
-            )
-
-        return prompt_templates.ChatPromptTemplate(
-            chat_template=[
-                {
-                    "role": "system",
-                    "content": self.instruction + f"\n\n{self._few_shot_system_prompt_intro}"
-                },
-                *self._render_demos(),
-                {
-                    "role": "user",
-                    "content": json.dumps({param: f"{{{{{param}}}}}" for param in self.task_input_parameters})
-                }
-            ]
-        )
-    
-    def _render_demos(self) -> List[ChatItem]:
-        """
-        Renders demo examples in the following format:
-
-        [
-            {
-                "role": "user",
-                "content": "\n{\n\"input_field1\": \"value1\",\n\"input_field2\": \"value2\"\n}\n"
-            },
-            {
-                "role": "assistant",
-                "content": "expected_response_1"
-            },
-            {
-                "role": "user",
-                "content": "\n{\n\"input_field1\": \"value3\",\n\"input_field2\": \"value4\"\n}\n"
-            },
-            {
-                "role": "assistant",
-                "content": "expected_response_2"
-            }
-        ]
-        """
-        chat_items: List[ChatItem] = []
-        
-        for example in self.demo_examples:
-            inputs = {param: example[param] for param in self.task_input_parameters}
-            
-            formatted_input = json.dumps(inputs, indent=2)
-            
-            user_message: ChatItem = {
-                "role": "user",
-                "content": f"\n{formatted_input}\n"
-            }
-            
-            assistant_message: ChatItem = {
-                "role": "assistant",
-                "content": example[self.task_output_parameter]
-            }
-
-            chat_items.append(user_message)
-            chat_items.append(assistant_message)
-            
-        return chat_items

opik_optimizer/few_shot_bayesian_optimizer/prompt_templates.py

@@ -1,80 +0,0 @@
-import abc
-from typing import Dict, List, Literal, Any
-from typing_extensions import override
-import opik
-from opik.api_objects.prompt import prompt_template as opik_prompt_template
-
-ChatItem = Dict[Literal["role", "content"], str]
-
-
-class BaseTemplate(abc.ABC):
-    @abc.abstractmethod
-    def format(self, **kwargs: Any) -> Any:
-        raise NotImplementedError
-
-class PromptTemplate(BaseTemplate):
-    """Wrapper for opik PromptTemplate which is a subclass of BaseTemplate."""
-    def __init__(
-        self, 
-        template: str,
-        validate_placeholders: bool = False,
-        type: opik.PromptType = opik.PromptType.MUSTACHE
-    ) -> None:
-        self._opik_prompt_template = opik_prompt_template.PromptTemplate(
-            template=template,
-            validate_placeholders=validate_placeholders,
-            type=type
-        )
-
-    @override
-    def format(self, **kwargs: Any) -> str:
-        return self._opik_prompt_template.format(**kwargs)
-
-
-class ChatItemTemplate(BaseTemplate):
-    def __init__(
-        self, 
-        role: str,
-        prompt_template: PromptTemplate
-    ) -> None:
-        self._role = role
-        self._prompt_template = prompt_template
-
-    @override
-    def format(self, **kwargs: Any) -> ChatItem:
-        return {
-            "role": self._role,
-            "content": self._prompt_template.format(**kwargs)
-        }
-
-class ChatPromptTemplate(BaseTemplate):
-    def __init__(
-        self, 
-        chat_template: List[Dict[str, str]],
-        type: opik.PromptType = opik.PromptType.MUSTACHE,
-        validate_placeholders: bool = False,
-    ) -> None:
-        self._raw_chat_template = chat_template
-        self._type = type
-        self._validate_placeholders = validate_placeholders
-        self._init_chat_template_items()
-
-    def _init_chat_template_items(self) -> None:
-        self._chat_template_items: List[ChatItemTemplate] = [
-            ChatItemTemplate(
-                role=item["role"],
-                prompt_template=PromptTemplate(
-                    item["content"],
-                    type=self._type,
-                    validate_placeholders=self._validate_placeholders,
-                )
-            )
-            for item in self._raw_chat_template
-        ]
-
-    @override
-    def format(self, **kwargs: Any) -> List[ChatItem]:
-        return [
-            item.format(**kwargs)
-            for item in self._chat_template_items
-        ]