opik-optimizer 0.9.0rc0__py3-none-any.whl → 0.9.1__py3-none-any.whl

opik_optimizer/__init__.py

@@ -12,8 +12,8 @@ from .logging_config import setup_logging
 from .meta_prompt_optimizer import MetaPromptOptimizer
 from .mipro_optimizer import MiproOptimizer
 from .optimization_config.chat_prompt import ChatPrompt
-from .optimization_result import OptimizationResult
 from .optimization_config.configs import TaskConfig
+from .optimization_result import OptimizationResult
 
 __version__ = importlib.metadata.version("opik_optimizer")
 

opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py

@@ -39,8 +39,9 @@ Your task:
     - Add a section title in XML or markdown format. The examples will be provided as `example_1\nexample_2\n...` with each example following the example template.
 - Analyze the examples to infer a consistent structure, and create a single string few_shot_example_template using the Python .format() style. Make sure to follow the following instructions:
     - Unless absolutely relevant, do not return an object but instead a string that can be inserted as part of {FEW_SHOT_EXAMPLE_PLACEHOLDER}
-    - Make sure to include the variables as part of this string so we can before string formatting with actual examples. Only variables available in the examples can be used. Do not use anything else, do not apply any transformations to the variables either.
-    - The few shot examples should include the expected response as the goal is to provide examples of the expected output format.
+    - Make sure to include the variables as part of this string so we can before string formatting with actual examples. Only variables available in the examples can be used.
+    - Do not apply any transformations to the variables either, only the variable name should be included in the format `{{<variable_name>}}`
+    - The few shot examples should include the expected response as the goal is to provide examples of the response.
     - Ensure the format of the few shot examples are consistent with how the model will be called
 
 Return your output as a JSON object with:
@@ -252,7 +253,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                     processed_demo_examples.append(
                         fewshot_prompt_template.example_template.format(**processed_example)
                     )
-                except Exception as e:
+                except Exception:
                     logger.error(f"Failed to format fewshot prompt template {fewshot_prompt_template} with example: {processed_example} ")
                     raise
             few_shot_examples = "\n\n".join(processed_demo_examples)
@@ -301,7 +302,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             trial_config = {
                 "demo_examples": demo_examples,
                 "message_list_with_placeholder": fewshot_prompt_template.message_list_with_placeholder,
-                "message_list": messages
+                "message_list": messages_for_reporting
             }
             trial.set_user_attr("score", score)
             trial.set_user_attr("config", trial_config)
@@ -562,10 +563,11 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         self, messages: List[Dict[str, str]], few_shot_examples: Optional[str] = None
     ):
         def llm_task(dataset_item: Dict[str, Any]) -> Dict[str, Any]:
-            prompt_ = [{
-                "role": item["role"],
-                "content": item["content"].format(**dataset_item)
-            } for item in messages]
+            for key, value in dataset_item.items():
+                prompt_ = [{
+                    "role": item["role"],
+                    "content": item["content"].replace("{" + key + "}", str(value))
+                } for item in messages]
 
             if few_shot_examples:
                 prompt_ = [{
@@ -584,4 +586,4 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
                 mappers.EVALUATED_LLM_TASK_OUTPUT: response.choices[0].message.content
             }
 
-        return llm_task, messages
+        return llm_task

opik_optimizer/optimization_config/chat_prompt.py

@@ -94,8 +94,8 @@ class ChatPrompt:
         }
 
     @classmethod
-    def model_validate(cls, obj: Any, *, strict: bool | None = None, from_attributes: bool | None = None, 
-                      context: Any | None = None, by_alias: bool | None = None, by_name: bool | None = None) -> 'ChatPrompt':
+    def model_validate(cls, obj: Any, *, strict: Optional[bool] = None, from_attributes: Optional[bool] = None, 
+                      context: Optional[Any] = None, by_alias: Optional[bool] = None, by_name: Optional[bool] = None) -> 'ChatPrompt':
         """Custom validation method to handle nested objects during deserialization."""
         return ChatPrompt(
             system=obj.get('system', None),

opik_optimizer/task_evaluator.py

@@ -11,16 +11,23 @@ def _create_metric_class(metric: Callable):
     class MetricClass(base_metric.BaseMetric):
         def __init__(self):
             self.name = metric.__name__
-        
+
         def score(self, llm_output, **kwargs) -> score_result.ScoreResult:
             try:
                 metric_val = metric(dataset_item=kwargs, llm_output=llm_output)
                 if isinstance(metric_val , score_result.ScoreResult):
-                    return metric_val
+                    return score_result.ScoreResult(
+                        name = self.name,
+                        value = metric_val.value,
+                        scoring_failed=metric_val.scoring_failed,
+                        metadata=metric_val.metadata,
+                        reason=metric_val.reason
+                    )
                 else:
                     return score_result.ScoreResult(
                         name = self.name,
-                        value = metric_val
+                        value = metric_val,
+                        scoring_failed=False
                     )
             except Exception:
                 return score_result.ScoreResult(
@@ -71,7 +78,7 @@ def evaluate(
         items = [item for item in items if item.get("id") in dataset_item_ids]
 
     eval_metrics = [_create_metric_class(metric)]
-
+    
     if optimization_id is not None:
         result = opik_evaluator.evaluate_optimization_trial(
             optimization_id=optimization_id,

{opik_optimizer-0.9.0rc0.dist-info → opik_optimizer-0.9.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: opik_optimizer
-Version: 0.9.0rc0
+Version: 0.9.1
 Summary: Agent optimization with Opik
 Home-page: https://github.com/comet-ml/opik
 Author: Comet ML

{opik_optimizer-0.9.0rc0.dist-info → opik_optimizer-0.9.1.dist-info}/RECORD

@@ -1,11 +1,11 @@
-opik_optimizer/__init__.py,sha256=wAeC8Hy0ENiQnLkoqSBeJfkqaw_vrZRfMka8a6qnX6Y,1089
+opik_optimizer/__init__.py,sha256=CmFF0SEE_zdvlOLFeZceXqDBHzcsCGDb59b9I9Yeg6g,1089
 opik_optimizer/_throttle.py,sha256=ztub8qlwz4u0GVA2TIoLig0D1Cs0hJ7_o_SnT_C7Nmk,1360
 opik_optimizer/base_optimizer.py,sha256=mg5D5m2hIzq3XbVjRhx8c_HuXWZWaRE2J6QtkHnxkqE,4439
 opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
 opik_optimizer/logging_config.py,sha256=XECPnSoh8ghbllv1F0vj6ofO8YmE2HL0coLWjLdaNTU,2780
 opik_optimizer/optimization_result.py,sha256=PeDIoNIHaJsi9WsDoKPgO6sW3CkPKZKK7RvY8tmjUN0,7508
 opik_optimizer/reporting_utils.py,sha256=pRcRhE9w1q6PVdTmqrTyqIlUmeMAknwJJFT99FG0tuk,5523
-opik_optimizer/task_evaluator.py,sha256=xqkrnh72_U23QSW_8X48mh83LaFFFooAIgpgFWol8_c,4040
+opik_optimizer/task_evaluator.py,sha256=212shzapI7GtrqrFvn_6wkvWR5_Zhm30CR5t-tW0bV8,4380
 opik_optimizer/utils.py,sha256=y7I58vESRphuMLA4cfH3sNiSuntyiqG8hwK5UuwVdt4,8239
 opik_optimizer/data/hotpot-500.json,sha256=YXxCtuvYvxSu5u0y4559a6b1qwgAYsWzT_SUKv_21ew,76862
 opik_optimizer/datasets/__init__.py,sha256=V4LVDOaRjwzaYvhdQ3V6CAwFaeKnxyTV1lp_ES9Z31E,691
@@ -27,7 +27,7 @@ opik_optimizer/evolutionary_optimizer/__init__.py,sha256=OQ2ART5g-7EVGOISvTGY-Ab
 opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py,sha256=cnpga8ytSjx5lNq2URLBCyV9s5r1s9_eKD4BU7rWW_g,76259
 opik_optimizer/evolutionary_optimizer/reporting.py,sha256=Gl52sH7XaU4GXUhFt_FcfjYFN3MghnDt1ISULATjbP4,9944
 opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
-opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=Pb0GHUEf5DdnLJaYI5PVnar2F-hhWslEEYwWftnZ0Gs,24332
+opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=trQ7lAdPyNpMrCwdQeq67FCosQuoN3uvLM6lXoJPkQQ,24457
 opik_optimizer/few_shot_bayesian_optimizer/reporting.py,sha256=j1mNEQyFT7YUVlMU1TxPZxrf5sPwiHZ2nx1fOL4ZIog,4756
 opik_optimizer/meta_prompt_optimizer/__init__.py,sha256=syiN2_fMm5iZDQezZCHYe-ZiGOIPlBkLt49Sa1kuR70,97
 opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py,sha256=LrN8kGoB-qm5Tvncpmcy2qd79vxkcMokei2sMXrv0jw,34404
@@ -38,11 +38,11 @@ opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=wQP3D5g3X2e0h05vJy_
 opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=pfD8toZVCpqSDdGwyOUvAeyORyGyYqrua71JFzVw2GA,23305
 opik_optimizer/mipro_optimizer/utils.py,sha256=-d9xOKxmYbKwpNM2aheKQVf3gxCh5B1ENuAvzc38xe8,2509
 opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-opik_optimizer/optimization_config/chat_prompt.py,sha256=1_15vmtIawzp60Eu_lROu7Py2NLjeiRzk0Jhutu7pbU,3485
+opik_optimizer/optimization_config/chat_prompt.py,sha256=nGSElB4CyOegYi06oFmaVeFKI6XRv6LJOWJ1axhjyyY,3500
 opik_optimizer/optimization_config/configs.py,sha256=HzpEP84bnqtDs76dtmPGecDQ-Ux2wIk0JVv7A2gsE3k,496
 opik_optimizer/optimization_config/mappers.py,sha256=RMUWwYvXNCJe6w1jYiT6EX218UYZS1PUMMe12OjNEug,1692
-opik_optimizer-0.9.0rc0.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
-opik_optimizer-0.9.0rc0.dist-info/METADATA,sha256=ir4vFau8I-2ZMDTojCxnIdy-VUBgLTUjPjSGaqX0HlY,6591
-opik_optimizer-0.9.0rc0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-opik_optimizer-0.9.0rc0.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
-opik_optimizer-0.9.0rc0.dist-info/RECORD,,
+opik_optimizer-0.9.1.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
+opik_optimizer-0.9.1.dist-info/METADATA,sha256=XodSnPMwsIwJ2WF618unt3iXHoCbYfokqEM7xnN9vW4,6588
+opik_optimizer-0.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+opik_optimizer-0.9.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-0.9.1.dist-info/RECORD,,