opik-optimizer 0.7.1__py3-none-any.whl → 0.7.2__py3-none-any.whl

opik_optimizer/__init__.py

@@ -7,42 +7,20 @@ __version__ = importlib.metadata.version("opik_optimizer")
 # Using WARNING as a sensible default to avoid flooding users with INFO/DEBUG
 setup_logging(level=logging.WARNING)
 
-
-# Lazy imports to avoid circular dependencies
-def __getattr__(name):
-    if name == "MiproOptimizer":
-        from .mipro_optimizer import MiproOptimizer
-
-        return MiproOptimizer
-    elif name == "BaseOptimizer":
-        from .base_optimizer import BaseOptimizer
-
-        return BaseOptimizer
-    elif name == "MetaPromptOptimizer":
-        from .meta_prompt_optimizer import MetaPromptOptimizer
-
-        return MetaPromptOptimizer
-    elif name == "FewShotBayesianOptimizer":
-        from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
-
-        return FewShotBayesianOptimizer
-    elif name in ["MetricConfig", "OptimizationConfig", "TaskConfig"]:
-        from .optimization_config.configs import (
-            MetricConfig,
-            OptimizationConfig,
-            TaskConfig,
-        )
-
-        return locals()[name]
-    elif name in ["from_dataset_field", "from_llm_response_text"]:
-        from .optimization_config.mappers import (
-            from_dataset_field,
-            from_llm_response_text,
-        )
-
-        return locals()[name]
-    raise AttributeError(f"module 'opik_optimizer' has no attribute '{name}'")
-
+# Regular imports
+from .mipro_optimizer import MiproOptimizer
+from .base_optimizer import BaseOptimizer
+from .meta_prompt_optimizer import MetaPromptOptimizer
+from .few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from .optimization_config.configs import (
+    MetricConfig,
+    OptimizationConfig,
+    TaskConfig,
+)
+from .optimization_config.mappers import (
+    from_dataset_field,
+    from_llm_response_text,
+)
 
 from opik.evaluation.models.litellm import warning_filters
 

opik_optimizer/base_optimizer.py

@@ -4,8 +4,6 @@ import logging
 import time
 
 import litellm
-from opik.evaluation import metrics
-from opik.opik_context import get_current_span_data
 from opik.rest_api.core import ApiError
 
 from pydantic import BaseModel

opik_optimizer/demo/cache.py

@@ -9,6 +9,7 @@ import requests
 NAMED_CACHES = {
     "test": "https://drive.google.com/file/d/1RifNtpN-pl0DW49daRaAMJwW7MCsOh6y/view?usp=sharing",
     "test2": "https://drive.google.com/uc?id=1RifNtpN-pl0DW49daRaAMJwW7MCsOh6y&export=download",
+    "opik-workshop": "https://drive.google.com/file/d/1l0aK6KhDPs2bFsQTkfzvOvfacJlhdmHr/view?usp=sharing",
 }
 CACHE_DIR = os.path.expanduser("~/.litellm_cache")
 

opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py

@@ -85,7 +85,7 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
         split_idx = int(len(dataset) * train_ratio)
         return dataset[:split_idx], dataset[split_idx:]
 
-    def _optimize_prompt(
+    def _optimize_prompt(   
         self,
         dataset: Union[str, Dataset],
         metric_config: MetricConfig,
@@ -171,8 +171,10 @@ class FewShotBayesianOptimizer(base_optimizer.BaseOptimizer):
             n_examples = trial.suggest_int(
                 "n_examples", self.min_examples, self.max_examples
             )
-            available_indices = list(range(len(dataset_items)))
-            example_indices = random.sample(available_indices, n_examples)
+            example_indices = [
+                trial.suggest_categorical(f"example_{i}", list(range(len(dataset_items))))
+                for i in range(n_examples)
+            ]
             trial.set_user_attr("example_indices", example_indices)
 
             instruction = task_config.instruction_prompt

opik_optimizer-0.7.2.dist-info/METADATA

@@ -0,0 +1,172 @@
+Metadata-Version: 2.4
+Name: opik_optimizer
+Version: 0.7.2
+Summary: Agent optimization with Opik
+Home-page: https://github.com/comet-ml/opik
+Author: Comet ML
+Author-email: support@comet.com
+Classifier: Development Status :: 3 - Alpha
+Classifier: Intended Audience :: Developers
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Requires-Python: >=3.9,<3.13
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: opik>=1.7.17
+Requires-Dist: dspy<3,>=2.6.18
+Requires-Dist: litellm
+Requires-Dist: tqdm
+Requires-Dist: datasets
+Requires-Dist: optuna
+Requires-Dist: pydantic
+Requires-Dist: pandas
+Requires-Dist: hf_xet
+Provides-Extra: dev
+Requires-Dist: adalflow; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-conv; extra == "dev"
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: provides-extra
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Opik Optimizer
+
+The Opik Opitmizer can refine your prompts to get better performance
+from your LLMs. You can use a variety of algorithms, including:
+
+* FewShotBayesianOptimizer
+* MiproOptimizer
+* MetaPromptOptimizer
+
+## Quickstart
+
+
+[Open Quickstart Notebook in Colab](https://colab.research.google.com/github/comet-ml/opik/blob/main/sdks/opik_optimizer/notebooks/OpikOptimizerIntro.ipynb)
+
+
+## Setup
+
+1. Configure Opik:
+   ```bash
+   # Install Comet ML CLI
+   pip install opik
+
+   # Configure your API key
+   opik configure
+   # When prompted, enter your Opik API key
+   ```
+
+2. Set up your environment variables:
+   ```bash
+   # OpenAI API key for LLM access
+   export OPENAI_API_KEY=your_openai_api_key
+   ```
+
+3. Install the package:
+   ```bash
+   pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+   ```
+
+You'll need:
+
+1. An LLM model name
+2. An Opik Dataset (or Opik Dataset name)
+3. An Opik Metric (possibly a custom one)
+4. A starting prompt (string)
+
+## Example
+
+We have prepared some sample datasets for testing:
+
+* "tiny-test"
+* "halu-eval-300"
+* "hotpot-300"
+
+You can see how to use those below:
+
+```python
+from opik.evaluation.metrics import LevenshteinRatio
+from opik_optimizer.few_shot_bayesian_optimizer import FewShotBayesianOptimizer
+from opik_optimizer.demo import get_or_create_dataset
+
+from opik_optimizer import (
+    OptimizationConfig,
+    MetricConfig,
+    TaskConfig,
+    from_dataset_field,
+    from_llm_response_text,
+)
+
+hot_pot_dataset = get_or_create_dataset("hotpot-300")
+
+# For chat prompts instruction doesn't need to contain input parameters from dataset examples.
+prompt_instruction = """
+Answer the question.
+"""
+
+initial_prompt_no_examples = [
+    {"role": "system", "content": prompt_instruction},
+    {"role": "user", "content": "{{question}}"},
+]
+
+optimizer = FewShotBayesianOptimizer(
+    model="gpt-4o-mini",
+    project_name="optimize-few-shot-bayesian-hotpot",
+    min_examples=3,
+    max_examples=8,
+    n_threads=16,
+    seed=42,
+)
+
+optimization_config = OptimizationConfig(
+    dataset=hot_pot_dataset,
+    objective=MetricConfig(
+        metric=LevenshteinRatio(),
+        inputs={
+            "output": from_llm_response_text(),
+            "reference": from_dataset_field(name="answer"),
+        },
+    ),
+    task=TaskConfig(
+        instruction_prompt=prompt_instruction,
+        input_dataset_fields=["question"],
+        output_dataset_field="answer",
+        use_chat_prompt=True,
+    ),
+)
+
+result = optimizer.optimize_prompt(optimization_config, n_trials=10)
+print(result)
+```
+
+More examples can be found in the `scripts` folder.
+
+## Installation
+
+```bash
+pip install git+https://github.com/comet-ml/opik#subdirectory=sdks/opik_optimizer
+```
+
+## Development
+
+To use the Opik Optimizer from source:
+
+```bash
+git clone git clone git@github.com:comet-ml/opik
+cd sdks/opik_optimizer
+pip install -e .
+```
+
+## Requirements
+
+- Python 3.10+
+- Opik API key
+- OpenAI API key (or other LLM provider)

{opik_optimizer-0.7.1.dist-info → opik_optimizer-0.7.2.dist-info}/RECORD

@@ -1,6 +1,6 @@
-opik_optimizer/__init__.py,sha256=RQc6N4ca7WS34oXGx6pQQjAh9QVRfjtX6dlVS1oJvLw,1847
+opik_optimizer/__init__.py,sha256=x5QSFom-TtmmUNzqyYIJY2AujMJXbyhXs2oz44-4Af0,1121
 opik_optimizer/_throttle.py,sha256=7vcHoISqXbysymwdb1LPAFJB28tOmih9zzZQWajpH0k,1494
-opik_optimizer/base_optimizer.py,sha256=mHi5b_8Ang6o_kl9m0x8NyXMRYq9OFyaidRfw_wWdEY,8348
+opik_optimizer/base_optimizer.py,sha256=EEFx9ekIV5IjLaf18e6zFzxBO44uxkeHelGmC70GPpo,8260
 opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
 opik_optimizer/logging_config.py,sha256=ELevhxtflYinTo-jVvyQYZbXG7FgAe_b5dPa9y5uLWw,2774
 opik_optimizer/meta_prompt_optimizer.py,sha256=tRxA4YD2AWvKl7fJNg6Oxxay8iyqvdJ8zXy59-qKyUM,46802
@@ -8,10 +8,10 @@ opik_optimizer/optimization_result.py,sha256=9zdDV2MXeLYk7U8OqgMmSU-DdPV6qgYQWS2
 opik_optimizer/task_evaluator.py,sha256=MafDMaLeW0_yGPrumLvYF0HzQUKrnpAlM_0N_TPG8tw,3695
 opik_optimizer/utils.py,sha256=HivUsNzbt7BcuZeEvikdER1DaTPUFLJrpaVQ8raZYD8,3637
 opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
-opik_optimizer/demo/cache.py,sha256=U0ovsBT7s39RSy34Mr-vTacdVk_jERKbRhdlGT6gF_0,3562
+opik_optimizer/demo/cache.py,sha256=5WqK8rSiijzU6s4VHIjLuL1LR5i1yHtY-x5FZTduSus,3669
 opik_optimizer/demo/datasets.py,sha256=hD6JZAQotEDQb4nK7dbnurquILqQsrFRF7nUwon_iXE,22930
 opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
-opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=zh88ElxKcoIkgkwZyXlQ0rwkxkPAoG8BW0DdvRpPhyQ,15106
+opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=NL7cDnD7Mg7_fGK1iDOXJ9PrV-M6lCFmZttioMRKbeQ,15150
 opik_optimizer/few_shot_bayesian_optimizer/prompt_parameter.py,sha256=EDsSIFAUOfiZKWLrOAaBDB7Exk7cmIs4ccI95kVa7JY,3118
 opik_optimizer/few_shot_bayesian_optimizer/prompt_templates.py,sha256=HmvD-UeT3aKiiet5cUtULXe6iFPEOo6hxyDE0pH2LnQ,2424
 opik_optimizer/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -23,8 +23,8 @@ opik_optimizer/mipro_optimizer/utils.py,sha256=4et1JA1QInX3h6Is-_RqzliFwJqkm6tlA
 opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 opik_optimizer/optimization_config/configs.py,sha256=MYL9H2UAqeyGBlBGWbOZ-6Snto4ZMuXnypgvVuUSW1Y,1132
 opik_optimizer/optimization_config/mappers.py,sha256=RXgTMxPzTQ1AHGke6Zca6rTcfCI7IkCKhQYciaEGSAo,1698
-opik_optimizer-0.7.1.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
-opik_optimizer-0.7.1.dist-info/METADATA,sha256=e1-4IpXS8iiaUavpPw91k8vfGmCZj0Yc3blgUMSX3GU,962
-opik_optimizer-0.7.1.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
-opik_optimizer-0.7.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
-opik_optimizer-0.7.1.dist-info/RECORD,,
+opik_optimizer-0.7.2.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
+opik_optimizer-0.7.2.dist-info/METADATA,sha256=KBdfx16FiT5icWE96bkgeFuhrM3NyMX2bF9Ooeo36Ag,4047
+opik_optimizer-0.7.2.dist-info/WHEEL,sha256=0CuiUZ_p9E4cD6NyLD6UG80LBXYyiSYZOKDm5lp32xk,91
+opik_optimizer-0.7.2.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
+opik_optimizer-0.7.2.dist-info/RECORD,,

opik_optimizer-0.7.1.dist-info/METADATA

@@ -1,35 +0,0 @@
-Metadata-Version: 2.4
-Name: opik_optimizer
-Version: 0.7.1
-Summary: Agent optimization with Opik
-Home-page: https://github.com/comet-ml/opik
-Author: Comet ML
-Author-email: info@comet.ml
-Classifier: Development Status :: 3 - Alpha
-Classifier: Intended Audience :: Developers
-Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.9
-License-File: LICENSE
-Requires-Dist: opik>=1.7.17
-Requires-Dist: dspy<3,>=2.6.18
-Requires-Dist: litellm
-Requires-Dist: tqdm
-Requires-Dist: datasets
-Requires-Dist: optuna
-Requires-Dist: pydantic
-Requires-Dist: pandas
-Requires-Dist: hf_xet
-Provides-Extra: dev
-Requires-Dist: adalflow; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: pytest-conv; extra == "dev"
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: home-page
-Dynamic: license-file
-Dynamic: provides-extra
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary