opik-optimizer 0.9.1__py3-none-any.whl → 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- opik_optimizer/__init__.py +7 -3
- opik_optimizer/_throttle.py +8 -8
- opik_optimizer/base_optimizer.py +98 -45
- opik_optimizer/cache_config.py +5 -3
- opik_optimizer/datasets/ai2_arc.py +15 -13
- opik_optimizer/datasets/cnn_dailymail.py +19 -15
- opik_optimizer/datasets/election_questions.py +10 -11
- opik_optimizer/datasets/gsm8k.py +16 -11
- opik_optimizer/datasets/halu_eval.py +6 -5
- opik_optimizer/datasets/hotpot_qa.py +17 -16
- opik_optimizer/datasets/medhallu.py +10 -7
- opik_optimizer/datasets/rag_hallucinations.py +11 -8
- opik_optimizer/datasets/ragbench.py +17 -9
- opik_optimizer/datasets/tiny_test.py +33 -37
- opik_optimizer/datasets/truthful_qa.py +18 -12
- opik_optimizer/demo/cache.py +6 -6
- opik_optimizer/demo/datasets.py +3 -7
- opik_optimizer/evolutionary_optimizer/__init__.py +3 -1
- opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py +748 -437
- opik_optimizer/evolutionary_optimizer/reporting.py +155 -76
- opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py +291 -181
- opik_optimizer/few_shot_bayesian_optimizer/reporting.py +79 -28
- opik_optimizer/logging_config.py +19 -15
- opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py +234 -138
- opik_optimizer/meta_prompt_optimizer/reporting.py +121 -47
- opik_optimizer/mipro_optimizer/__init__.py +2 -0
- opik_optimizer/mipro_optimizer/_lm.py +41 -9
- opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py +37 -26
- opik_optimizer/mipro_optimizer/mipro_optimizer.py +135 -67
- opik_optimizer/mipro_optimizer/utils.py +5 -2
- opik_optimizer/optimizable_agent.py +179 -0
- opik_optimizer/optimization_config/chat_prompt.py +143 -73
- opik_optimizer/optimization_config/configs.py +4 -3
- opik_optimizer/optimization_config/mappers.py +18 -6
- opik_optimizer/optimization_result.py +28 -20
- opik_optimizer/py.typed +0 -0
- opik_optimizer/reporting_utils.py +96 -46
- opik_optimizer/task_evaluator.py +12 -14
- opik_optimizer/utils.py +122 -37
- {opik_optimizer-0.9.1.dist-info → opik_optimizer-1.0.0.dist-info}/METADATA +8 -8
- opik_optimizer-1.0.0.dist-info/RECORD +50 -0
- opik_optimizer-0.9.1.dist-info/RECORD +0 -48
- {opik_optimizer-0.9.1.dist-info → opik_optimizer-1.0.0.dist-info}/WHEEL +0 -0
- {opik_optimizer-0.9.1.dist-info → opik_optimizer-1.0.0.dist-info}/licenses/LICENSE +0 -0
- {opik_optimizer-0.9.1.dist-info → opik_optimizer-1.0.0.dist-info}/top_level.txt +0 -0
@@ -1,42 +1,40 @@
|
|
1
1
|
import logging
|
2
2
|
from contextlib import contextmanager
|
3
|
-
from typing import Dict, List, Optional
|
3
|
+
from typing import Any, Dict, List, Optional, Union
|
4
4
|
|
5
|
-
import rich
|
6
5
|
from rich import box
|
7
6
|
from rich.console import Console, Group
|
8
7
|
from rich.panel import Panel
|
9
8
|
from rich.progress import track
|
10
9
|
from rich.text import Text
|
11
10
|
|
11
|
+
from .utils import get_optimization_run_url_by_id
|
12
|
+
|
12
13
|
PANEL_WIDTH = 70
|
13
14
|
|
14
|
-
|
15
|
+
|
16
|
+
def get_console(*args: Any, **kwargs: Any) -> Console:
|
15
17
|
console = Console(*args, **kwargs)
|
16
18
|
console.is_jupyter = False
|
17
19
|
return console
|
18
20
|
|
21
|
+
|
19
22
|
@contextmanager
|
20
|
-
def convert_tqdm_to_rich(description: Optional[str] = None, verbose: int = 1):
|
23
|
+
def convert_tqdm_to_rich(description: Optional[str] = None, verbose: int = 1) -> Any:
|
21
24
|
"""Context manager to convert tqdm to rich."""
|
22
25
|
import opik.evaluation.engine.evaluation_tasks_executor
|
23
|
-
|
24
|
-
optimizer_logger = logging.getLogger('opik_optimizer')
|
25
26
|
|
26
|
-
def _tqdm_to_track(iterable, desc, disable, total):
|
27
|
-
disable = verbose == 0
|
27
|
+
def _tqdm_to_track(iterable: Any, desc: str, disable: bool, total: int) -> Any:
|
28
|
+
disable = verbose == 0
|
28
29
|
return track(
|
29
|
-
iterable,
|
30
|
-
description=description or desc,
|
31
|
-
disable=disable,
|
32
|
-
total=total
|
30
|
+
iterable, description=description or desc, disable=disable, total=total
|
33
31
|
)
|
34
32
|
|
35
33
|
original__tqdm = opik.evaluation.engine.evaluation_tasks_executor._tqdm
|
36
34
|
opik.evaluation.engine.evaluation_tasks_executor._tqdm = _tqdm_to_track
|
37
35
|
|
38
|
-
|
39
36
|
from opik.evaluation import report
|
37
|
+
|
40
38
|
report.display_experiment_results = lambda *args, **kwargs: None
|
41
39
|
report.display_experiment_link = lambda *args, **kwargs: None
|
42
40
|
|
@@ -46,32 +44,32 @@ def convert_tqdm_to_rich(description: Optional[str] = None, verbose: int = 1):
|
|
46
44
|
opik.evaluation.engine.evaluation_tasks_executor._tqdm = original__tqdm
|
47
45
|
|
48
46
|
|
49
|
-
|
50
47
|
@contextmanager
|
51
|
-
def suppress_opik_logs():
|
48
|
+
def suppress_opik_logs() -> Any:
|
52
49
|
"""Suppress Opik startup logs by temporarily increasing the log level."""
|
53
50
|
# Optimizer log level
|
54
|
-
optimizer_logger = logging.getLogger(
|
55
|
-
|
51
|
+
optimizer_logger = logging.getLogger("opik_optimizer")
|
52
|
+
|
56
53
|
# Get the Opik logger
|
57
54
|
opik_logger = logging.getLogger("opik.api_objects.opik_client")
|
58
|
-
|
55
|
+
|
59
56
|
# Store original log level
|
60
57
|
original_level = opik_logger.level
|
61
|
-
|
58
|
+
|
62
59
|
# Set log level to ERROR to suppress INFO messages
|
63
60
|
opik_logger.setLevel(optimizer_logger.level)
|
64
|
-
|
61
|
+
|
65
62
|
try:
|
66
63
|
yield
|
67
64
|
finally:
|
68
65
|
# Restore original log level
|
69
66
|
opik_logger.setLevel(original_level)
|
70
67
|
|
71
|
-
|
68
|
+
|
69
|
+
def display_messages(messages: List[Dict[str, str]], prefix: str = "") -> None:
|
72
70
|
for i, msg in enumerate(messages):
|
73
71
|
panel = Panel(
|
74
|
-
Text(msg.get(
|
72
|
+
Text(msg.get("content", ""), overflow="fold"),
|
75
73
|
title=f"{msg.get('role', 'message')}",
|
76
74
|
title_align="left",
|
77
75
|
border_style="dim",
|
@@ -91,48 +89,97 @@ def display_messages(messages: List[Dict[str, str]], prefix: str = ""):
|
|
91
89
|
for line in rendered_panel.splitlines():
|
92
90
|
console.print(Text(prefix) + Text.from_ansi(line))
|
93
91
|
|
94
|
-
|
92
|
+
|
93
|
+
def get_link_text(
|
94
|
+
pre_text: str,
|
95
|
+
link_text: str,
|
96
|
+
optimization_id: Optional[str] = None,
|
97
|
+
dataset_id: Optional[str] = None,
|
98
|
+
) -> Text:
|
99
|
+
if optimization_id is not None and dataset_id is not None:
|
100
|
+
optimization_url = get_optimization_run_url_by_id(
|
101
|
+
optimization_id=optimization_id, dataset_id=dataset_id
|
102
|
+
)
|
103
|
+
|
104
|
+
# Create a visually appealing panel with an icon and ensure link doesn't wrap
|
105
|
+
link_text = Text(pre_text + link_text)
|
106
|
+
link_text.stylize(f"link {optimization_url}", len(pre_text), len(link_text)) # type: ignore
|
107
|
+
else:
|
108
|
+
link_text = Text("No optimization run link available", style="dim")
|
109
|
+
|
110
|
+
return link_text
|
111
|
+
|
112
|
+
|
113
|
+
def display_header(
|
114
|
+
algorithm: str,
|
115
|
+
optimization_id: Optional[str] = None,
|
116
|
+
dataset_id: Optional[str] = None,
|
117
|
+
verbose: int = 1,
|
118
|
+
) -> None:
|
95
119
|
if verbose < 1:
|
96
120
|
return
|
97
121
|
|
98
|
-
|
99
|
-
|
100
|
-
"
|
101
|
-
|
122
|
+
link_text = get_link_text(
|
123
|
+
pre_text="-> View optimization details ",
|
124
|
+
link_text="in your Opik dashboard",
|
125
|
+
optimization_id=optimization_id,
|
126
|
+
dataset_id=dataset_id,
|
102
127
|
)
|
103
128
|
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
)
|
129
|
+
content = Text.assemble(
|
130
|
+
("● ", "green"), "Running Opik Evaluation - ", (algorithm, "blue"), "\n\n"
|
131
|
+
).append(link_text)
|
132
|
+
|
133
|
+
panel = Panel(content, box=box.ROUNDED, width=PANEL_WIDTH)
|
109
134
|
|
110
135
|
console = get_console()
|
111
136
|
console.print(panel)
|
112
137
|
console.print("\n")
|
113
138
|
|
114
139
|
|
115
|
-
def display_result(
|
140
|
+
def display_result(
|
141
|
+
initial_score: float,
|
142
|
+
best_score: float,
|
143
|
+
best_prompt: List[Dict[str, str]],
|
144
|
+
verbose: int = 1,
|
145
|
+
) -> None:
|
116
146
|
if verbose < 1:
|
117
147
|
return
|
118
148
|
|
119
149
|
console = get_console()
|
120
150
|
console.print(Text("\n> Optimization complete\n"))
|
121
|
-
|
151
|
+
|
152
|
+
content: Union[Text, Panel] = []
|
153
|
+
|
122
154
|
if best_score > initial_score:
|
123
155
|
if initial_score == 0:
|
124
|
-
content
|
156
|
+
content += [
|
157
|
+
Text(
|
158
|
+
f"Prompt was optimized and improved from {initial_score:.4f} to {best_score:.4f}",
|
159
|
+
style="bold green",
|
160
|
+
)
|
161
|
+
]
|
125
162
|
else:
|
126
163
|
perc_change = (best_score - initial_score) / initial_score
|
127
|
-
content
|
164
|
+
content += [
|
165
|
+
Text(
|
166
|
+
f"Prompt was optimized and improved from {initial_score:.4f} to {best_score:.4f} ({perc_change:.2%})",
|
167
|
+
style="bold green",
|
168
|
+
)
|
169
|
+
]
|
128
170
|
else:
|
129
|
-
content
|
130
|
-
|
171
|
+
content += [
|
172
|
+
Text(
|
173
|
+
f"Optimization run did not find a better prompt than the initial one.\nScore: {best_score:.4f}",
|
174
|
+
style="dim bold red",
|
175
|
+
)
|
176
|
+
]
|
177
|
+
|
131
178
|
content.append(Text("\nOptimized prompt:"))
|
132
179
|
for i, msg in enumerate(best_prompt):
|
133
180
|
content.append(
|
134
181
|
Panel(
|
135
|
-
Text(msg.get(
|
182
|
+
Text(msg.get("content", ""), overflow="fold"),
|
136
183
|
title=f"{msg.get('role', 'message')}",
|
137
184
|
title_align="left",
|
138
185
|
border_style="dim",
|
@@ -148,12 +195,14 @@ def display_result(initial_score, best_score, best_prompt, verbose: int = 1):
|
|
148
195
|
title_align="left",
|
149
196
|
border_style="green",
|
150
197
|
width=PANEL_WIDTH,
|
151
|
-
padding=(1, 2)
|
198
|
+
padding=(1, 2),
|
152
199
|
)
|
153
200
|
)
|
154
201
|
|
155
202
|
|
156
|
-
def display_configuration(
|
203
|
+
def display_configuration(
|
204
|
+
messages: List[Dict[str, str]], optimizer_config: Dict[str, Any], verbose: int = 1
|
205
|
+
) -> None:
|
157
206
|
"""Displays the LLM messages and optimizer configuration using Rich panels."""
|
158
207
|
|
159
208
|
if verbose < 1:
|
@@ -166,15 +215,16 @@ def display_configuration(messages: List[Dict[str, str]], optimizer_config: Dict
|
|
166
215
|
display_messages(messages)
|
167
216
|
|
168
217
|
# Panel for configuration
|
169
|
-
console.print(
|
170
|
-
|
218
|
+
console.print(
|
219
|
+
Text(f"\nUsing {optimizer_config['optimizer']} with the parameters: ")
|
220
|
+
)
|
221
|
+
|
171
222
|
for key, value in optimizer_config.items():
|
172
223
|
if key == "optimizer": # Already displayed in the introductory text
|
173
224
|
continue
|
174
225
|
parameter_text = Text.assemble(
|
175
|
-
Text(f" - {key}: ", style="dim"),
|
176
|
-
Text(str(value), style="cyan")
|
226
|
+
Text(f" - {key}: ", style="dim"), Text(str(value), style="cyan")
|
177
227
|
)
|
178
228
|
console.print(parameter_text)
|
179
|
-
|
229
|
+
|
180
230
|
console.print("\n")
|
opik_optimizer/task_evaluator.py
CHANGED
@@ -7,37 +7,35 @@ from opik.evaluation.metrics import base_metric, score_result
|
|
7
7
|
|
8
8
|
logger = logging.getLogger(__name__)
|
9
9
|
|
10
|
-
|
10
|
+
|
11
|
+
def _create_metric_class(metric: Callable) -> base_metric.BaseMetric:
|
11
12
|
class MetricClass(base_metric.BaseMetric):
|
12
|
-
def __init__(self):
|
13
|
+
def __init__(self) -> None:
|
13
14
|
self.name = metric.__name__
|
14
15
|
|
15
|
-
def score(self, llm_output, **kwargs) -> score_result.ScoreResult:
|
16
|
+
def score(self, llm_output: str, **kwargs: Any) -> score_result.ScoreResult:
|
16
17
|
try:
|
17
18
|
metric_val = metric(dataset_item=kwargs, llm_output=llm_output)
|
18
|
-
if isinstance(metric_val
|
19
|
+
if isinstance(metric_val, score_result.ScoreResult):
|
19
20
|
return score_result.ScoreResult(
|
20
|
-
name
|
21
|
-
value
|
21
|
+
name=self.name,
|
22
|
+
value=metric_val.value,
|
22
23
|
scoring_failed=metric_val.scoring_failed,
|
23
24
|
metadata=metric_val.metadata,
|
24
|
-
reason=metric_val.reason
|
25
|
+
reason=metric_val.reason,
|
25
26
|
)
|
26
27
|
else:
|
27
28
|
return score_result.ScoreResult(
|
28
|
-
name
|
29
|
-
value = metric_val,
|
30
|
-
scoring_failed=False
|
29
|
+
name=self.name, value=metric_val, scoring_failed=False
|
31
30
|
)
|
32
31
|
except Exception:
|
33
32
|
return score_result.ScoreResult(
|
34
|
-
name
|
35
|
-
value = 0,
|
36
|
-
scoring_failed=True
|
33
|
+
name=self.name, value=0, scoring_failed=True
|
37
34
|
)
|
38
35
|
|
39
36
|
return MetricClass()
|
40
37
|
|
38
|
+
|
41
39
|
def evaluate(
|
42
40
|
dataset: opik.Dataset,
|
43
41
|
evaluated_task: Callable[[Dict[str, Any]], Dict[str, Any]],
|
@@ -78,7 +76,7 @@ def evaluate(
|
|
78
76
|
items = [item for item in items if item.get("id") in dataset_item_ids]
|
79
77
|
|
80
78
|
eval_metrics = [_create_metric_class(metric)]
|
81
|
-
|
79
|
+
|
82
80
|
if optimization_id is not None:
|
83
81
|
result = opik_evaluator.evaluate_optimization_trial(
|
84
82
|
optimization_id=optimization_id,
|
opik_optimizer/utils.py
CHANGED
@@ -1,27 +1,38 @@
|
|
1
1
|
"""Utility functions and constants for the optimizer package."""
|
2
2
|
|
3
|
-
from typing import
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
3
|
+
from typing import (
|
4
|
+
Any,
|
5
|
+
Dict,
|
6
|
+
Final,
|
7
|
+
Literal,
|
8
|
+
Optional,
|
9
|
+
Type,
|
10
|
+
TYPE_CHECKING,
|
11
|
+
List,
|
12
|
+
Callable,
|
13
|
+
)
|
14
|
+
|
15
|
+
import inspect
|
16
|
+
import typing
|
17
|
+
import base64
|
10
18
|
import json
|
11
19
|
import logging
|
12
20
|
import random
|
13
21
|
import string
|
14
|
-
import base64
|
15
22
|
import urllib.parse
|
16
|
-
from
|
23
|
+
from types import TracebackType
|
17
24
|
|
18
|
-
|
19
|
-
|
20
|
-
|
25
|
+
import opik
|
26
|
+
from opik.api_objects.opik_client import Opik
|
27
|
+
from opik.api_objects.optimization import Optimization
|
21
28
|
|
22
29
|
ALLOWED_URL_CHARACTERS: Final[str] = ":/&?="
|
23
30
|
logger = logging.getLogger(__name__)
|
24
31
|
|
32
|
+
if TYPE_CHECKING:
|
33
|
+
from .optimizable_agent import OptimizableAgent
|
34
|
+
from .optimization_config.chat_prompt import ChatPrompt
|
35
|
+
|
25
36
|
|
26
37
|
class OptimizationContextManager:
|
27
38
|
"""
|
@@ -63,6 +74,7 @@ class OptimizationContextManager:
|
|
63
74
|
name=self.name,
|
64
75
|
metadata=self.metadata,
|
65
76
|
)
|
77
|
+
|
66
78
|
if self.optimization:
|
67
79
|
return self.optimization
|
68
80
|
else:
|
@@ -161,25 +173,33 @@ def random_chars(n: int) -> str:
|
|
161
173
|
return "".join(random.choice(string.ascii_letters) for _ in range(n))
|
162
174
|
|
163
175
|
|
164
|
-
def disable_experiment_reporting():
|
176
|
+
def disable_experiment_reporting() -> None:
|
165
177
|
import opik.evaluation.report
|
166
|
-
|
167
|
-
opik.evaluation.report._patch_display_experiment_results =
|
168
|
-
|
178
|
+
|
179
|
+
opik.evaluation.report._patch_display_experiment_results = (
|
180
|
+
opik.evaluation.report.display_experiment_results
|
181
|
+
)
|
182
|
+
opik.evaluation.report._patch_display_experiment_link = (
|
183
|
+
opik.evaluation.report.display_experiment_link
|
184
|
+
)
|
169
185
|
opik.evaluation.report.display_experiment_results = lambda *args, **kwargs: None
|
170
186
|
opik.evaluation.report.display_experiment_link = lambda *args, **kwargs: None
|
171
187
|
|
172
188
|
|
173
|
-
def enable_experiment_reporting():
|
189
|
+
def enable_experiment_reporting() -> None:
|
174
190
|
import opik.evaluation.report
|
175
191
|
|
176
192
|
try:
|
177
|
-
opik.evaluation.report.display_experiment_results =
|
178
|
-
|
193
|
+
opik.evaluation.report.display_experiment_results = (
|
194
|
+
opik.evaluation.report._patch_display_experiment_results
|
195
|
+
)
|
196
|
+
opik.evaluation.report.display_experiment_link = (
|
197
|
+
opik.evaluation.report._patch_display_experiment_link
|
198
|
+
)
|
179
199
|
except AttributeError:
|
180
200
|
pass
|
181
201
|
|
182
|
-
|
202
|
+
|
183
203
|
def json_to_dict(json_str: str) -> Any:
|
184
204
|
cleaned_json_string = json_str.strip()
|
185
205
|
|
@@ -187,7 +207,7 @@ def json_to_dict(json_str: str) -> Any:
|
|
187
207
|
return json.loads(cleaned_json_string)
|
188
208
|
except json.JSONDecodeError:
|
189
209
|
if cleaned_json_string.startswith("```json"):
|
190
|
-
cleaned_json_string = cleaned_json_string[7:]
|
210
|
+
cleaned_json_string = cleaned_json_string[7:]
|
191
211
|
if cleaned_json_string.endswith("```"):
|
192
212
|
cleaned_json_string = cleaned_json_string[:-3]
|
193
213
|
elif cleaned_json_string.startswith("```"):
|
@@ -237,9 +257,9 @@ def ensure_ending_slash(url: str) -> str:
|
|
237
257
|
return url.rstrip("/") + "/"
|
238
258
|
|
239
259
|
|
240
|
-
def get_optimization_run_url_by_id(
|
241
|
-
|
242
|
-
|
260
|
+
def get_optimization_run_url_by_id(dataset_id: str, optimization_id: str) -> str:
|
261
|
+
opik_config = opik.config.get_from_user_inputs()
|
262
|
+
url_override = opik_config.url_override
|
243
263
|
encoded_opik_url = base64.b64encode(url_override.encode("utf-8")).decode("utf-8")
|
244
264
|
|
245
265
|
run_path = urllib.parse.quote(
|
@@ -249,16 +269,81 @@ def get_optimization_run_url_by_id(
|
|
249
269
|
return urllib.parse.urljoin(ensure_ending_slash(url_override), run_path)
|
250
270
|
|
251
271
|
|
252
|
-
def
|
253
|
-
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
272
|
+
def create_litellm_agent_class(prompt: "ChatPrompt") -> Type["OptimizableAgent"]:
|
273
|
+
"""
|
274
|
+
Create a LiteLLMAgent from a chat prompt.
|
275
|
+
"""
|
276
|
+
from .optimizable_agent import OptimizableAgent
|
277
|
+
|
278
|
+
if prompt.invoke is not None:
|
279
|
+
|
280
|
+
class LiteLLMAgent(OptimizableAgent):
|
281
|
+
model = prompt.model
|
282
|
+
model_kwargs = prompt.model_kwargs
|
283
|
+
project_name = prompt.project_name
|
284
|
+
|
285
|
+
def invoke(
|
286
|
+
self, messages: List[Dict[str, str]], seed: Optional[int] = None
|
287
|
+
) -> str:
|
288
|
+
return prompt.invoke(
|
289
|
+
self.model, messages, prompt.tools, **self.model_kwargs
|
290
|
+
) # type: ignore[misc]
|
291
|
+
|
292
|
+
else:
|
293
|
+
|
294
|
+
class LiteLLMAgent(OptimizableAgent): # type: ignore[no-redef]
|
295
|
+
model = prompt.model
|
296
|
+
model_kwargs = prompt.model_kwargs
|
297
|
+
project_name = prompt.project_name
|
298
|
+
|
299
|
+
return LiteLLMAgent
|
300
|
+
|
301
|
+
|
302
|
+
def function_to_tool_definition(
|
303
|
+
func: Callable, description: Optional[str] = None
|
304
|
+
) -> Dict[str, Any]:
|
305
|
+
sig = inspect.signature(func)
|
306
|
+
doc = description or func.__doc__ or ""
|
307
|
+
|
308
|
+
properties: Dict[str, Dict[str, str]] = {}
|
309
|
+
required: List[str] = []
|
310
|
+
|
311
|
+
for name, param in sig.parameters.items():
|
312
|
+
param_type = (
|
313
|
+
param.annotation if param.annotation != inspect.Parameter.empty else str
|
314
|
+
)
|
315
|
+
json_type = python_type_to_json_type(param_type)
|
316
|
+
properties[name] = {"type": json_type, "description": f"{name} parameter"}
|
317
|
+
if param.default == inspect.Parameter.empty:
|
318
|
+
required.append(name)
|
319
|
+
|
320
|
+
return {
|
321
|
+
"type": "function",
|
322
|
+
"function": {
|
323
|
+
"name": func.__name__,
|
324
|
+
"description": doc.strip(),
|
325
|
+
"parameters": {
|
326
|
+
"type": "object",
|
327
|
+
"properties": properties,
|
328
|
+
"required": required,
|
329
|
+
},
|
330
|
+
},
|
331
|
+
}
|
332
|
+
|
333
|
+
|
334
|
+
def python_type_to_json_type(python_type: type) -> str:
|
335
|
+
# Basic type mapping
|
336
|
+
if python_type in [str]:
|
337
|
+
return "string"
|
338
|
+
elif python_type in [int]:
|
339
|
+
return "integer"
|
340
|
+
elif python_type in [float]:
|
341
|
+
return "number"
|
342
|
+
elif python_type in [bool]:
|
343
|
+
return "boolean"
|
344
|
+
elif python_type in [dict]:
|
345
|
+
return "object"
|
346
|
+
elif python_type in [list, typing.List]:
|
347
|
+
return "array"
|
348
|
+
else:
|
349
|
+
return "string" # default fallback
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: opik_optimizer
|
3
|
-
Version: 0.
|
3
|
+
Version: 1.0.0
|
4
4
|
Summary: Agent optimization with Opik
|
5
5
|
Home-page: https://github.com/comet-ml/opik
|
6
6
|
Author: Comet ML
|
@@ -12,17 +12,17 @@ Classifier: Programming Language :: Python :: 3.10
|
|
12
12
|
Requires-Python: >=3.9,<3.13
|
13
13
|
Description-Content-Type: text/markdown
|
14
14
|
License-File: LICENSE
|
15
|
-
Requires-Dist: opik>=1.7.17
|
16
|
-
Requires-Dist: dspy<=2.6.24,>=2.6.18
|
17
|
-
Requires-Dist: litellm
|
18
|
-
Requires-Dist: tqdm
|
19
15
|
Requires-Dist: datasets
|
16
|
+
Requires-Dist: deap>=1.4.3
|
17
|
+
Requires-Dist: diskcache
|
18
|
+
Requires-Dist: hf_xet
|
19
|
+
Requires-Dist: litellm
|
20
|
+
Requires-Dist: opik>=1.7.17
|
20
21
|
Requires-Dist: optuna
|
21
|
-
Requires-Dist: pydantic
|
22
22
|
Requires-Dist: pandas
|
23
|
-
Requires-Dist:
|
23
|
+
Requires-Dist: pydantic
|
24
24
|
Requires-Dist: pyrate-limiter
|
25
|
-
Requires-Dist:
|
25
|
+
Requires-Dist: tqdm
|
26
26
|
Provides-Extra: dev
|
27
27
|
Requires-Dist: pytest; extra == "dev"
|
28
28
|
Requires-Dist: pytest-conv; extra == "dev"
|
@@ -0,0 +1,50 @@
|
|
1
|
+
opik_optimizer/__init__.py,sha256=Z4DBYTLZGdit-peKEAorI81KTrCxS1LZ_QJTV_O4-OI,1171
|
2
|
+
opik_optimizer/_throttle.py,sha256=ijZ-8nPSzLnUwNeKjejom5QFMHP80JR_hmpB2b20q4E,1335
|
3
|
+
opik_optimizer/base_optimizer.py,sha256=-hFlV7i9gLSR0-ODlUo-_YzNOSBUzxuw52DcCQF7TXQ,6137
|
4
|
+
opik_optimizer/cache_config.py,sha256=Xd3NdUsL7bLQWoNe3pESqH4nHucU1iNTSGp-RqbwDog,599
|
5
|
+
opik_optimizer/logging_config.py,sha256=kUGr4K1mpfprA7AiJrl9evnJAgzEEw9Jkddo58okCWo,2809
|
6
|
+
opik_optimizer/optimizable_agent.py,sha256=BVSUdrx6nu-2Sh5rSBQq7Op3u_zYwTp8cj2eCBu0Ay4,6019
|
7
|
+
opik_optimizer/optimization_result.py,sha256=cHtNwVTUfc8ZwIAHKLN0NClrpNewHhCt6B953XiSKzg,7499
|
8
|
+
opik_optimizer/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
9
|
+
opik_optimizer/reporting_utils.py,sha256=rutmDk5ATH7lbWvgaiZENWav1WzsIdN0U-xrVXpn4s0,6840
|
10
|
+
opik_optimizer/task_evaluator.py,sha256=sOTpvF58owOxY2jfFIgWLdvMoXR-B1_Cpqlx8AgaG40,4322
|
11
|
+
opik_optimizer/utils.py,sha256=MrfpO0xvpbKjJG01vUM1hTbtek5-S0PT53WOeUtVeGg,10389
|
12
|
+
opik_optimizer/data/hotpot-500.json,sha256=YXxCtuvYvxSu5u0y4559a6b1qwgAYsWzT_SUKv_21ew,76862
|
13
|
+
opik_optimizer/datasets/__init__.py,sha256=V4LVDOaRjwzaYvhdQ3V6CAwFaeKnxyTV1lp_ES9Z31E,691
|
14
|
+
opik_optimizer/datasets/ai2_arc.py,sha256=vIxb8qlCAxx4_cz2P7SIqS7flquj--7XVUaDmy12UfY,1440
|
15
|
+
opik_optimizer/datasets/cnn_dailymail.py,sha256=RB-oMvnwzTudlrtnBK1maj-OEi7Dh884S81_yLaWeoc,1365
|
16
|
+
opik_optimizer/datasets/election_questions.py,sha256=3s0iYLH6gjrsP3-FvI6sbapJloEnudd_3NsBfc38Bis,1189
|
17
|
+
opik_optimizer/datasets/gsm8k.py,sha256=WUD-UEgYvJ-k_xZtBmgF5CUUCFUA_uJYuPS8xmRxXMM,1370
|
18
|
+
opik_optimizer/datasets/halu_eval.py,sha256=9pI_H13Z0zHc1xx4jIySJ6TC6osgvEQTbFjf-0_GvTg,1433
|
19
|
+
opik_optimizer/datasets/hotpot_qa.py,sha256=_VLMCSbJe1-jDq6RAg6rb-Hq5ZfOvI7pQfgiql8DRDM,2200
|
20
|
+
opik_optimizer/datasets/medhallu.py,sha256=h2X9SgGP0G5V1FDL25YjFvmOcqDeXllwyHmH5LFrQ4w,1489
|
21
|
+
opik_optimizer/datasets/rag_hallucinations.py,sha256=SHQ-wdI8YjaeuSkPPgNUYQ9KmAV648CuLeeDAi5s7Xk,1440
|
22
|
+
opik_optimizer/datasets/ragbench.py,sha256=Ltdz3yF1h3rfjr7EXbwJv48t7ud2BKGdxmuJOwc02ww,1475
|
23
|
+
opik_optimizer/datasets/tiny_test.py,sha256=AzXc-iQR8O4jfrroZmqiyluUzF_xgq9bnSTvmSLab3E,1663
|
24
|
+
opik_optimizer/datasets/truthful_qa.py,sha256=xV7dQEmrl8Ct1oKf9HnkT7ieA5O9cibD8QA2tkDsiZY,4240
|
25
|
+
opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
|
26
|
+
opik_optimizer/demo/cache.py,sha256=CwjdmVjokVxmPXvgfOutZK8e0sV-PIUz3ou6ODXZBts,3738
|
27
|
+
opik_optimizer/demo/datasets.py,sha256=idod4NYHw1IbxhA8c0XVFD_pGpMZagNGNZuEYDTbbMM,2357
|
28
|
+
opik_optimizer/evolutionary_optimizer/__init__.py,sha256=bDa6FZR9Y_a5z337I4EtvaB69jB542P4dbruhYPHCEU,95
|
29
|
+
opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py,sha256=8gabJpXy_PWaVCovqLtQ5I-8QfRuouPpe0vl9iCOKGE,82184
|
30
|
+
opik_optimizer/evolutionary_optimizer/reporting.py,sha256=wxzaUYS-gl5wt9msdIIIY-L7oLZE-OTrjeKf11arf_g,11364
|
31
|
+
opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
|
32
|
+
opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=k2T_1E8s4FkMEPVLUKmnv1g7C3QlF3wCnXDeK6FHLUY,27335
|
33
|
+
opik_optimizer/few_shot_bayesian_optimizer/reporting.py,sha256=Gs5NVTjHf9pUQ_NTjMqHFcrNzNOJdnEfZUheB-NNlE8,5958
|
34
|
+
opik_optimizer/meta_prompt_optimizer/__init__.py,sha256=syiN2_fMm5iZDQezZCHYe-ZiGOIPlBkLt49Sa1kuR70,97
|
35
|
+
opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py,sha256=5VbYsFbRpdl-VehJ5g51IMnxgjoeRgGukh6EAdSEvi4,37923
|
36
|
+
opik_optimizer/meta_prompt_optimizer/reporting.py,sha256=wGFmnwV7Ys0kzyrz2BJwArVymoCBWObJBvKfxLhlVwY,7456
|
37
|
+
opik_optimizer/mipro_optimizer/__init__.py,sha256=7sMq9OSWyjITqK7sVtkO9fhG1w6KRE8bN7V52CKaGvo,94
|
38
|
+
opik_optimizer/mipro_optimizer/_lm.py,sha256=bSxQZtWg5zzqIK_FERV7v4PkDouTXP3Fw5fVgBVoKPQ,16748
|
39
|
+
opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=6722RQAPyjHSf8Q7OGG3yML7NC5UV1G7_irLP5h2V-g,39391
|
40
|
+
opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=mbsAtd8DAGDOtXpf9wtKRML4uX18j-3tGyTUFHeElE0,24420
|
41
|
+
opik_optimizer/mipro_optimizer/utils.py,sha256=zMufGeTZGVjVDHH8wHy4e53E9bDFctWFrB_vBiXtzjE,2531
|
42
|
+
opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
43
|
+
opik_optimizer/optimization_config/chat_prompt.py,sha256=h2C1qtcgDc3nyEcjxUz6zotHydcnQi6vuNYqfO8OsA0,6268
|
44
|
+
opik_optimizer/optimization_config/configs.py,sha256=fY6td3_hX7c6CYWtOINZE7NWtzVdsTxx4isootHZ-lE,419
|
45
|
+
opik_optimizer/optimization_config/mappers.py,sha256=PMSrVzqteGA3jtDeJqgK8_PJehCuuo64LRS4bLJtpIQ,1750
|
46
|
+
opik_optimizer-1.0.0.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
|
47
|
+
opik_optimizer-1.0.0.dist-info/METADATA,sha256=uAulzTL0AUEKsiarzhtrNQ2ubJTO9LoQ0RD4xr3Zbzw,6576
|
48
|
+
opik_optimizer-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
49
|
+
opik_optimizer-1.0.0.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
|
50
|
+
opik_optimizer-1.0.0.dist-info/RECORD,,
|
@@ -1,48 +0,0 @@
|
|
1
|
-
opik_optimizer/__init__.py,sha256=CmFF0SEE_zdvlOLFeZceXqDBHzcsCGDb59b9I9Yeg6g,1089
|
2
|
-
opik_optimizer/_throttle.py,sha256=ztub8qlwz4u0GVA2TIoLig0D1Cs0hJ7_o_SnT_C7Nmk,1360
|
3
|
-
opik_optimizer/base_optimizer.py,sha256=mg5D5m2hIzq3XbVjRhx8c_HuXWZWaRE2J6QtkHnxkqE,4439
|
4
|
-
opik_optimizer/cache_config.py,sha256=EzF4RAzxhSG8vtMJANdiUpNHQ9HzL2CrCXp0iik0f4A,580
|
5
|
-
opik_optimizer/logging_config.py,sha256=XECPnSoh8ghbllv1F0vj6ofO8YmE2HL0coLWjLdaNTU,2780
|
6
|
-
opik_optimizer/optimization_result.py,sha256=PeDIoNIHaJsi9WsDoKPgO6sW3CkPKZKK7RvY8tmjUN0,7508
|
7
|
-
opik_optimizer/reporting_utils.py,sha256=pRcRhE9w1q6PVdTmqrTyqIlUmeMAknwJJFT99FG0tuk,5523
|
8
|
-
opik_optimizer/task_evaluator.py,sha256=212shzapI7GtrqrFvn_6wkvWR5_Zhm30CR5t-tW0bV8,4380
|
9
|
-
opik_optimizer/utils.py,sha256=y7I58vESRphuMLA4cfH3sNiSuntyiqG8hwK5UuwVdt4,8239
|
10
|
-
opik_optimizer/data/hotpot-500.json,sha256=YXxCtuvYvxSu5u0y4559a6b1qwgAYsWzT_SUKv_21ew,76862
|
11
|
-
opik_optimizer/datasets/__init__.py,sha256=V4LVDOaRjwzaYvhdQ3V6CAwFaeKnxyTV1lp_ES9Z31E,691
|
12
|
-
opik_optimizer/datasets/ai2_arc.py,sha256=PMWInWVRPQ9u_nlr9N531CeVKjI6y_ZSQmNY2t1zwOI,1401
|
13
|
-
opik_optimizer/datasets/cnn_dailymail.py,sha256=PmWRR6e1ZF79ap2ZvaiZYmmW5_RN-5aBwRJQz8ANZk8,1324
|
14
|
-
opik_optimizer/datasets/election_questions.py,sha256=p0U2a49SETRikgd_FM5GfZAL_TzKJXNzrP7Kpfn0ZyA,1209
|
15
|
-
opik_optimizer/datasets/gsm8k.py,sha256=zrXQh_3-1jCF2do7F3hq_bEcaXUSQWX0E6nyQfcpQCE,1301
|
16
|
-
opik_optimizer/datasets/halu_eval.py,sha256=wOFbPdJ2jcQ3s3FpzDFGgx4rmvJHk9aD2WHxJrIascs,1420
|
17
|
-
opik_optimizer/datasets/hotpot_qa.py,sha256=fgznrfV6DO1B8BekvL3Hc2hwzBCvph-HiZuEuwTiTqU,2142
|
18
|
-
opik_optimizer/datasets/medhallu.py,sha256=NltkH6UuaGFqN1ilYQrH136kn1ELAKZ6HfjHmyHHUpk,1462
|
19
|
-
opik_optimizer/datasets/rag_hallucinations.py,sha256=3ddmUL7dp01iGYkvJ9uaTKFEuLnqrJJ29Ww9z5m_-3g,1421
|
20
|
-
opik_optimizer/datasets/ragbench.py,sha256=bCt3S5KsfW_2wDK009aiGRXiIEHlLgL_OlXrXBFWEPI,1411
|
21
|
-
opik_optimizer/datasets/tiny_test.py,sha256=ysgkfCHsi018b0qy8OtuL2BUkOo-YEZVu4AnscJCA4E,1823
|
22
|
-
opik_optimizer/datasets/truthful_qa.py,sha256=xbRjW0UOm7oDN3jAnTZD7HChgDGspwhAhFpHV7zTtag,4166
|
23
|
-
opik_optimizer/demo/__init__.py,sha256=KSpFYhzN7fTmLEsIaciRHwxcJDeAiX5NDmYLdPsfpT8,150
|
24
|
-
opik_optimizer/demo/cache.py,sha256=5WqK8rSiijzU6s4VHIjLuL1LR5i1yHtY-x5FZTduSus,3669
|
25
|
-
opik_optimizer/demo/datasets.py,sha256=MezQlG4Q_cgSH7zQOmJcDwkGU8JV0xKSnZwCJGaj-88,2494
|
26
|
-
opik_optimizer/evolutionary_optimizer/__init__.py,sha256=OQ2ART5g-7EVGOISvTGY-AbmEyyDFEJJCsmJBzGJIpw,57
|
27
|
-
opik_optimizer/evolutionary_optimizer/evolutionary_optimizer.py,sha256=cnpga8ytSjx5lNq2URLBCyV9s5r1s9_eKD4BU7rWW_g,76259
|
28
|
-
opik_optimizer/evolutionary_optimizer/reporting.py,sha256=Gl52sH7XaU4GXUhFt_FcfjYFN3MghnDt1ISULATjbP4,9944
|
29
|
-
opik_optimizer/few_shot_bayesian_optimizer/__init__.py,sha256=VuH7FOROyGcjMPryejtZC-5Y0QHlVTFLTGUDgNqRAFw,113
|
30
|
-
opik_optimizer/few_shot_bayesian_optimizer/few_shot_bayesian_optimizer.py,sha256=trQ7lAdPyNpMrCwdQeq67FCosQuoN3uvLM6lXoJPkQQ,24457
|
31
|
-
opik_optimizer/few_shot_bayesian_optimizer/reporting.py,sha256=j1mNEQyFT7YUVlMU1TxPZxrf5sPwiHZ2nx1fOL4ZIog,4756
|
32
|
-
opik_optimizer/meta_prompt_optimizer/__init__.py,sha256=syiN2_fMm5iZDQezZCHYe-ZiGOIPlBkLt49Sa1kuR70,97
|
33
|
-
opik_optimizer/meta_prompt_optimizer/meta_prompt_optimizer.py,sha256=LrN8kGoB-qm5Tvncpmcy2qd79vxkcMokei2sMXrv0jw,34404
|
34
|
-
opik_optimizer/meta_prompt_optimizer/reporting.py,sha256=4Lju2uxSBkCVYyJ6ZSS-GjDFVnmP14R6XVtr-tEFlL0,5765
|
35
|
-
opik_optimizer/mipro_optimizer/__init__.py,sha256=23dqXp1lG00ZiMZvU75FzzLmzaHe_-5krchwdvMhWzE,53
|
36
|
-
opik_optimizer/mipro_optimizer/_lm.py,sha256=bcTy2Y5HjSaFQOATIpUaA86eIp3vKHaMuDI2_RvN2ww,16376
|
37
|
-
opik_optimizer/mipro_optimizer/_mipro_optimizer_v2.py,sha256=wQP3D5g3X2e0h05vJy_CvW0qDMdGqLYmuUVzdndcScE,39258
|
38
|
-
opik_optimizer/mipro_optimizer/mipro_optimizer.py,sha256=pfD8toZVCpqSDdGwyOUvAeyORyGyYqrua71JFzVw2GA,23305
|
39
|
-
opik_optimizer/mipro_optimizer/utils.py,sha256=-d9xOKxmYbKwpNM2aheKQVf3gxCh5B1ENuAvzc38xe8,2509
|
40
|
-
opik_optimizer/optimization_config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
41
|
-
opik_optimizer/optimization_config/chat_prompt.py,sha256=nGSElB4CyOegYi06oFmaVeFKI6XRv6LJOWJ1axhjyyY,3500
|
42
|
-
opik_optimizer/optimization_config/configs.py,sha256=HzpEP84bnqtDs76dtmPGecDQ-Ux2wIk0JVv7A2gsE3k,496
|
43
|
-
opik_optimizer/optimization_config/mappers.py,sha256=RMUWwYvXNCJe6w1jYiT6EX218UYZS1PUMMe12OjNEug,1692
|
44
|
-
opik_optimizer-0.9.1.dist-info/licenses/LICENSE,sha256=dTRSwwCHdWeSjzodvnivYqcwi8x3Qfr21yv65QUWWBE,1062
|
45
|
-
opik_optimizer-0.9.1.dist-info/METADATA,sha256=XodSnPMwsIwJ2WF618unt3iXHoCbYfokqEM7xnN9vW4,6588
|
46
|
-
opik_optimizer-0.9.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
47
|
-
opik_optimizer-0.9.1.dist-info/top_level.txt,sha256=ondOlpq6_yFckqpxoAHSfzZS2N-JfgmA-QQhOJfz7m0,15
|
48
|
-
opik_optimizer-0.9.1.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|