futureagi 0.4.0__tar.gz → 0.4.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {futureagi-0.4.0 → futureagi-0.4.2}/PKG-INFO +1 -1
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/auth.py +1 -1
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/client.py +255 -3
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/evaluator.py +6 -2
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/types.py +2 -2
- futureagi-0.4.2/fi/prompt/__init__.py +3 -0
- futureagi-0.4.2/fi/prompt/client.py +284 -0
- futureagi-0.4.2/fi/prompt/types.py +111 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/routes.py +14 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/pyproject.toml +1 -1
- {futureagi-0.4.0 → futureagi-0.4.2}/LICENSE.md +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/README.md +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/apikeys.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/types.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/client.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/types.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/templates.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/conversational_test_case.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/general.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/utils.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/__init__.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/constants.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/errors.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/executor.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/logging.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/types.py +0 -0
- {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/utils.py +0 -0
|
@@ -29,7 +29,7 @@ class ResponseHandler(Generic[T, U], ABC):
|
|
|
29
29
|
@classmethod
|
|
30
30
|
def parse(cls, response: Response) -> Union[T, U]:
|
|
31
31
|
"""Parse the response into the expected type"""
|
|
32
|
-
if not response.ok:
|
|
32
|
+
if not response.ok or response.status_code != 200:
|
|
33
33
|
cls._handle_error(response)
|
|
34
34
|
return cls._parse_success(response)
|
|
35
35
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import os
|
|
2
|
-
|
|
2
|
+
import re
|
|
3
|
+
from typing import List, Optional, Union, Dict, Any
|
|
3
4
|
|
|
4
5
|
import pandas as pd
|
|
5
6
|
from requests import Response
|
|
@@ -23,6 +24,8 @@ from fi.utils.constants import (
|
|
|
23
24
|
from fi.utils.errors import InvalidAuthError, NoDatasetFound
|
|
24
25
|
from fi.utils.routes import Routes
|
|
25
26
|
from fi.utils.utils import get_tempfile_path
|
|
27
|
+
from fi.evals.templates import EvalTemplate
|
|
28
|
+
from fi.evals.evaluator import EvalInfoResponseHandler
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
@@ -33,8 +36,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
33
36
|
def _parse_success(cls, response: Response) -> Union[DatasetConfig, DatasetTable]:
|
|
34
37
|
"""Parse successful response into DatasetResponse"""
|
|
35
38
|
data = response.json()
|
|
36
|
-
|
|
37
|
-
print("response", response.url)
|
|
39
|
+
|
|
38
40
|
if response.url.endswith(Routes.dataset_names.value):
|
|
39
41
|
datasets = data["result"]["datasets"]
|
|
40
42
|
if not datasets:
|
|
@@ -50,6 +52,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
|
50
52
|
)
|
|
51
53
|
elif Routes.dataset_table.value.split("/")[-2] in response.url:
|
|
52
54
|
id = response.url.split("/")[-3]
|
|
55
|
+
print(data)
|
|
53
56
|
columns = [
|
|
54
57
|
Column(
|
|
55
58
|
id=column["id"],
|
|
@@ -131,6 +134,7 @@ class DatasetClient(APIKeyAuth):
|
|
|
131
134
|
"""
|
|
132
135
|
|
|
133
136
|
_dataset_id_cache = {}
|
|
137
|
+
__static_attributes__ = ['get_dataset_config']
|
|
134
138
|
|
|
135
139
|
def __init__(
|
|
136
140
|
self,
|
|
@@ -226,6 +230,254 @@ class DatasetClient(APIKeyAuth):
|
|
|
226
230
|
|
|
227
231
|
self._add_rows(rows)
|
|
228
232
|
return self
|
|
233
|
+
|
|
234
|
+
def get_column_id(self, column_name: str)-> Optional[str]:
|
|
235
|
+
if not self.dataset_config or not self.dataset_config.id:
|
|
236
|
+
raise ValueError("No dataset configured")
|
|
237
|
+
|
|
238
|
+
url = f"{self._base_url}/{Routes.dataset_table.value.format(dataset_id=str(self.dataset_config.id))}"
|
|
239
|
+
dataset_table = self.request(
|
|
240
|
+
config=RequestConfig(
|
|
241
|
+
method=HttpMethod.POST,
|
|
242
|
+
url=url,
|
|
243
|
+
json={
|
|
244
|
+
"page_size": 1,
|
|
245
|
+
"current_page_index": 0
|
|
246
|
+
}
|
|
247
|
+
),
|
|
248
|
+
response_handler=DatasetResponseHandler
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
for column in dataset_table.columns:
|
|
252
|
+
if column.name == column_name:
|
|
253
|
+
return str(column.id)
|
|
254
|
+
return None
|
|
255
|
+
|
|
256
|
+
def add_run_prompt(
|
|
257
|
+
self,
|
|
258
|
+
name: str,
|
|
259
|
+
model: str,
|
|
260
|
+
messages: List[Dict[str, str]],
|
|
261
|
+
output_format: str = "string",
|
|
262
|
+
concurrency: int = 5,
|
|
263
|
+
max_tokens: int = 500,
|
|
264
|
+
temperature: float = 0.5,
|
|
265
|
+
presence_penalty: float = 1,
|
|
266
|
+
frequency_penalty: float = 1,
|
|
267
|
+
top_p: float = 1,
|
|
268
|
+
tools: Optional[List[Dict]] = None,
|
|
269
|
+
tool_choice: Optional[Any] = None,
|
|
270
|
+
response_format: Optional[Dict] = None
|
|
271
|
+
) -> "DatasetClient":
|
|
272
|
+
|
|
273
|
+
if not self.dataset_config:
|
|
274
|
+
raise ValueError("dataset_config must be set")
|
|
275
|
+
|
|
276
|
+
processed_messages = []
|
|
277
|
+
referenced_columns = set()
|
|
278
|
+
|
|
279
|
+
for msg in messages:
|
|
280
|
+
if "role" not in msg:
|
|
281
|
+
msg["role"] = "user"
|
|
282
|
+
|
|
283
|
+
if "content" in msg:
|
|
284
|
+
content = msg["content"]
|
|
285
|
+
column_refs = re.findall(r'\{\{(.*?)\}\}', content)
|
|
286
|
+
|
|
287
|
+
for col_name in column_refs:
|
|
288
|
+
col_id = self.get_column_id(col_name)
|
|
289
|
+
if not col_id:
|
|
290
|
+
raise ValueError(f"Referenced column '{col_name}' not found in dataset")
|
|
291
|
+
referenced_columns.add(col_name)
|
|
292
|
+
content = content.replace(f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}")
|
|
293
|
+
msg["content"] = content
|
|
294
|
+
|
|
295
|
+
processed_messages.append(msg)
|
|
296
|
+
|
|
297
|
+
config = {
|
|
298
|
+
"datasetId": str(self.dataset_config.id),
|
|
299
|
+
"name": name,
|
|
300
|
+
"config": {
|
|
301
|
+
"model": model,
|
|
302
|
+
"outputFormat": output_format,
|
|
303
|
+
"concurrency": concurrency,
|
|
304
|
+
"messages": processed_messages,
|
|
305
|
+
"maxTokens": max_tokens,
|
|
306
|
+
"temperature": temperature,
|
|
307
|
+
"presencePenalty": presence_penalty,
|
|
308
|
+
"frequencyPenalty": frequency_penalty,
|
|
309
|
+
"topP": top_p,
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
if tools:
|
|
314
|
+
config["config"]["tools"] = tools
|
|
315
|
+
if tool_choice is not None:
|
|
316
|
+
config["config"]["toolChoice"] = tool_choice
|
|
317
|
+
if response_format:
|
|
318
|
+
config["config"]["responseFormat"] = response_format
|
|
319
|
+
|
|
320
|
+
url = f"{self._base_url}/{Routes.dataset_add_run_prompt_column.value}"
|
|
321
|
+
self.request(
|
|
322
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, json=config),
|
|
323
|
+
response_handler=DatasetResponseHandler
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
return self
|
|
327
|
+
|
|
328
|
+
def add_evaluation(
|
|
329
|
+
self,
|
|
330
|
+
name: str,
|
|
331
|
+
eval_template: str,
|
|
332
|
+
input_column_name: Optional[str] = None,
|
|
333
|
+
output_column_name: Optional[str] = None,
|
|
334
|
+
context_column_name: Optional[str] = None,
|
|
335
|
+
expected_column_name: Optional[str] = None,
|
|
336
|
+
save_as_template: bool = False,
|
|
337
|
+
run: bool = True,
|
|
338
|
+
reason_column: bool = False,
|
|
339
|
+
config: Optional[Dict[str, Any]] = None
|
|
340
|
+
) -> "DatasetClient":
|
|
341
|
+
|
|
342
|
+
if not self.dataset_config or not self.dataset_config.id:
|
|
343
|
+
raise ValueError("No dataset configured")
|
|
344
|
+
|
|
345
|
+
template_classes = {
|
|
346
|
+
cls.__name__: cls
|
|
347
|
+
for cls in EvalTemplate.__subclasses__()
|
|
348
|
+
}
|
|
349
|
+
if eval_template not in template_classes:
|
|
350
|
+
raise ValueError(f"Unknown template name: {eval_template}")
|
|
351
|
+
|
|
352
|
+
eval_id = template_classes[eval_template].eval_id
|
|
353
|
+
|
|
354
|
+
url = f"{self._base_url}/sdk/api/v1/eval/{eval_id}/"
|
|
355
|
+
template_details = self.request(
|
|
356
|
+
config=RequestConfig(
|
|
357
|
+
method=HttpMethod.GET,
|
|
358
|
+
url=url
|
|
359
|
+
),
|
|
360
|
+
response_handler=EvalInfoResponseHandler
|
|
361
|
+
)
|
|
362
|
+
|
|
363
|
+
template_id = template_details["id"]
|
|
364
|
+
required_keys = template_details["config"]["required_keys"]
|
|
365
|
+
|
|
366
|
+
mapping = {}
|
|
367
|
+
|
|
368
|
+
if "input" in required_keys:
|
|
369
|
+
if not input_column_name:
|
|
370
|
+
raise ValueError(f"Input column required for {eval_template}")
|
|
371
|
+
input_column_id = self.get_column_id(input_column_name)
|
|
372
|
+
if not input_column_id:
|
|
373
|
+
raise ValueError(f"Column '{input_column_name}' not found in dataset")
|
|
374
|
+
mapping["input"] = input_column_id
|
|
375
|
+
|
|
376
|
+
if "output" in required_keys or "response" in required_keys:
|
|
377
|
+
if not output_column_name:
|
|
378
|
+
raise ValueError(f"Output column required for {eval_template}")
|
|
379
|
+
output_column_id = self.get_column_id(output_column_name)
|
|
380
|
+
if not output_column_id:
|
|
381
|
+
raise ValueError(f"Column '{output_column_name}' not found in dataset")
|
|
382
|
+
if "output" in required_keys:
|
|
383
|
+
mapping["output"] = output_column_id
|
|
384
|
+
else:
|
|
385
|
+
mapping["response"] = output_column_id
|
|
386
|
+
|
|
387
|
+
if "context" in required_keys:
|
|
388
|
+
if not context_column_name:
|
|
389
|
+
raise ValueError(f"Context column required for {eval_template}")
|
|
390
|
+
context_column_id = self.get_column_id(context_column_name)
|
|
391
|
+
if not context_column_id:
|
|
392
|
+
raise ValueError(f"Column '{context_column_name}' not found in dataset")
|
|
393
|
+
mapping["context"] = context_column_id
|
|
394
|
+
|
|
395
|
+
if "expected_response" in required_keys:
|
|
396
|
+
if not expected_column_name:
|
|
397
|
+
raise ValueError(f"Expected response column required for {eval_template}")
|
|
398
|
+
expected_column_id = self.get_column_id(expected_column_name)
|
|
399
|
+
if not expected_column_id:
|
|
400
|
+
raise ValueError(f"Column '{expected_column_name}' not found in dataset")
|
|
401
|
+
mapping["expected_response"] = expected_column_id
|
|
402
|
+
|
|
403
|
+
eval_config = {
|
|
404
|
+
"template_id": template_id,
|
|
405
|
+
"run": run,
|
|
406
|
+
"name": name,
|
|
407
|
+
"saveAsTemplate": save_as_template,
|
|
408
|
+
"config": {
|
|
409
|
+
"mapping": mapping,
|
|
410
|
+
"config": config or {},
|
|
411
|
+
"reasonColumn": reason_column
|
|
412
|
+
}
|
|
413
|
+
}
|
|
414
|
+
|
|
415
|
+
url = f"{self._base_url}/model-hub/develops/{str(self.dataset_config.id)}/add_user_eval/"
|
|
416
|
+
self.request(
|
|
417
|
+
config=RequestConfig(
|
|
418
|
+
method=HttpMethod.POST,
|
|
419
|
+
url=url,
|
|
420
|
+
json=eval_config
|
|
421
|
+
),
|
|
422
|
+
response_handler=DatasetResponseHandler
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
return self
|
|
426
|
+
|
|
427
|
+
def add_optimization(
|
|
428
|
+
self,
|
|
429
|
+
optimization_name: str,
|
|
430
|
+
prompt_column_name: str,
|
|
431
|
+
optimize_type: str = "PROMPT_TEMPLATE",
|
|
432
|
+
model_config: Optional[Dict[str, Any]] = None
|
|
433
|
+
) -> "DatasetClient":
|
|
434
|
+
|
|
435
|
+
if not self.dataset_config or not self.dataset_config.id:
|
|
436
|
+
raise ValueError("No dataset configured")
|
|
437
|
+
|
|
438
|
+
column_id = self.get_column_id(prompt_column_name)
|
|
439
|
+
if not column_id:
|
|
440
|
+
raise ValueError(f"Prompt Column '{prompt_column_name}' not found in dataset")
|
|
441
|
+
|
|
442
|
+
#requires mapping input for userEvalTemplateIds
|
|
443
|
+
metrics_url = f"{self._base_url}/model-hub/metrics/by-column/"
|
|
444
|
+
metrics_response = self.request(
|
|
445
|
+
config=RequestConfig(
|
|
446
|
+
method=HttpMethod.GET,
|
|
447
|
+
url=metrics_url,
|
|
448
|
+
params={"column_id": column_id}
|
|
449
|
+
),
|
|
450
|
+
response_handler=DatasetResponseHandler
|
|
451
|
+
)
|
|
452
|
+
|
|
453
|
+
eval_template_ids = []
|
|
454
|
+
for metric in metrics_response["result"]:
|
|
455
|
+
if metric.get("config", {}).get("mapping", {}).get("input") == column_id:
|
|
456
|
+
eval_template_ids.append(metric["id"])
|
|
457
|
+
|
|
458
|
+
if not eval_template_ids:
|
|
459
|
+
raise ValueError(f"No evaluation templates found for column '{prompt_column_name}'")
|
|
460
|
+
|
|
461
|
+
payload = {
|
|
462
|
+
"name": optimization_name,
|
|
463
|
+
"columnId": column_id,
|
|
464
|
+
"optimizeType": optimize_type,
|
|
465
|
+
"userEvalTemplateIds": eval_template_ids,
|
|
466
|
+
"datasetId": str(self.dataset_config.id),
|
|
467
|
+
"modelConfig": model_config or {}
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
opt_url = f"{self._base_url}/model-hub/optimisation/create/"
|
|
471
|
+
self.request(
|
|
472
|
+
config=RequestConfig(
|
|
473
|
+
method=HttpMethod.POST,
|
|
474
|
+
url=opt_url,
|
|
475
|
+
json=payload
|
|
476
|
+
),
|
|
477
|
+
response_handler=DatasetResponseHandler
|
|
478
|
+
)
|
|
479
|
+
|
|
480
|
+
return self
|
|
229
481
|
|
|
230
482
|
# Protected internal methods
|
|
231
483
|
def _fetch_dataset_config(self, dataset_name):
|
|
@@ -23,8 +23,12 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
|
|
|
23
23
|
def _handle_error(cls, response: Response) -> None:
|
|
24
24
|
if response.status_code == 400:
|
|
25
25
|
response.raise_for_status()
|
|
26
|
-
|
|
26
|
+
elif response.status_code == 403:
|
|
27
27
|
raise InvalidAuthError()
|
|
28
|
+
else:
|
|
29
|
+
raise Exception(
|
|
30
|
+
f"Error in evaluation: {response.status_code}, response: {response.text}"
|
|
31
|
+
)
|
|
28
32
|
|
|
29
33
|
@classmethod
|
|
30
34
|
def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
|
|
@@ -39,7 +43,7 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
|
|
|
39
43
|
"""
|
|
40
44
|
eval_results = []
|
|
41
45
|
|
|
42
|
-
for result in response.get("result",
|
|
46
|
+
for result in response.get("result", {}):
|
|
43
47
|
for evaluation in result.get("evaluations", []):
|
|
44
48
|
eval_results.append(
|
|
45
49
|
EvalResult(
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
from typing import Dict, Optional
|
|
2
|
+
|
|
3
|
+
from fi.api.auth import APIKeyAuth, ResponseHandler
|
|
4
|
+
from fi.api.types import HttpMethod, RequestConfig
|
|
5
|
+
from fi.prompt.types import ModelConfig, PromptTemplate
|
|
6
|
+
from fi.utils.errors import InvalidAuthError
|
|
7
|
+
from fi.utils.routes import Routes
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class PromptResponseHandler(ResponseHandler[Dict, PromptTemplate]):
|
|
11
|
+
"""Handles responses for prompt requests"""
|
|
12
|
+
|
|
13
|
+
@classmethod
|
|
14
|
+
def _parse_success(cls, response) -> Dict:
|
|
15
|
+
"""Handles responses for prompt requests"""
|
|
16
|
+
data = response.json()
|
|
17
|
+
# print(response.url, response.request.method, response.json())
|
|
18
|
+
|
|
19
|
+
# Handle search endpoint
|
|
20
|
+
if "search=" in response.url:
|
|
21
|
+
results = data.get("results", [])
|
|
22
|
+
name = response.url.split("search=")[1]
|
|
23
|
+
for item in results:
|
|
24
|
+
if item["name"] == name:
|
|
25
|
+
return item["id"]
|
|
26
|
+
raise ValueError(f"No template found with the given name: {name}")
|
|
27
|
+
|
|
28
|
+
# Handle GET template by ID endpoint
|
|
29
|
+
if response.request.method == HttpMethod.GET.value:
|
|
30
|
+
prompt_config = data.get("promptConfig", {})
|
|
31
|
+
# Transform response data to match PromptTemplate structure
|
|
32
|
+
model_config = ModelConfig(
|
|
33
|
+
model_name=prompt_config.get("configuration", {}).get(
|
|
34
|
+
"modelName", "gpt-4o-mini"
|
|
35
|
+
),
|
|
36
|
+
temperature=prompt_config.get("configuration", {}).get(
|
|
37
|
+
"temperature", 0.7
|
|
38
|
+
),
|
|
39
|
+
frequency_penalty=prompt_config.get("configuration", {}).get(
|
|
40
|
+
"frequencyPenalty", 0
|
|
41
|
+
),
|
|
42
|
+
presence_penalty=prompt_config.get("configuration", {}).get(
|
|
43
|
+
"presencePenalty", 0
|
|
44
|
+
),
|
|
45
|
+
max_tokens=prompt_config.get("configuration", {}).get(
|
|
46
|
+
"maxTokens", 1000
|
|
47
|
+
),
|
|
48
|
+
top_p=prompt_config.get("configuration", {}).get("topP", 1.0),
|
|
49
|
+
response_format=prompt_config.get("configuration", {}).get(
|
|
50
|
+
"responseFormat", None
|
|
51
|
+
),
|
|
52
|
+
tool_choice=prompt_config.get("configuration", {}).get(
|
|
53
|
+
"toolChoice", None
|
|
54
|
+
),
|
|
55
|
+
tools=prompt_config.get("configuration", {}).get("tools", None),
|
|
56
|
+
)
|
|
57
|
+
template_data = {
|
|
58
|
+
"id": data.get("id"),
|
|
59
|
+
"name": data.get("name"),
|
|
60
|
+
"description": data.get("description", ""),
|
|
61
|
+
"messages": prompt_config.get("messages", []),
|
|
62
|
+
"model_configuration": model_config,
|
|
63
|
+
"variable_names": data.get("variableNames", {}),
|
|
64
|
+
"version": data.get("version"),
|
|
65
|
+
"is_default": data.get("isDefault", True),
|
|
66
|
+
"evaluation_configs": data.get("evaluationConfigs", []),
|
|
67
|
+
"status": data.get("status"),
|
|
68
|
+
"error_message": data.get("errorMessage"),
|
|
69
|
+
}
|
|
70
|
+
return PromptTemplate(**template_data)
|
|
71
|
+
|
|
72
|
+
if response.request.method == HttpMethod.POST.value and response.url.endswith(
|
|
73
|
+
Routes.create_template.value
|
|
74
|
+
):
|
|
75
|
+
return data["result"]
|
|
76
|
+
|
|
77
|
+
# Return raw data for other endpoints
|
|
78
|
+
return data
|
|
79
|
+
|
|
80
|
+
@classmethod
|
|
81
|
+
def _handle_error(cls, response) -> None:
|
|
82
|
+
if response.status_code == 403:
|
|
83
|
+
raise InvalidAuthError()
|
|
84
|
+
else:
|
|
85
|
+
response.raise_for_status()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class PromptClient(APIKeyAuth):
|
|
89
|
+
_template_id_cache = {}
|
|
90
|
+
|
|
91
|
+
def __init__(
|
|
92
|
+
self,
|
|
93
|
+
template: Optional[PromptTemplate] = None,
|
|
94
|
+
fi_api_key: Optional[str] = None,
|
|
95
|
+
fi_secret_key: Optional[str] = None,
|
|
96
|
+
fi_base_url: Optional[str] = None,
|
|
97
|
+
**kwargs,
|
|
98
|
+
):
|
|
99
|
+
super().__init__(
|
|
100
|
+
fi_api_key=fi_api_key,
|
|
101
|
+
fi_secret_key=fi_secret_key,
|
|
102
|
+
fi_base_url=fi_base_url,
|
|
103
|
+
**kwargs,
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
if template and not template.id:
|
|
107
|
+
try:
|
|
108
|
+
self.template = self._fetch_template_by_name(template.name)
|
|
109
|
+
except Exception:
|
|
110
|
+
print(
|
|
111
|
+
"Warning: Template not found in the backend. Create a new template before running."
|
|
112
|
+
)
|
|
113
|
+
self.template = template
|
|
114
|
+
else:
|
|
115
|
+
self.template = template
|
|
116
|
+
if self.template:
|
|
117
|
+
print(
|
|
118
|
+
f"Current template: {self.template.name} does not exist in the backend. Please create it first before running." # noqa: E713
|
|
119
|
+
)
|
|
120
|
+
else:
|
|
121
|
+
print("No template provided. Please provide a template before running.")
|
|
122
|
+
|
|
123
|
+
if not self.template:
|
|
124
|
+
raise ValueError("No template configured")
|
|
125
|
+
|
|
126
|
+
def generate(self, requirements: str) -> "PromptClient":
|
|
127
|
+
"""Generate a prompt and return self for chaining"""
|
|
128
|
+
if not self.template:
|
|
129
|
+
raise ValueError("No template configured")
|
|
130
|
+
response = self.request(
|
|
131
|
+
config=RequestConfig(
|
|
132
|
+
method=HttpMethod.POST,
|
|
133
|
+
url=self._base_url + "/" + Routes.generate_prompt.value,
|
|
134
|
+
json={"statement": requirements},
|
|
135
|
+
),
|
|
136
|
+
response_handler=PromptResponseHandler,
|
|
137
|
+
)
|
|
138
|
+
self.template.messages[-1].content = response["result"]["prompt"]
|
|
139
|
+
return self
|
|
140
|
+
|
|
141
|
+
def improve(self, requirements: str) -> "PromptClient":
|
|
142
|
+
"""Improve prompt and return self for chaining"""
|
|
143
|
+
if not self.template:
|
|
144
|
+
raise ValueError("No template configured")
|
|
145
|
+
|
|
146
|
+
existing_prompt = (
|
|
147
|
+
self.template.messages[-1].content if self.template.messages else ""
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
improved_response = self.request(
|
|
151
|
+
config=RequestConfig(
|
|
152
|
+
method=HttpMethod.POST,
|
|
153
|
+
url=self._base_url + "/" + Routes.improve_prompt.value,
|
|
154
|
+
json={
|
|
155
|
+
"existing_prompt": existing_prompt,
|
|
156
|
+
"improvement_requirements": requirements,
|
|
157
|
+
},
|
|
158
|
+
),
|
|
159
|
+
response_handler=PromptResponseHandler,
|
|
160
|
+
)
|
|
161
|
+
self.template.messages[-1].content = improved_response["result"]["prompt"]
|
|
162
|
+
return self
|
|
163
|
+
|
|
164
|
+
def create(self) -> "PromptClient":
|
|
165
|
+
"""Create a draft prompt template and return self for chaining"""
|
|
166
|
+
if not self.template:
|
|
167
|
+
raise ValueError("template must be set")
|
|
168
|
+
|
|
169
|
+
if self.template.id:
|
|
170
|
+
print(
|
|
171
|
+
f"Template {self.template.name} already exists in the backend. Please use a different name to create a new template."
|
|
172
|
+
)
|
|
173
|
+
return self
|
|
174
|
+
|
|
175
|
+
method = HttpMethod.POST
|
|
176
|
+
url = self._base_url + "/" + Routes.create_template.value
|
|
177
|
+
messages = [message.model_dump() for message in self.template.messages]
|
|
178
|
+
json = {
|
|
179
|
+
"name": self.template.name,
|
|
180
|
+
"prompt_config": {
|
|
181
|
+
"messages": messages,
|
|
182
|
+
"configuration": self.template.model_configuration.model_dump(),
|
|
183
|
+
},
|
|
184
|
+
"variable_names": self.template.variable_names,
|
|
185
|
+
"description": self.template.description,
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
response = self.request(
|
|
189
|
+
config=RequestConfig(
|
|
190
|
+
method=method,
|
|
191
|
+
url=url,
|
|
192
|
+
json=json,
|
|
193
|
+
),
|
|
194
|
+
response_handler=PromptResponseHandler,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
self.template.id = response["id"]
|
|
198
|
+
self.template.name = response["name"]
|
|
199
|
+
|
|
200
|
+
return self
|
|
201
|
+
|
|
202
|
+
def run(
|
|
203
|
+
self,
|
|
204
|
+
variables: Optional[Dict] = None,
|
|
205
|
+
) -> Dict:
|
|
206
|
+
"""Run a prompt template and return the response"""
|
|
207
|
+
if not self.template:
|
|
208
|
+
raise ValueError("No template configured")
|
|
209
|
+
|
|
210
|
+
# validate variable names
|
|
211
|
+
if self.template.variable_names:
|
|
212
|
+
for var_name in self.template.variable_names:
|
|
213
|
+
if var_name not in variables:
|
|
214
|
+
raise ValueError(
|
|
215
|
+
f"Variable name {var_name} not found in variable_names" # noqa: E713
|
|
216
|
+
)
|
|
217
|
+
|
|
218
|
+
messages = [message.model_dump() for message in self.template.messages]
|
|
219
|
+
model_config = {
|
|
220
|
+
"model": self.template.model_configuration.model_name,
|
|
221
|
+
"temperature": self.template.model_configuration.temperature,
|
|
222
|
+
"max_tokens": self.template.model_configuration.max_tokens,
|
|
223
|
+
"top_p": self.template.model_configuration.top_p,
|
|
224
|
+
"frequency_penalty": self.template.model_configuration.frequency_penalty,
|
|
225
|
+
"presence_penalty": self.template.model_configuration.presence_penalty,
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
# Transform variable_names to match expected format
|
|
229
|
+
formatted_variables = {
|
|
230
|
+
k: [v] if not isinstance(v, list) else v
|
|
231
|
+
for k, v in (variables or {}).items()
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
response = self.request(
|
|
235
|
+
config=RequestConfig(
|
|
236
|
+
method=HttpMethod.POST,
|
|
237
|
+
url=self._base_url
|
|
238
|
+
+ "/"
|
|
239
|
+
+ Routes.run_template.value.format(template_id=self.template.id),
|
|
240
|
+
json={
|
|
241
|
+
"prompt_config": [
|
|
242
|
+
{"messages": messages, "configuration": model_config}
|
|
243
|
+
],
|
|
244
|
+
"variable_names": formatted_variables,
|
|
245
|
+
"evaluation_configs": self.template.evaluation_configs,
|
|
246
|
+
"name": self.template.name,
|
|
247
|
+
"is_run": True,
|
|
248
|
+
},
|
|
249
|
+
),
|
|
250
|
+
response_handler=PromptResponseHandler,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
return response
|
|
254
|
+
|
|
255
|
+
# Keep existing methods but update them to work with PromptTemplate
|
|
256
|
+
def _fetch_template_by_name(self, name: str) -> PromptTemplate:
|
|
257
|
+
"""Fetch template configuration by name"""
|
|
258
|
+
template_id = self._fetch_id_by_name(name)
|
|
259
|
+
return self._fetch_template_by_id(template_id)
|
|
260
|
+
|
|
261
|
+
def _fetch_template_by_id(self, template_id: str) -> PromptTemplate:
|
|
262
|
+
"""Fetch template configuration by ID"""
|
|
263
|
+
response = self.request(
|
|
264
|
+
config=RequestConfig(
|
|
265
|
+
method=HttpMethod.GET,
|
|
266
|
+
url=self._base_url
|
|
267
|
+
+ "/"
|
|
268
|
+
+ Routes.get_template_by_id.value.format(template_id=template_id),
|
|
269
|
+
),
|
|
270
|
+
response_handler=PromptResponseHandler,
|
|
271
|
+
)
|
|
272
|
+
return response
|
|
273
|
+
|
|
274
|
+
def _fetch_id_by_name(self, name: str) -> str:
|
|
275
|
+
"""Fetch template ID by name"""
|
|
276
|
+
response = self.request(
|
|
277
|
+
config=RequestConfig(
|
|
278
|
+
method=HttpMethod.GET,
|
|
279
|
+
url=self._base_url + "/" + Routes.get_template_id_by_name.value,
|
|
280
|
+
params={"search": name},
|
|
281
|
+
),
|
|
282
|
+
response_handler=PromptResponseHandler,
|
|
283
|
+
)
|
|
284
|
+
return response
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import uuid
|
|
2
|
+
from typing import Any, Dict, List, Optional
|
|
3
|
+
|
|
4
|
+
from pydantic import BaseModel, Field, field_validator
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
# Base class for all messages
|
|
8
|
+
class MessageBase(BaseModel):
|
|
9
|
+
role: str
|
|
10
|
+
content: str
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
# UserMessage class with variable handling
|
|
14
|
+
class UserMessage(MessageBase):
|
|
15
|
+
role: str = "user"
|
|
16
|
+
content: str
|
|
17
|
+
variable_names: Optional[List[str]] = []
|
|
18
|
+
|
|
19
|
+
@field_validator("content")
|
|
20
|
+
def validate_content(cls, v: str) -> str:
|
|
21
|
+
"""Validate user message content."""
|
|
22
|
+
if not v.strip():
|
|
23
|
+
raise ValueError("Message content cannot be empty.")
|
|
24
|
+
return v.strip()
|
|
25
|
+
|
|
26
|
+
def model_post_init(self, __context: Any) -> None:
|
|
27
|
+
"""Initialize variable names by inferring them from the content if not already set."""
|
|
28
|
+
if self.variable_names == []:
|
|
29
|
+
# Extract variable names from content by looking for {{name}} patterns
|
|
30
|
+
content = self.content
|
|
31
|
+
var_names = []
|
|
32
|
+
parts = content.split("{{")
|
|
33
|
+
for part in parts[1:]: # Skip first part before any {{
|
|
34
|
+
if "}}" in part:
|
|
35
|
+
var_name = part.split("}}")[0].strip()
|
|
36
|
+
var_names.append(var_name)
|
|
37
|
+
self.variable_names = var_names
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
# SystemMessage class
|
|
41
|
+
class SystemMessage(MessageBase):
|
|
42
|
+
role: str = "system"
|
|
43
|
+
content: str
|
|
44
|
+
|
|
45
|
+
@field_validator("content")
|
|
46
|
+
def validate_content(cls, v: str) -> str:
|
|
47
|
+
"""Validate system message content."""
|
|
48
|
+
if not v.strip():
|
|
49
|
+
raise ValueError("Message content cannot be empty.")
|
|
50
|
+
return v.strip()
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# AssistantMessage class
|
|
54
|
+
class AssistantMessage(MessageBase):
|
|
55
|
+
role: str = "assistant"
|
|
56
|
+
content: str
|
|
57
|
+
|
|
58
|
+
@field_validator("content")
|
|
59
|
+
def validate_content(cls, v: str) -> str:
|
|
60
|
+
"""Validate assistant message content."""
|
|
61
|
+
if not v.strip():
|
|
62
|
+
raise ValueError("Message content cannot be empty.")
|
|
63
|
+
return v.strip()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class ModelConfig(BaseModel):
|
|
67
|
+
model_name: str = "gpt-4o-mini"
|
|
68
|
+
temperature: Optional[float] = Field(default=0.7, ge=0, le=2)
|
|
69
|
+
frequency_penalty: Optional[float] = Field(default=0, ge=-2, le=2)
|
|
70
|
+
presence_penalty: Optional[float] = Field(default=0, ge=-2, le=2)
|
|
71
|
+
max_tokens: Optional[int] = Field(default=1000, gt=0)
|
|
72
|
+
top_p: Optional[float] = Field(default=1.0, ge=0, le=1)
|
|
73
|
+
response_format: Optional[dict] = None
|
|
74
|
+
tool_choice: Optional[str] = None
|
|
75
|
+
tools: Optional[List[dict]] = None
|
|
76
|
+
|
|
77
|
+
class Config:
|
|
78
|
+
frozen = True
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
class PromptTemplate(BaseModel):
|
|
82
|
+
id: Optional[uuid.UUID] = None
|
|
83
|
+
name: str
|
|
84
|
+
messages: Optional[List[MessageBase]] = []
|
|
85
|
+
model_configuration: Optional[ModelConfig] = ModelConfig()
|
|
86
|
+
variable_names: Optional[Dict[str, List[str]]] = {}
|
|
87
|
+
description: Optional[str] = None
|
|
88
|
+
version: Optional[str] = None
|
|
89
|
+
is_default: bool = True
|
|
90
|
+
evaluation_configs: Optional[List[Dict]] = None
|
|
91
|
+
status: Optional[str] = None
|
|
92
|
+
error_message: Optional[str] = None
|
|
93
|
+
|
|
94
|
+
@field_validator("name")
|
|
95
|
+
def validate_name(cls, v: str) -> str:
|
|
96
|
+
"""Ensure name is valid."""
|
|
97
|
+
if not v.strip():
|
|
98
|
+
raise ValueError("Prompt name cannot be empty.")
|
|
99
|
+
if len(v) > 255:
|
|
100
|
+
raise ValueError("Prompt name is too long (max 255 characters).")
|
|
101
|
+
return v.strip()
|
|
102
|
+
|
|
103
|
+
def model_post_init(self, __context: Any) -> None:
|
|
104
|
+
"""Initialize variable names by inferring them from the content if not already set."""
|
|
105
|
+
if self.variable_names == {}:
|
|
106
|
+
# extract variable names from messages
|
|
107
|
+
for message in self.messages:
|
|
108
|
+
if isinstance(message, UserMessage):
|
|
109
|
+
for var_name in message.variable_names:
|
|
110
|
+
if var_name not in self.variable_names:
|
|
111
|
+
self.variable_names[var_name] = []
|
|
@@ -21,6 +21,20 @@ class Routes(str, Enum):
|
|
|
21
21
|
dataset_delete = "model-hub/develops/delete_dataset/"
|
|
22
22
|
dataset_add_rows = "model-hub/develops/{dataset_id}/add_rows/"
|
|
23
23
|
dataset_add_columns = "model-hub/develops/{dataset_id}/add_columns/"
|
|
24
|
+
|
|
25
|
+
# prompt
|
|
26
|
+
generate_prompt = "model-hub/prompt-templates/generate-prompt/"
|
|
27
|
+
improve_prompt = "model-hub/prompt-templates/improve-prompt/"
|
|
28
|
+
run_template = "model-hub/prompt-templates/{template_id}/run_template/"
|
|
29
|
+
create_template = "model-hub/prompt-templates/create-draft/"
|
|
30
|
+
delete_template = "model-hub/prompt-templates/{template_id}"
|
|
31
|
+
get_template_by_id = "model-hub/prompt-templates/{template_id}"
|
|
32
|
+
get_template_id_by_name = "model-hub/prompt-templates/"
|
|
33
|
+
|
|
24
34
|
# model provider
|
|
25
35
|
model_hub_api_keys = "model-hub/api-keys/"
|
|
26
36
|
model_hub_default_provider = "model-hub/default-provider/"
|
|
37
|
+
|
|
38
|
+
dataset_add_run_prompt_column = "model-hub/develops/add_run_prompt_column/"
|
|
39
|
+
dataset_add_evaluation = "model-hub/develops/{dataset_id}/add_user_eval/"
|
|
40
|
+
dataset_optimization_create = "model-hub/optimisation/create/"
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|