futureagi 0.4.0__tar.gz → 0.4.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. {futureagi-0.4.0 → futureagi-0.4.2}/PKG-INFO +1 -1
  2. {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/auth.py +1 -1
  3. {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/client.py +255 -3
  4. {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/evaluator.py +6 -2
  5. {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/types.py +2 -2
  6. futureagi-0.4.2/fi/prompt/__init__.py +3 -0
  7. futureagi-0.4.2/fi/prompt/client.py +284 -0
  8. futureagi-0.4.2/fi/prompt/types.py +111 -0
  9. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/routes.py +14 -0
  10. {futureagi-0.4.0 → futureagi-0.4.2}/pyproject.toml +1 -1
  11. {futureagi-0.4.0 → futureagi-0.4.2}/LICENSE.md +0 -0
  12. {futureagi-0.4.0 → futureagi-0.4.2}/README.md +0 -0
  13. {futureagi-0.4.0 → futureagi-0.4.2}/fi/__init__.py +0 -0
  14. {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/__init__.py +0 -0
  15. {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/apikeys.py +0 -0
  16. {futureagi-0.4.0 → futureagi-0.4.2}/fi/api/types.py +0 -0
  17. {futureagi-0.4.0 → futureagi-0.4.2}/fi/client.py +0 -0
  18. {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/__init__.py +0 -0
  19. {futureagi-0.4.0 → futureagi-0.4.2}/fi/datasets/types.py +0 -0
  20. {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/__init__.py +0 -0
  21. {futureagi-0.4.0 → futureagi-0.4.2}/fi/evals/templates.py +0 -0
  22. {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/__init__.py +0 -0
  23. {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/providers/__init__.py +0 -0
  24. {futureagi-0.4.0 → futureagi-0.4.2}/fi/integrations/providers/types.py +0 -0
  25. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/__init__.py +0 -0
  26. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/conversational_test_case.py +0 -0
  27. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/general.py +0 -0
  28. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/llm_test_case.py +0 -0
  29. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/mllm_test_case.py +0 -0
  30. {futureagi-0.4.0 → futureagi-0.4.2}/fi/testcases/utils.py +0 -0
  31. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/__init__.py +0 -0
  32. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/constants.py +0 -0
  33. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/errors.py +0 -0
  34. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/executor.py +0 -0
  35. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/logging.py +0 -0
  36. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/types.py +0 -0
  37. {futureagi-0.4.0 → futureagi-0.4.2}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: futureagi
3
- Version: 0.4.0
3
+ Version: 0.4.2
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -29,7 +29,7 @@ class ResponseHandler(Generic[T, U], ABC):
29
29
  @classmethod
30
30
  def parse(cls, response: Response) -> Union[T, U]:
31
31
  """Parse the response into the expected type"""
32
- if not response.ok:
32
+ if not response.ok or response.status_code != 200:
33
33
  cls._handle_error(response)
34
34
  return cls._parse_success(response)
35
35
 
@@ -1,5 +1,6 @@
1
1
  import os
2
- from typing import List, Optional, Union
2
+ import re
3
+ from typing import List, Optional, Union, Dict, Any
3
4
 
4
5
  import pandas as pd
5
6
  from requests import Response
@@ -23,6 +24,8 @@ from fi.utils.constants import (
23
24
  from fi.utils.errors import InvalidAuthError, NoDatasetFound
24
25
  from fi.utils.routes import Routes
25
26
  from fi.utils.utils import get_tempfile_path
27
+ from fi.evals.templates import EvalTemplate
28
+ from fi.evals.evaluator import EvalInfoResponseHandler
26
29
 
27
30
 
28
31
  class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
@@ -33,8 +36,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
33
36
  def _parse_success(cls, response: Response) -> Union[DatasetConfig, DatasetTable]:
34
37
  """Parse successful response into DatasetResponse"""
35
38
  data = response.json()
36
- print("data", data)
37
- print("response", response.url)
39
+
38
40
  if response.url.endswith(Routes.dataset_names.value):
39
41
  datasets = data["result"]["datasets"]
40
42
  if not datasets:
@@ -50,6 +52,7 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
50
52
  )
51
53
  elif Routes.dataset_table.value.split("/")[-2] in response.url:
52
54
  id = response.url.split("/")[-3]
55
+ print(data)
53
56
  columns = [
54
57
  Column(
55
58
  id=column["id"],
@@ -131,6 +134,7 @@ class DatasetClient(APIKeyAuth):
131
134
  """
132
135
 
133
136
  _dataset_id_cache = {}
137
+ __static_attributes__ = ['get_dataset_config']
134
138
 
135
139
  def __init__(
136
140
  self,
@@ -226,6 +230,254 @@ class DatasetClient(APIKeyAuth):
226
230
 
227
231
  self._add_rows(rows)
228
232
  return self
233
+
234
+ def get_column_id(self, column_name: str)-> Optional[str]:
235
+ if not self.dataset_config or not self.dataset_config.id:
236
+ raise ValueError("No dataset configured")
237
+
238
+ url = f"{self._base_url}/{Routes.dataset_table.value.format(dataset_id=str(self.dataset_config.id))}"
239
+ dataset_table = self.request(
240
+ config=RequestConfig(
241
+ method=HttpMethod.POST,
242
+ url=url,
243
+ json={
244
+ "page_size": 1,
245
+ "current_page_index": 0
246
+ }
247
+ ),
248
+ response_handler=DatasetResponseHandler
249
+ )
250
+
251
+ for column in dataset_table.columns:
252
+ if column.name == column_name:
253
+ return str(column.id)
254
+ return None
255
+
256
+ def add_run_prompt(
257
+ self,
258
+ name: str,
259
+ model: str,
260
+ messages: List[Dict[str, str]],
261
+ output_format: str = "string",
262
+ concurrency: int = 5,
263
+ max_tokens: int = 500,
264
+ temperature: float = 0.5,
265
+ presence_penalty: float = 1,
266
+ frequency_penalty: float = 1,
267
+ top_p: float = 1,
268
+ tools: Optional[List[Dict]] = None,
269
+ tool_choice: Optional[Any] = None,
270
+ response_format: Optional[Dict] = None
271
+ ) -> "DatasetClient":
272
+
273
+ if not self.dataset_config:
274
+ raise ValueError("dataset_config must be set")
275
+
276
+ processed_messages = []
277
+ referenced_columns = set()
278
+
279
+ for msg in messages:
280
+ if "role" not in msg:
281
+ msg["role"] = "user"
282
+
283
+ if "content" in msg:
284
+ content = msg["content"]
285
+ column_refs = re.findall(r'\{\{(.*?)\}\}', content)
286
+
287
+ for col_name in column_refs:
288
+ col_id = self.get_column_id(col_name)
289
+ if not col_id:
290
+ raise ValueError(f"Referenced column '{col_name}' not found in dataset")
291
+ referenced_columns.add(col_name)
292
+ content = content.replace(f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}")
293
+ msg["content"] = content
294
+
295
+ processed_messages.append(msg)
296
+
297
+ config = {
298
+ "datasetId": str(self.dataset_config.id),
299
+ "name": name,
300
+ "config": {
301
+ "model": model,
302
+ "outputFormat": output_format,
303
+ "concurrency": concurrency,
304
+ "messages": processed_messages,
305
+ "maxTokens": max_tokens,
306
+ "temperature": temperature,
307
+ "presencePenalty": presence_penalty,
308
+ "frequencyPenalty": frequency_penalty,
309
+ "topP": top_p,
310
+ }
311
+ }
312
+
313
+ if tools:
314
+ config["config"]["tools"] = tools
315
+ if tool_choice is not None:
316
+ config["config"]["toolChoice"] = tool_choice
317
+ if response_format:
318
+ config["config"]["responseFormat"] = response_format
319
+
320
+ url = f"{self._base_url}/{Routes.dataset_add_run_prompt_column.value}"
321
+ self.request(
322
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=config),
323
+ response_handler=DatasetResponseHandler
324
+ )
325
+
326
+ return self
327
+
328
+ def add_evaluation(
329
+ self,
330
+ name: str,
331
+ eval_template: str,
332
+ input_column_name: Optional[str] = None,
333
+ output_column_name: Optional[str] = None,
334
+ context_column_name: Optional[str] = None,
335
+ expected_column_name: Optional[str] = None,
336
+ save_as_template: bool = False,
337
+ run: bool = True,
338
+ reason_column: bool = False,
339
+ config: Optional[Dict[str, Any]] = None
340
+ ) -> "DatasetClient":
341
+
342
+ if not self.dataset_config or not self.dataset_config.id:
343
+ raise ValueError("No dataset configured")
344
+
345
+ template_classes = {
346
+ cls.__name__: cls
347
+ for cls in EvalTemplate.__subclasses__()
348
+ }
349
+ if eval_template not in template_classes:
350
+ raise ValueError(f"Unknown template name: {eval_template}")
351
+
352
+ eval_id = template_classes[eval_template].eval_id
353
+
354
+ url = f"{self._base_url}/sdk/api/v1/eval/{eval_id}/"
355
+ template_details = self.request(
356
+ config=RequestConfig(
357
+ method=HttpMethod.GET,
358
+ url=url
359
+ ),
360
+ response_handler=EvalInfoResponseHandler
361
+ )
362
+
363
+ template_id = template_details["id"]
364
+ required_keys = template_details["config"]["required_keys"]
365
+
366
+ mapping = {}
367
+
368
+ if "input" in required_keys:
369
+ if not input_column_name:
370
+ raise ValueError(f"Input column required for {eval_template}")
371
+ input_column_id = self.get_column_id(input_column_name)
372
+ if not input_column_id:
373
+ raise ValueError(f"Column '{input_column_name}' not found in dataset")
374
+ mapping["input"] = input_column_id
375
+
376
+ if "output" in required_keys or "response" in required_keys:
377
+ if not output_column_name:
378
+ raise ValueError(f"Output column required for {eval_template}")
379
+ output_column_id = self.get_column_id(output_column_name)
380
+ if not output_column_id:
381
+ raise ValueError(f"Column '{output_column_name}' not found in dataset")
382
+ if "output" in required_keys:
383
+ mapping["output"] = output_column_id
384
+ else:
385
+ mapping["response"] = output_column_id
386
+
387
+ if "context" in required_keys:
388
+ if not context_column_name:
389
+ raise ValueError(f"Context column required for {eval_template}")
390
+ context_column_id = self.get_column_id(context_column_name)
391
+ if not context_column_id:
392
+ raise ValueError(f"Column '{context_column_name}' not found in dataset")
393
+ mapping["context"] = context_column_id
394
+
395
+ if "expected_response" in required_keys:
396
+ if not expected_column_name:
397
+ raise ValueError(f"Expected response column required for {eval_template}")
398
+ expected_column_id = self.get_column_id(expected_column_name)
399
+ if not expected_column_id:
400
+ raise ValueError(f"Column '{expected_column_name}' not found in dataset")
401
+ mapping["expected_response"] = expected_column_id
402
+
403
+ eval_config = {
404
+ "template_id": template_id,
405
+ "run": run,
406
+ "name": name,
407
+ "saveAsTemplate": save_as_template,
408
+ "config": {
409
+ "mapping": mapping,
410
+ "config": config or {},
411
+ "reasonColumn": reason_column
412
+ }
413
+ }
414
+
415
+ url = f"{self._base_url}/model-hub/develops/{str(self.dataset_config.id)}/add_user_eval/"
416
+ self.request(
417
+ config=RequestConfig(
418
+ method=HttpMethod.POST,
419
+ url=url,
420
+ json=eval_config
421
+ ),
422
+ response_handler=DatasetResponseHandler
423
+ )
424
+
425
+ return self
426
+
427
+ def add_optimization(
428
+ self,
429
+ optimization_name: str,
430
+ prompt_column_name: str,
431
+ optimize_type: str = "PROMPT_TEMPLATE",
432
+ model_config: Optional[Dict[str, Any]] = None
433
+ ) -> "DatasetClient":
434
+
435
+ if not self.dataset_config or not self.dataset_config.id:
436
+ raise ValueError("No dataset configured")
437
+
438
+ column_id = self.get_column_id(prompt_column_name)
439
+ if not column_id:
440
+ raise ValueError(f"Prompt Column '{prompt_column_name}' not found in dataset")
441
+
442
+ #requires mapping input for userEvalTemplateIds
443
+ metrics_url = f"{self._base_url}/model-hub/metrics/by-column/"
444
+ metrics_response = self.request(
445
+ config=RequestConfig(
446
+ method=HttpMethod.GET,
447
+ url=metrics_url,
448
+ params={"column_id": column_id}
449
+ ),
450
+ response_handler=DatasetResponseHandler
451
+ )
452
+
453
+ eval_template_ids = []
454
+ for metric in metrics_response["result"]:
455
+ if metric.get("config", {}).get("mapping", {}).get("input") == column_id:
456
+ eval_template_ids.append(metric["id"])
457
+
458
+ if not eval_template_ids:
459
+ raise ValueError(f"No evaluation templates found for column '{prompt_column_name}'")
460
+
461
+ payload = {
462
+ "name": optimization_name,
463
+ "columnId": column_id,
464
+ "optimizeType": optimize_type,
465
+ "userEvalTemplateIds": eval_template_ids,
466
+ "datasetId": str(self.dataset_config.id),
467
+ "modelConfig": model_config or {}
468
+ }
469
+
470
+ opt_url = f"{self._base_url}/model-hub/optimisation/create/"
471
+ self.request(
472
+ config=RequestConfig(
473
+ method=HttpMethod.POST,
474
+ url=opt_url,
475
+ json=payload
476
+ ),
477
+ response_handler=DatasetResponseHandler
478
+ )
479
+
480
+ return self
229
481
 
230
482
  # Protected internal methods
231
483
  def _fetch_dataset_config(self, dataset_name):
@@ -23,8 +23,12 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
23
23
  def _handle_error(cls, response: Response) -> None:
24
24
  if response.status_code == 400:
25
25
  response.raise_for_status()
26
- if response.status_code == 403:
26
+ elif response.status_code == 403:
27
27
  raise InvalidAuthError()
28
+ else:
29
+ raise Exception(
30
+ f"Error in evaluation: {response.status_code}, response: {response.text}"
31
+ )
28
32
 
29
33
  @classmethod
30
34
  def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
@@ -39,7 +43,7 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
39
43
  """
40
44
  eval_results = []
41
45
 
42
- for result in response.get("result", []):
46
+ for result in response.get("result", {}):
43
47
  for evaluation in result.get("evaluations", []):
44
48
  eval_results.append(
45
49
  EvalResult(
@@ -51,8 +51,8 @@ class EvalResultMetric(BaseModel):
51
51
  Represents the LLM evaluation result metric.
52
52
  """
53
53
 
54
- id: str
55
- value: float
54
+ id: Union[str, int, float]
55
+ value: Union[str, int, float, List[Any]]
56
56
 
57
57
 
58
58
  class EvalResult(BaseModel):
@@ -0,0 +1,3 @@
1
+ from .client import PromptClient
2
+
3
+ __all__ = ["PromptClient"]
@@ -0,0 +1,284 @@
1
+ from typing import Dict, Optional
2
+
3
+ from fi.api.auth import APIKeyAuth, ResponseHandler
4
+ from fi.api.types import HttpMethod, RequestConfig
5
+ from fi.prompt.types import ModelConfig, PromptTemplate
6
+ from fi.utils.errors import InvalidAuthError
7
+ from fi.utils.routes import Routes
8
+
9
+
10
+ class PromptResponseHandler(ResponseHandler[Dict, PromptTemplate]):
11
+ """Handles responses for prompt requests"""
12
+
13
+ @classmethod
14
+ def _parse_success(cls, response) -> Dict:
15
+ """Handles responses for prompt requests"""
16
+ data = response.json()
17
+ # print(response.url, response.request.method, response.json())
18
+
19
+ # Handle search endpoint
20
+ if "search=" in response.url:
21
+ results = data.get("results", [])
22
+ name = response.url.split("search=")[1]
23
+ for item in results:
24
+ if item["name"] == name:
25
+ return item["id"]
26
+ raise ValueError(f"No template found with the given name: {name}")
27
+
28
+ # Handle GET template by ID endpoint
29
+ if response.request.method == HttpMethod.GET.value:
30
+ prompt_config = data.get("promptConfig", {})
31
+ # Transform response data to match PromptTemplate structure
32
+ model_config = ModelConfig(
33
+ model_name=prompt_config.get("configuration", {}).get(
34
+ "modelName", "gpt-4o-mini"
35
+ ),
36
+ temperature=prompt_config.get("configuration", {}).get(
37
+ "temperature", 0.7
38
+ ),
39
+ frequency_penalty=prompt_config.get("configuration", {}).get(
40
+ "frequencyPenalty", 0
41
+ ),
42
+ presence_penalty=prompt_config.get("configuration", {}).get(
43
+ "presencePenalty", 0
44
+ ),
45
+ max_tokens=prompt_config.get("configuration", {}).get(
46
+ "maxTokens", 1000
47
+ ),
48
+ top_p=prompt_config.get("configuration", {}).get("topP", 1.0),
49
+ response_format=prompt_config.get("configuration", {}).get(
50
+ "responseFormat", None
51
+ ),
52
+ tool_choice=prompt_config.get("configuration", {}).get(
53
+ "toolChoice", None
54
+ ),
55
+ tools=prompt_config.get("configuration", {}).get("tools", None),
56
+ )
57
+ template_data = {
58
+ "id": data.get("id"),
59
+ "name": data.get("name"),
60
+ "description": data.get("description", ""),
61
+ "messages": prompt_config.get("messages", []),
62
+ "model_configuration": model_config,
63
+ "variable_names": data.get("variableNames", {}),
64
+ "version": data.get("version"),
65
+ "is_default": data.get("isDefault", True),
66
+ "evaluation_configs": data.get("evaluationConfigs", []),
67
+ "status": data.get("status"),
68
+ "error_message": data.get("errorMessage"),
69
+ }
70
+ return PromptTemplate(**template_data)
71
+
72
+ if response.request.method == HttpMethod.POST.value and response.url.endswith(
73
+ Routes.create_template.value
74
+ ):
75
+ return data["result"]
76
+
77
+ # Return raw data for other endpoints
78
+ return data
79
+
80
+ @classmethod
81
+ def _handle_error(cls, response) -> None:
82
+ if response.status_code == 403:
83
+ raise InvalidAuthError()
84
+ else:
85
+ response.raise_for_status()
86
+
87
+
88
+ class PromptClient(APIKeyAuth):
89
+ _template_id_cache = {}
90
+
91
+ def __init__(
92
+ self,
93
+ template: Optional[PromptTemplate] = None,
94
+ fi_api_key: Optional[str] = None,
95
+ fi_secret_key: Optional[str] = None,
96
+ fi_base_url: Optional[str] = None,
97
+ **kwargs,
98
+ ):
99
+ super().__init__(
100
+ fi_api_key=fi_api_key,
101
+ fi_secret_key=fi_secret_key,
102
+ fi_base_url=fi_base_url,
103
+ **kwargs,
104
+ )
105
+
106
+ if template and not template.id:
107
+ try:
108
+ self.template = self._fetch_template_by_name(template.name)
109
+ except Exception:
110
+ print(
111
+ "Warning: Template not found in the backend. Create a new template before running."
112
+ )
113
+ self.template = template
114
+ else:
115
+ self.template = template
116
+ if self.template:
117
+ print(
118
+ f"Current template: {self.template.name} does not exist in the backend. Please create it first before running." # noqa: E713
119
+ )
120
+ else:
121
+ print("No template provided. Please provide a template before running.")
122
+
123
+ if not self.template:
124
+ raise ValueError("No template configured")
125
+
126
+ def generate(self, requirements: str) -> "PromptClient":
127
+ """Generate a prompt and return self for chaining"""
128
+ if not self.template:
129
+ raise ValueError("No template configured")
130
+ response = self.request(
131
+ config=RequestConfig(
132
+ method=HttpMethod.POST,
133
+ url=self._base_url + "/" + Routes.generate_prompt.value,
134
+ json={"statement": requirements},
135
+ ),
136
+ response_handler=PromptResponseHandler,
137
+ )
138
+ self.template.messages[-1].content = response["result"]["prompt"]
139
+ return self
140
+
141
+ def improve(self, requirements: str) -> "PromptClient":
142
+ """Improve prompt and return self for chaining"""
143
+ if not self.template:
144
+ raise ValueError("No template configured")
145
+
146
+ existing_prompt = (
147
+ self.template.messages[-1].content if self.template.messages else ""
148
+ )
149
+
150
+ improved_response = self.request(
151
+ config=RequestConfig(
152
+ method=HttpMethod.POST,
153
+ url=self._base_url + "/" + Routes.improve_prompt.value,
154
+ json={
155
+ "existing_prompt": existing_prompt,
156
+ "improvement_requirements": requirements,
157
+ },
158
+ ),
159
+ response_handler=PromptResponseHandler,
160
+ )
161
+ self.template.messages[-1].content = improved_response["result"]["prompt"]
162
+ return self
163
+
164
+ def create(self) -> "PromptClient":
165
+ """Create a draft prompt template and return self for chaining"""
166
+ if not self.template:
167
+ raise ValueError("template must be set")
168
+
169
+ if self.template.id:
170
+ print(
171
+ f"Template {self.template.name} already exists in the backend. Please use a different name to create a new template."
172
+ )
173
+ return self
174
+
175
+ method = HttpMethod.POST
176
+ url = self._base_url + "/" + Routes.create_template.value
177
+ messages = [message.model_dump() for message in self.template.messages]
178
+ json = {
179
+ "name": self.template.name,
180
+ "prompt_config": {
181
+ "messages": messages,
182
+ "configuration": self.template.model_configuration.model_dump(),
183
+ },
184
+ "variable_names": self.template.variable_names,
185
+ "description": self.template.description,
186
+ }
187
+
188
+ response = self.request(
189
+ config=RequestConfig(
190
+ method=method,
191
+ url=url,
192
+ json=json,
193
+ ),
194
+ response_handler=PromptResponseHandler,
195
+ )
196
+
197
+ self.template.id = response["id"]
198
+ self.template.name = response["name"]
199
+
200
+ return self
201
+
202
+ def run(
203
+ self,
204
+ variables: Optional[Dict] = None,
205
+ ) -> Dict:
206
+ """Run a prompt template and return the response"""
207
+ if not self.template:
208
+ raise ValueError("No template configured")
209
+
210
+ # validate variable names
211
+ if self.template.variable_names:
212
+ for var_name in self.template.variable_names:
213
+ if var_name not in variables:
214
+ raise ValueError(
215
+ f"Variable name {var_name} not found in variable_names" # noqa: E713
216
+ )
217
+
218
+ messages = [message.model_dump() for message in self.template.messages]
219
+ model_config = {
220
+ "model": self.template.model_configuration.model_name,
221
+ "temperature": self.template.model_configuration.temperature,
222
+ "max_tokens": self.template.model_configuration.max_tokens,
223
+ "top_p": self.template.model_configuration.top_p,
224
+ "frequency_penalty": self.template.model_configuration.frequency_penalty,
225
+ "presence_penalty": self.template.model_configuration.presence_penalty,
226
+ }
227
+
228
+ # Transform variable_names to match expected format
229
+ formatted_variables = {
230
+ k: [v] if not isinstance(v, list) else v
231
+ for k, v in (variables or {}).items()
232
+ }
233
+
234
+ response = self.request(
235
+ config=RequestConfig(
236
+ method=HttpMethod.POST,
237
+ url=self._base_url
238
+ + "/"
239
+ + Routes.run_template.value.format(template_id=self.template.id),
240
+ json={
241
+ "prompt_config": [
242
+ {"messages": messages, "configuration": model_config}
243
+ ],
244
+ "variable_names": formatted_variables,
245
+ "evaluation_configs": self.template.evaluation_configs,
246
+ "name": self.template.name,
247
+ "is_run": True,
248
+ },
249
+ ),
250
+ response_handler=PromptResponseHandler,
251
+ )
252
+
253
+ return response
254
+
255
+ # Keep existing methods but update them to work with PromptTemplate
256
+ def _fetch_template_by_name(self, name: str) -> PromptTemplate:
257
+ """Fetch template configuration by name"""
258
+ template_id = self._fetch_id_by_name(name)
259
+ return self._fetch_template_by_id(template_id)
260
+
261
+ def _fetch_template_by_id(self, template_id: str) -> PromptTemplate:
262
+ """Fetch template configuration by ID"""
263
+ response = self.request(
264
+ config=RequestConfig(
265
+ method=HttpMethod.GET,
266
+ url=self._base_url
267
+ + "/"
268
+ + Routes.get_template_by_id.value.format(template_id=template_id),
269
+ ),
270
+ response_handler=PromptResponseHandler,
271
+ )
272
+ return response
273
+
274
+ def _fetch_id_by_name(self, name: str) -> str:
275
+ """Fetch template ID by name"""
276
+ response = self.request(
277
+ config=RequestConfig(
278
+ method=HttpMethod.GET,
279
+ url=self._base_url + "/" + Routes.get_template_id_by_name.value,
280
+ params={"search": name},
281
+ ),
282
+ response_handler=PromptResponseHandler,
283
+ )
284
+ return response
@@ -0,0 +1,111 @@
1
+ import uuid
2
+ from typing import Any, Dict, List, Optional
3
+
4
+ from pydantic import BaseModel, Field, field_validator
5
+
6
+
7
+ # Base class for all messages
8
+ class MessageBase(BaseModel):
9
+ role: str
10
+ content: str
11
+
12
+
13
+ # UserMessage class with variable handling
14
+ class UserMessage(MessageBase):
15
+ role: str = "user"
16
+ content: str
17
+ variable_names: Optional[List[str]] = []
18
+
19
+ @field_validator("content")
20
+ def validate_content(cls, v: str) -> str:
21
+ """Validate user message content."""
22
+ if not v.strip():
23
+ raise ValueError("Message content cannot be empty.")
24
+ return v.strip()
25
+
26
+ def model_post_init(self, __context: Any) -> None:
27
+ """Initialize variable names by inferring them from the content if not already set."""
28
+ if self.variable_names == []:
29
+ # Extract variable names from content by looking for {{name}} patterns
30
+ content = self.content
31
+ var_names = []
32
+ parts = content.split("{{")
33
+ for part in parts[1:]: # Skip first part before any {{
34
+ if "}}" in part:
35
+ var_name = part.split("}}")[0].strip()
36
+ var_names.append(var_name)
37
+ self.variable_names = var_names
38
+
39
+
40
+ # SystemMessage class
41
+ class SystemMessage(MessageBase):
42
+ role: str = "system"
43
+ content: str
44
+
45
+ @field_validator("content")
46
+ def validate_content(cls, v: str) -> str:
47
+ """Validate system message content."""
48
+ if not v.strip():
49
+ raise ValueError("Message content cannot be empty.")
50
+ return v.strip()
51
+
52
+
53
+ # AssistantMessage class
54
+ class AssistantMessage(MessageBase):
55
+ role: str = "assistant"
56
+ content: str
57
+
58
+ @field_validator("content")
59
+ def validate_content(cls, v: str) -> str:
60
+ """Validate assistant message content."""
61
+ if not v.strip():
62
+ raise ValueError("Message content cannot be empty.")
63
+ return v.strip()
64
+
65
+
66
+ class ModelConfig(BaseModel):
67
+ model_name: str = "gpt-4o-mini"
68
+ temperature: Optional[float] = Field(default=0.7, ge=0, le=2)
69
+ frequency_penalty: Optional[float] = Field(default=0, ge=-2, le=2)
70
+ presence_penalty: Optional[float] = Field(default=0, ge=-2, le=2)
71
+ max_tokens: Optional[int] = Field(default=1000, gt=0)
72
+ top_p: Optional[float] = Field(default=1.0, ge=0, le=1)
73
+ response_format: Optional[dict] = None
74
+ tool_choice: Optional[str] = None
75
+ tools: Optional[List[dict]] = None
76
+
77
+ class Config:
78
+ frozen = True
79
+
80
+
81
+ class PromptTemplate(BaseModel):
82
+ id: Optional[uuid.UUID] = None
83
+ name: str
84
+ messages: Optional[List[MessageBase]] = []
85
+ model_configuration: Optional[ModelConfig] = ModelConfig()
86
+ variable_names: Optional[Dict[str, List[str]]] = {}
87
+ description: Optional[str] = None
88
+ version: Optional[str] = None
89
+ is_default: bool = True
90
+ evaluation_configs: Optional[List[Dict]] = None
91
+ status: Optional[str] = None
92
+ error_message: Optional[str] = None
93
+
94
+ @field_validator("name")
95
+ def validate_name(cls, v: str) -> str:
96
+ """Ensure name is valid."""
97
+ if not v.strip():
98
+ raise ValueError("Prompt name cannot be empty.")
99
+ if len(v) > 255:
100
+ raise ValueError("Prompt name is too long (max 255 characters).")
101
+ return v.strip()
102
+
103
+ def model_post_init(self, __context: Any) -> None:
104
+ """Initialize variable names by inferring them from the content if not already set."""
105
+ if self.variable_names == {}:
106
+ # extract variable names from messages
107
+ for message in self.messages:
108
+ if isinstance(message, UserMessage):
109
+ for var_name in message.variable_names:
110
+ if var_name not in self.variable_names:
111
+ self.variable_names[var_name] = []
@@ -21,6 +21,20 @@ class Routes(str, Enum):
21
21
  dataset_delete = "model-hub/develops/delete_dataset/"
22
22
  dataset_add_rows = "model-hub/develops/{dataset_id}/add_rows/"
23
23
  dataset_add_columns = "model-hub/develops/{dataset_id}/add_columns/"
24
+
25
+ # prompt
26
+ generate_prompt = "model-hub/prompt-templates/generate-prompt/"
27
+ improve_prompt = "model-hub/prompt-templates/improve-prompt/"
28
+ run_template = "model-hub/prompt-templates/{template_id}/run_template/"
29
+ create_template = "model-hub/prompt-templates/create-draft/"
30
+ delete_template = "model-hub/prompt-templates/{template_id}"
31
+ get_template_by_id = "model-hub/prompt-templates/{template_id}"
32
+ get_template_id_by_name = "model-hub/prompt-templates/"
33
+
24
34
  # model provider
25
35
  model_hub_api_keys = "model-hub/api-keys/"
26
36
  model_hub_default_provider = "model-hub/default-provider/"
37
+
38
+ dataset_add_run_prompt_column = "model-hub/develops/add_run_prompt_column/"
39
+ dataset_add_evaluation = "model-hub/develops/{dataset_id}/add_user_eval/"
40
+ dataset_optimization_create = "model-hub/optimisation/create/"
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "futureagi"
3
- version = "0.4.0"
3
+ version = "0.4.2"
4
4
  description = "We help GenAI teams maintain high-accuracy for their Models in production."
5
5
  authors = ["Future AGI <no-reply@futureagi.com>"]
6
6
  readme = "README.md"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes