futureagi 0.3.6__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. {futureagi-0.3.6 → futureagi-0.4.0}/PKG-INFO +2 -1
  2. futureagi-0.4.0/fi/api/apikeys.py +147 -0
  3. {futureagi-0.3.6 → futureagi-0.4.0}/fi/api/auth.py +3 -1
  4. futureagi-0.4.0/fi/api/types.py +66 -0
  5. {futureagi-0.3.6 → futureagi-0.4.0}/fi/datasets/client.py +148 -6
  6. {futureagi-0.3.6 → futureagi-0.4.0}/fi/datasets/types.py +40 -7
  7. {futureagi-0.3.6 → futureagi-0.4.0}/fi/evals/__init__.py +1 -0
  8. {futureagi-0.3.6 → futureagi-0.4.0}/fi/evals/evaluator.py +24 -6
  9. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/errors.py +11 -0
  10. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/routes.py +11 -2
  11. {futureagi-0.3.6 → futureagi-0.4.0}/pyproject.toml +2 -1
  12. futureagi-0.3.6/fi/api/types.py +0 -27
  13. {futureagi-0.3.6 → futureagi-0.4.0}/LICENSE.md +0 -0
  14. {futureagi-0.3.6 → futureagi-0.4.0}/README.md +0 -0
  15. {futureagi-0.3.6 → futureagi-0.4.0}/fi/__init__.py +0 -0
  16. {futureagi-0.3.6 → futureagi-0.4.0}/fi/api/__init__.py +0 -0
  17. {futureagi-0.3.6 → futureagi-0.4.0}/fi/client.py +0 -0
  18. {futureagi-0.3.6 → futureagi-0.4.0}/fi/datasets/__init__.py +0 -0
  19. {futureagi-0.3.6 → futureagi-0.4.0}/fi/evals/templates.py +0 -0
  20. {futureagi-0.3.6 → futureagi-0.4.0}/fi/evals/types.py +0 -0
  21. {futureagi-0.3.6 → futureagi-0.4.0}/fi/integrations/__init__.py +0 -0
  22. {futureagi-0.3.6 → futureagi-0.4.0}/fi/integrations/providers/__init__.py +0 -0
  23. {futureagi-0.3.6 → futureagi-0.4.0}/fi/integrations/providers/types.py +0 -0
  24. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/__init__.py +0 -0
  25. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/conversational_test_case.py +0 -0
  26. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/general.py +0 -0
  27. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/llm_test_case.py +0 -0
  28. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/mllm_test_case.py +0 -0
  29. {futureagi-0.3.6 → futureagi-0.4.0}/fi/testcases/utils.py +0 -0
  30. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/__init__.py +0 -0
  31. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/constants.py +0 -0
  32. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/executor.py +0 -0
  33. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/logging.py +0 -0
  34. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/types.py +0 -0
  35. {futureagi-0.3.6 → futureagi-0.4.0}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: futureagi
3
- Version: 0.3.6
3
+ Version: 0.4.0
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -21,6 +21,7 @@ Requires-Dist: pytz (==2024.1)
21
21
  Requires-Dist: requests (==2.32.3)
22
22
  Requires-Dist: requests-futures (==1.0.1)
23
23
  Requires-Dist: six (==1.16.0)
24
+ Requires-Dist: tqdm (>=4.67.1,<5.0.0)
24
25
  Requires-Dist: tzdata (==2024.1)
25
26
  Requires-Dist: urllib3 (==2.2.2)
26
27
  Description-Content-Type: text/markdown
@@ -0,0 +1,147 @@
1
+ from typing import List, Optional, Union
2
+
3
+ from requests import Response
4
+
5
+ from fi.api.auth import APIKeyAuth, ResponseHandler
6
+ from fi.api.types import ApiKey, HttpMethod, ModelProvider, RequestConfig
7
+ from fi.utils.routes import Routes
8
+
9
+
10
+ class ProviderAPIKeyResponseHandler(ResponseHandler[ApiKey, None]):
11
+ """Handles responses for API key operations"""
12
+
13
+ @classmethod
14
+ def _parse_success(cls, response: Response) -> Union[ApiKey, List[ApiKey]]:
15
+ data = response.json()
16
+ method = response.request.method
17
+ if method == HttpMethod.POST.value:
18
+ return {
19
+ "success": True,
20
+ }
21
+ elif method == HttpMethod.GET.value:
22
+ data = data["results"]
23
+ result = []
24
+ for api_key in data:
25
+ result.append(ApiKey(**api_key))
26
+ return result
27
+ else:
28
+ return data
29
+
30
+ @classmethod
31
+ def _handle_error(cls, response: Response) -> None:
32
+ if response.status_code >= 400:
33
+ response.raise_for_status()
34
+
35
+
36
+ class ProviderAPIKeyClient(APIKeyAuth):
37
+ """Client for API key operations
38
+
39
+ This client can be used in two ways:
40
+ 1. As class methods for simple one-off operations:
41
+ ProviderAPIKeyClient.set_api_key(api_key)
42
+
43
+ 2. As an instance for chained operations:
44
+ client = ProviderAPIKeyClient()
45
+ client.set(api_key).get(provider)
46
+ """
47
+
48
+ def __init__(
49
+ self,
50
+ api_key: ApiKey,
51
+ fi_api_key: Optional[str] = None,
52
+ fi_secret_key: Optional[str] = None,
53
+ fi_base_url: Optional[str] = None,
54
+ ):
55
+ """
56
+ Initialize the ProviderAPIKeyClient
57
+
58
+ Args:
59
+ fi_api_key (str): The API key for the organization
60
+ fi_secret_key (str): The secret key for the organization
61
+ fi_base_url (str): The base URL for the organization
62
+ api_key (ApiKey): The API key to set
63
+
64
+ Returns:
65
+ ProviderAPIKeyClient: The initialized client
66
+ """
67
+ super().__init__(fi_api_key, fi_secret_key, fi_base_url)
68
+ self.api_key = api_key
69
+
70
+ # Instance methods for chaining
71
+ def set(self) -> "ProviderAPIKeyClient":
72
+ """Create a new API key and return self for chaining"""
73
+ if self.api_key.key is None:
74
+ raise ValueError("API key is required")
75
+ response = self._set_api_key(self.api_key) # noqa: F841
76
+ return self
77
+
78
+ def get(self) -> Optional[ApiKey]:
79
+ """Get a specific API key by provider"""
80
+ response = self._get_api_key(self.api_key.provider)
81
+ self.api_key = response
82
+ return response
83
+
84
+ # Protected internal methods
85
+ def _set_api_key(self, api_key: ApiKey) -> ApiKey:
86
+ """Internal method for setting API key"""
87
+ response = self.request(
88
+ config=RequestConfig(
89
+ method=HttpMethod.POST,
90
+ url=self._base_url + "/" + Routes.model_hub_api_keys.value,
91
+ json={
92
+ "provider": api_key.provider.value,
93
+ "key": api_key.key,
94
+ },
95
+ ),
96
+ response_handler=ProviderAPIKeyResponseHandler,
97
+ )
98
+ return response
99
+
100
+ def _get_api_key(self, provider: ModelProvider) -> Optional[ApiKey]:
101
+ """Internal method to get API key by provider"""
102
+ response = self.request(
103
+ config=RequestConfig(
104
+ method=HttpMethod.GET,
105
+ url=self._base_url + "/" + Routes.model_hub_api_keys.value,
106
+ ),
107
+ response_handler=ProviderAPIKeyResponseHandler,
108
+ )
109
+ if response:
110
+ for api_key in response:
111
+ if api_key.provider == provider:
112
+ return api_key
113
+ return None
114
+
115
+ # Class methods for simple operations
116
+ @classmethod
117
+ def _get_instance(cls, api_key: ApiKey, **kwargs) -> "ProviderAPIKeyClient":
118
+ """Create a new ProviderAPIKeyClient instance"""
119
+ instance = cls(api_key, **kwargs) if isinstance(cls, type) else cls
120
+ instance.api_key = api_key
121
+ return instance
122
+
123
+ @classmethod
124
+ def set_api_key(cls, api_key: ApiKey, **kwargs) -> ApiKey:
125
+ """Class method for simple API key creation"""
126
+ instance = cls._get_instance(api_key, **kwargs)
127
+ return instance.set()
128
+
129
+ @classmethod
130
+ def get_api_key(cls, provider: ModelProvider, **kwargs) -> Optional[ApiKey]:
131
+ """Class method for simple API key retrieval"""
132
+ api_key = ApiKey(provider=provider)
133
+ instance = cls._get_instance(api_key, **kwargs)
134
+ return instance.get()
135
+
136
+ @classmethod
137
+ def list_api_keys(cls, **kwargs) -> List[ApiKey]:
138
+ """List all API keys for the organization"""
139
+ instance = cls._get_instance(ApiKey(), **kwargs)
140
+ response = instance.request(
141
+ config=RequestConfig(
142
+ method=HttpMethod.GET,
143
+ url=instance._base_url + "/" + Routes.model_hub_api_keys.value,
144
+ ),
145
+ response_handler=ProviderAPIKeyResponseHandler,
146
+ )
147
+ return response
@@ -16,7 +16,7 @@ from fi.utils.constants import (
16
16
  DEFAULT_TIMEOUT,
17
17
  SECRET_KEY_ENVVAR_NAME,
18
18
  )
19
- from fi.utils.errors import MissingAuthError
19
+ from fi.utils.errors import MissingAuthError, NoDatasetFound
20
20
  from fi.utils.executor import BoundedExecutor
21
21
 
22
22
  T = TypeVar("T")
@@ -98,6 +98,8 @@ class HttpClient:
98
98
  return response
99
99
 
100
100
  except Exception as e:
101
+ if isinstance(e, NoDatasetFound):
102
+ raise e
101
103
  if attempt == config.retry_attempts - 1:
102
104
  raise e
103
105
  time.sleep(config.retry_delay)
@@ -0,0 +1,66 @@
1
+ from enum import Enum
2
+ from typing import Any, Dict, Optional
3
+
4
+ from pydantic import BaseModel
5
+
6
+
7
+ class HttpMethod(Enum):
8
+ GET = "GET"
9
+ POST = "POST"
10
+ PUT = "PUT"
11
+ DELETE = "DELETE"
12
+ PATCH = "PATCH"
13
+
14
+
15
+ class RequestConfig(BaseModel):
16
+ """Configuration for an HTTP request"""
17
+
18
+ method: HttpMethod
19
+ url: str
20
+ headers: Optional[Dict[str, str]] = {}
21
+ params: Optional[Dict[str, Any]] = {}
22
+ files: Optional[Dict[str, Any]] = {}
23
+ data: Optional[Dict[str, Any]] = {}
24
+ json: Optional[Dict[str, Any]] = {}
25
+ timeout: Optional[int] = None
26
+ retry_attempts: int = 3
27
+ retry_delay: float = 1.0
28
+
29
+
30
+ class ModelProvider(str, Enum):
31
+ OPENAI = "openai"
32
+ ANTHROPIC = "anthropic"
33
+ GOOGLE = "google"
34
+ AZURE = "azure"
35
+ AWS = "aws"
36
+ COHERE = "cohere"
37
+ FIREWORKS = "fireworks_ai"
38
+ ANYSCALE = "anyscale"
39
+ PERPLEXITY = "perplexity"
40
+ DEEPINFRA = "deepinfra"
41
+ OLLAMA = "ollama"
42
+ CLOUDFLARE = "cloudflare"
43
+ VOYAGE = "voyage"
44
+ DATABRICKS = "databricks"
45
+ TEXT_COMPLETION_OPENAI = "text-completion-openai"
46
+ TEXT_COMPLETION_CODESTRAL = "text-completion-codestral"
47
+ FIREWORKS_EMBEDDING = "fireworks_ai-embedding-models"
48
+ VERTEX_AI_TEXT = "vertex_ai-text-models"
49
+ VERTEX_AI_CHAT = "vertex_ai-chat-models"
50
+ VERTEX_AI_CODE_TEXT = "vertex_ai-code-text-models"
51
+ SAGEMAKER = "sagemaker"
52
+ BEDROCK = "bedrock"
53
+ TOGETHER_AI = "together_ai"
54
+ MISTRAL = "mistral"
55
+ DEEPSEEK = "deepseek"
56
+ CODESTRAL = "codestral"
57
+ GROQ = "groq"
58
+ CEREBRAS = "cerebras"
59
+ FRIENDLIAI = "friendliai"
60
+ AZURE_AI = "azure_ai"
61
+ HUGGINGFACE = "huggingface"
62
+
63
+
64
+ class ApiKey(BaseModel):
65
+ provider: Optional[ModelProvider] = None
66
+ key: Optional[str] = None
@@ -20,7 +20,7 @@ from fi.utils.constants import (
20
20
  DATASET_TEMP_FILE_SUFFIX,
21
21
  PAGE_SIZE,
22
22
  )
23
- from fi.utils.errors import InvalidAuthError
23
+ from fi.utils.errors import InvalidAuthError, NoDatasetFound
24
24
  from fi.utils.routes import Routes
25
25
  from fi.utils.utils import get_tempfile_path
26
26
 
@@ -33,10 +33,12 @@ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
33
33
  def _parse_success(cls, response: Response) -> Union[DatasetConfig, DatasetTable]:
34
34
  """Parse successful response into DatasetResponse"""
35
35
  data = response.json()
36
+ print("data", data)
37
+ print("response", response.url)
36
38
  if response.url.endswith(Routes.dataset_names.value):
37
39
  datasets = data["result"]["datasets"]
38
40
  if not datasets:
39
- raise ValueError("No dataset found")
41
+ raise NoDatasetFound
40
42
  if len(datasets) > 1:
41
43
  raise ValueError(
42
44
  "Multiple datasets found. Please specify a dataset name."
@@ -132,10 +134,10 @@ class DatasetClient(APIKeyAuth):
132
134
 
133
135
  def __init__(
134
136
  self,
137
+ dataset_config: Optional[DatasetConfig] = None,
135
138
  fi_api_key: Optional[str] = None,
136
139
  fi_secret_key: Optional[str] = None,
137
140
  fi_base_url: Optional[str] = None,
138
- dataset_config: Optional[DatasetConfig] = None,
139
141
  **kwargs,
140
142
  ):
141
143
  super().__init__(
@@ -144,7 +146,14 @@ class DatasetClient(APIKeyAuth):
144
146
  fi_base_url=fi_base_url,
145
147
  **kwargs,
146
148
  )
147
- self.dataset_config = dataset_config
149
+
150
+ if dataset_config and not dataset_config.id:
151
+ try:
152
+ self.dataset_config = self._fetch_dataset_config(dataset_config.name)
153
+ except NoDatasetFound:
154
+ self.dataset_config = dataset_config
155
+ else:
156
+ self.dataset_config = None
148
157
 
149
158
  # Instance methods for chaining
150
159
  def create(
@@ -154,6 +163,9 @@ class DatasetClient(APIKeyAuth):
154
163
  if not self.dataset_config:
155
164
  raise ValueError("dataset_config must be set")
156
165
 
166
+ if self.dataset_config.id:
167
+ raise ValueError("dataset alredy exists")
168
+
157
169
  response = self._create_dataset(self.dataset_config, source)
158
170
  self.dataset_config.id = response.id
159
171
  return self
@@ -178,7 +190,59 @@ class DatasetClient(APIKeyAuth):
178
190
  raise ValueError("No dataset configured")
179
191
  return self.dataset_config
180
192
 
193
+ def add_columns(
194
+ self,
195
+ columns: List[Union[Column, dict]],
196
+ ):
197
+ if not self.dataset_config:
198
+ raise ValueError("dataset_config must be set")
199
+
200
+ if all(isinstance(column, dict) for column in columns):
201
+ columns = [
202
+ Column(name=column["name"], data_type=column["data_type"])
203
+ for column in columns
204
+ ]
205
+
206
+ self._add_columns(columns=columns)
207
+ return self
208
+
209
+ def add_rows(
210
+ self,
211
+ rows: List[Union[Row, dict]],
212
+ ):
213
+ if not self.dataset_config:
214
+ raise ValueError("dataset_config must be set")
215
+
216
+ if all(isinstance(row, dict) for row in rows):
217
+ rows = [
218
+ Row(
219
+ cells=[
220
+ Cell(column_name=cell["column_name"], value=cell["value"])
221
+ for cell in row["cells"]
222
+ ]
223
+ )
224
+ for row in rows
225
+ ]
226
+
227
+ self._add_rows(rows)
228
+ return self
229
+
181
230
  # Protected internal methods
231
+ def _fetch_dataset_config(self, dataset_name):
232
+ url = f"{self._base_url}/{Routes.dataset_names.value}"
233
+ try:
234
+ dataset_config = self.request(
235
+ config=RequestConfig(
236
+ method=HttpMethod.POST,
237
+ url=url,
238
+ json={"search_text": dataset_name},
239
+ ),
240
+ response_handler=DatasetResponseHandler,
241
+ )
242
+ return dataset_config
243
+ except NoDatasetFound as e:
244
+ raise e
245
+
182
246
  def _create_dataset(
183
247
  self,
184
248
  config: DatasetConfig,
@@ -296,11 +360,55 @@ class DatasetClient(APIKeyAuth):
296
360
  response_handler=DatasetResponseHandler,
297
361
  )
298
362
 
363
+ def _add_columns(self, columns: List[Column]) -> None:
364
+ """Add columns to the dataset"""
365
+
366
+ if not self.dataset_config or not getattr(self.dataset_config, "id", None):
367
+ raise ValueError("No dataset configured for column addition.")
368
+
369
+ if not isinstance(columns, list):
370
+ raise TypeError("Columns must be a list of Column.")
371
+
372
+ if not all(isinstance(column, Column) for column in columns):
373
+ raise ValueError("Each column must be a Column.")
374
+
375
+ serialized_columns = [column.to_dict() for column in columns]
376
+ url = f"{self._base_url}/{Routes.dataset_add_columns.value.format(dataset_id=str(self.dataset_config.id))}"
377
+ payload = {
378
+ "new_columns_data": serialized_columns,
379
+ }
380
+
381
+ self.request(
382
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
383
+ response_handler=DatasetResponseHandler,
384
+ )
385
+
386
+ def _add_rows(self, rows: List[Row], **kwargs) -> None:
387
+ """Add rows to the dataset"""
388
+ if not self.dataset_config or not self.dataset_config.id:
389
+ raise ValueError("No dataset configured for row addition")
390
+
391
+ serialized_rows = [row.to_dict() for row in rows]
392
+
393
+ url = f"{self._base_url}/{Routes.dataset_add_rows.value.format(dataset_id=str(self.dataset_config.id))}"
394
+ payload = {"rows": serialized_rows}
395
+
396
+ self.request(
397
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
398
+ response_handler=DatasetResponseHandler,
399
+ )
400
+
299
401
  # Class methods for simple operations
300
402
  @classmethod
301
- def _get_instance(cls, **kwargs) -> "DatasetClient":
403
+ def _get_instance(
404
+ cls, dataset_config: Optional[DatasetConfig] = None, **kwargs
405
+ ) -> "DatasetClient":
302
406
  """Create a new DatasetClient instance"""
303
- return cls(**kwargs) if isinstance(cls, type) else cls
407
+ return (
408
+ cls(dataset_config=dataset_config, **kwargs)
409
+ if isinstance(cls, type)
410
+ else cls
411
+ )
304
412
 
305
413
  @classmethod
306
414
  def create_dataset(
@@ -357,3 +465,37 @@ class DatasetClient(APIKeyAuth):
357
465
  instance.dataset_config = dataset_config
358
466
  cls._dataset_id_cache[cache_key] = instance
359
467
  return instance
468
+
469
+ @classmethod
470
+ def add_dataset_columns(
471
+ cls, dataset_name: str, columns: List[Union[Row, dict]], **kwargs
472
+ ):
473
+ if all(isinstance(column, dict) for column in columns):
474
+ columns = [
475
+ Column(name=column["name"], data_type=column["data_type"])
476
+ for column in columns
477
+ ]
478
+
479
+ instance = cls.get_dataset_config(dataset_name, **kwargs)
480
+ return instance.add_columns(columns=columns)
481
+
482
+ @classmethod
483
+ def add_dataset_rows(
484
+ cls,
485
+ dataset_name: str,
486
+ rows=List[Union[Row, dict]],
487
+ **kwargs,
488
+ ):
489
+ if all(isinstance(row, dict) for row in rows):
490
+ rows = [
491
+ Row(
492
+ cells=[
493
+ Cell(column_name=cell["column_name"], value=cell["value"])
494
+ for cell in row["cells"]
495
+ ]
496
+ )
497
+ for row in rows
498
+ ]
499
+
500
+ instance = cls.get_dataset_config(dataset_name, **kwargs)
501
+ return instance.add_rows(rows=rows)
@@ -65,10 +65,10 @@ class SourceChoices(str, Enum):
65
65
  class Column(BaseModel):
66
66
  """Column information for dataset tables"""
67
67
 
68
- id: uuid.UUID
68
+ id: uuid.UUID = Field(default_factory=uuid.uuid4)
69
69
  name: str
70
70
  data_type: DataTypeChoices
71
- source: SourceChoices
71
+ source: Optional[SourceChoices] = Field(default=SourceChoices.OTHERS)
72
72
  source_id: Optional[str] = None
73
73
  metadata: Optional[Dict[str, Any]] = Field(default_factory=dict)
74
74
  is_frozen: bool = False
@@ -86,6 +86,17 @@ class Column(BaseModel):
86
86
  raise ValueError("Column name too long (max 255 characters)")
87
87
  return v.strip()
88
88
 
89
+ def to_dict(self) -> Dict[str, Any]:
90
+ """Convert Column instance to a dictionary"""
91
+ return {
92
+ "id": str(self.id),
93
+ "name": self.name,
94
+ "data_type": self.data_type.value,
95
+ "source": self.source.value,
96
+ "source_id": self.source_id,
97
+ "metadata": self.metadata,
98
+ }
99
+
89
100
  class Config:
90
101
  frozen = True
91
102
 
@@ -93,9 +104,10 @@ class Column(BaseModel):
93
104
  class Cell(BaseModel):
94
105
  """Cell information for dataset tables"""
95
106
 
96
- column_id: uuid.UUID
97
- row_id: uuid.UUID
98
- value: Optional[str] = None
107
+ column_id: uuid.UUID = Field(default_factory=uuid.uuid4)
108
+ row_id: uuid.UUID = Field(default_factory=uuid.uuid4)
109
+ column_name: Optional[str] = None
110
+ value: Optional[Any] = None
99
111
  value_infos: Optional[List[Dict[str, Any]]] = Field(default_factory=list)
100
112
  metadata: Optional[Dict[str, Any]] = Field(default_factory=dict)
101
113
  status: Optional[str] = None
@@ -117,6 +129,19 @@ class Cell(BaseModel):
117
129
  return []
118
130
  return v
119
131
 
132
+ def to_dict(self) -> Dict[str, Any]:
133
+ """Convert Cell Instance to dictionary"""
134
+ return {
135
+ "column_id": str(self.column_id),
136
+ "row_id": str(self.row_id),
137
+ "column_name": self.column_name,
138
+ "value": self.value,
139
+ "value_infos": self.value_infos,
140
+ "metadata": self.metadata,
141
+ "status": self.status,
142
+ "failure_reason": self.failure_reason,
143
+ }
144
+
120
145
  class Config:
121
146
  frozen = True
122
147
 
@@ -124,8 +149,8 @@ class Cell(BaseModel):
124
149
  class Row(BaseModel):
125
150
  """Row information for dataset tables"""
126
151
 
127
- id: uuid.UUID
128
- order: int
152
+ id: uuid.UUID = Field(default_factory=uuid.uuid4)
153
+ order: Optional[int] = Field(default=0)
129
154
  cells: List[Cell]
130
155
 
131
156
  @field_validator("order")
@@ -142,6 +167,14 @@ class Row(BaseModel):
142
167
  raise ValueError("Row must have at least one cell")
143
168
  return v
144
169
 
170
+ def to_dict(self) -> Dict[str, Any]:
171
+ """Convert Row instance to a dictionary"""
172
+ return {
173
+ "id": str(self.id),
174
+ "order": self.order,
175
+ "cells": [cell.to_dict() for cell in self.cells],
176
+ }
177
+
145
178
  class Config:
146
179
  frozen = True
147
180
 
@@ -129,4 +129,5 @@ __all__ = [
129
129
  "Tone",
130
130
  "Toxicity",
131
131
  "TranslationAccuracy",
132
+ "EvalClient",
132
133
  ]
@@ -64,6 +64,27 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
64
64
  return BatchRunResult(eval_results=eval_results)
65
65
 
66
66
 
67
+ class EvalInfoResponseHandler(ResponseHandler[dict, None]):
68
+ """Handles responses for evaluation info requests"""
69
+
70
+ @classmethod
71
+ def _parse_success(cls, response: Response) -> dict:
72
+ data = response.json()
73
+ if "result" in data:
74
+ print("result", data["result"])
75
+ return data["result"]
76
+ else:
77
+ raise Exception(f"Failed to get evaluation info: {data}")
78
+
79
+ @classmethod
80
+ def _handle_error(cls, response: Response) -> None:
81
+ if response.status_code == 400:
82
+ response.raise_for_status()
83
+ if response.status_code == 403:
84
+ raise InvalidAuthError()
85
+ raise Exception(f"Failed to get evaluation info: {response.status_code}")
86
+
87
+
67
88
  class EvalClient(APIKeyAuth):
68
89
  """Client for evaluating LLM test cases"""
69
90
 
@@ -199,7 +220,6 @@ class EvalClient(APIKeyAuth):
199
220
 
200
221
  for template in eval_templates:
201
222
  eval_id = template.eval_id
202
-
203
223
  eval_info = self._get_eval_info(eval_id)
204
224
  template.name = eval_info["name"]
205
225
  template.description = eval_info["description"]
@@ -223,9 +243,7 @@ class EvalClient(APIKeyAuth):
223
243
  + Routes.evaluate_template.value.format(eval_id=eval_id)
224
244
  )
225
245
  response = self.request(
226
- config=RequestConfig(
227
- method=HttpMethod.GET,
228
- url=url,
229
- ),
246
+ config=RequestConfig(method=HttpMethod.GET, url=url),
247
+ response_handler=EvalInfoResponseHandler,
230
248
  )
231
- return response.json()["result"]
249
+ return response
@@ -23,6 +23,17 @@ class CustomException(Exception, ABC):
23
23
  pass
24
24
 
25
25
 
26
+ class NoDatasetFound(CustomException):
27
+ def __init__(self, *args):
28
+ super().__init__(*args)
29
+
30
+ def __repr__(self) -> str:
31
+ return "No_Dataset_Found"
32
+
33
+ def error_message(self) -> str:
34
+ return " No Existing Dataset Found for Current Dataset Name "
35
+
36
+
26
37
  class MissingAuthError(CustomException):
27
38
  def __init__(self, fi_api_key: Optional[str], fi_secret_key: Optional[str]) -> None:
28
39
  self.missing_api_key = fi_api_key is None
@@ -4,10 +4,14 @@ from enum import Enum
4
4
  class Routes(str, Enum):
5
5
  healthcheck = "healthcheck"
6
6
 
7
- evaluate = "sdk/api/v1/eval/"
7
+ # logging
8
8
  log_model = "sdk/api/v1/log/model/"
9
+
10
+ # evaluation
11
+ evaluate = "sdk/api/v1/eval/"
9
12
  evaluate_template = "sdk/api/v1/eval/{eval_id}/"
10
- api_keys = "model-hub/api-keys"
13
+
14
+ # dataset
11
15
  dataset = "model-hub/develops"
12
16
  dataset_names = "model-hub/develops/get-datasets-names/"
13
17
  dataset_empty = "model-hub/develops/create-empty-dataset/"
@@ -15,3 +19,8 @@ class Routes(str, Enum):
15
19
  dataset_huggingface = "model-hub/develops/create-dataset-from-huggingface/"
16
20
  dataset_table = "model-hub/develops/{dataset_id}/get-dataset-table/"
17
21
  dataset_delete = "model-hub/develops/delete_dataset/"
22
+ dataset_add_rows = "model-hub/develops/{dataset_id}/add_rows/"
23
+ dataset_add_columns = "model-hub/develops/{dataset_id}/add_columns/"
24
+ # model provider
25
+ model_hub_api_keys = "model-hub/api-keys/"
26
+ model_hub_default_provider = "model-hub/default-provider/"
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "futureagi"
3
- version = "0.3.6"
3
+ version = "0.4.0"
4
4
  description = "We help GenAI teams maintain high-accuracy for their Models in production."
5
5
  authors = ["Future AGI <no-reply@futureagi.com>"]
6
6
  readme = "README.md"
@@ -23,6 +23,7 @@ six = "1.16.0"
23
23
  tzdata = "2024.1"
24
24
  urllib3 = "2.2.2"
25
25
  pydantic = "^2.9.2"
26
+ tqdm = "^4.67.1"
26
27
 
27
28
 
28
29
  [tool.poetry.group.dev.dependencies]
@@ -1,27 +0,0 @@
1
- from enum import Enum
2
- from typing import Any, Dict, Optional
3
-
4
- from pydantic import BaseModel
5
-
6
-
7
- class HttpMethod(Enum):
8
- GET = "GET"
9
- POST = "POST"
10
- PUT = "PUT"
11
- DELETE = "DELETE"
12
- PATCH = "PATCH"
13
-
14
-
15
- class RequestConfig(BaseModel):
16
- """Configuration for an HTTP request"""
17
-
18
- method: HttpMethod
19
- url: str
20
- headers: Optional[Dict[str, str]] = {}
21
- params: Optional[Dict[str, Any]] = {}
22
- files: Optional[Dict[str, Any]] = {}
23
- data: Optional[Dict[str, Any]] = {}
24
- json: Optional[Dict[str, Any]] = {}
25
- timeout: Optional[int] = None
26
- retry_attempts: int = 3
27
- retry_delay: float = 1.0
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes