futureagi 0.3.3__tar.gz → 0.3.4__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of futureagi might be problematic. Click here for more details.

Files changed (36) hide show
  1. {futureagi-0.3.3 → futureagi-0.3.4}/PKG-INFO +1 -1
  2. {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/auth.py +8 -3
  3. {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/types.py +2 -0
  4. futureagi-0.3.4/fi/datasets/__init__.py +4 -0
  5. futureagi-0.3.4/fi/datasets/client.py +442 -0
  6. futureagi-0.3.4/fi/datasets/types.py +284 -0
  7. {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/__init__.py +28 -12
  8. {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/evaluator.py +45 -2
  9. futureagi-0.3.4/fi/evals/templates.py +334 -0
  10. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/conversational_test_case.py +0 -1
  11. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/constants.py +7 -1
  12. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/errors.py +3 -3
  13. futureagi-0.3.4/fi/utils/routes.py +17 -0
  14. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/types.py +6 -23
  15. futureagi-0.3.4/fi/utils/utils.py +29 -0
  16. {futureagi-0.3.3 → futureagi-0.3.4}/pyproject.toml +1 -1
  17. futureagi-0.3.3/fi/evals/templates.py +0 -1060
  18. futureagi-0.3.3/fi/utils/routes.py +0 -16
  19. futureagi-0.3.3/fi/utils/utils.py +0 -10
  20. {futureagi-0.3.3 → futureagi-0.3.4}/LICENSE.md +0 -0
  21. {futureagi-0.3.3 → futureagi-0.3.4}/README.md +0 -0
  22. {futureagi-0.3.3 → futureagi-0.3.4}/fi/__init__.py +0 -0
  23. {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/__init__.py +0 -0
  24. {futureagi-0.3.3 → futureagi-0.3.4}/fi/client.py +0 -0
  25. {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/types.py +0 -0
  26. {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/__init__.py +0 -0
  27. {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/providers/__init__.py +0 -0
  28. {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/providers/types.py +0 -0
  29. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/__init__.py +0 -0
  30. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/general.py +0 -0
  31. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/llm_test_case.py +0 -0
  32. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/mllm_test_case.py +0 -0
  33. {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/utils.py +0 -0
  34. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/__init__.py +0 -0
  35. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/executor.py +0 -0
  36. {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/logging.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: futureagi
3
- Version: 0.3.3
3
+ Version: 0.3.4
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -20,13 +20,14 @@ from fi.utils.errors import MissingAuthError
20
20
  from fi.utils.executor import BoundedExecutor
21
21
 
22
22
  T = TypeVar("T")
23
+ U = TypeVar("U", default=None)
23
24
 
24
25
 
25
- class ResponseHandler(Generic[T], ABC):
26
+ class ResponseHandler(Generic[T, U], ABC):
26
27
  """Handles response parsing and validation"""
27
28
 
28
29
  @classmethod
29
- def parse(cls, response: Response) -> T:
30
+ def parse(cls, response: Response) -> Union[T, U]:
30
31
  """Parse the response into the expected type"""
31
32
  if not response.ok:
32
33
  cls._handle_error(response)
@@ -34,7 +35,7 @@ class ResponseHandler(Generic[T], ABC):
34
35
 
35
36
  @classmethod
36
37
  @abstractmethod
37
- def _parse_success(cls, response: Response) -> T:
38
+ def _parse_success(cls, response: Response) -> Union[T, U]:
38
39
  """Parse successful response"""
39
40
  pass
40
41
 
@@ -77,6 +78,8 @@ class HttpClient:
77
78
  params = config.params or {}
78
79
  json = config.json or {}
79
80
  timeout = config.timeout or self._default_timeout
81
+ files = config.files or {}
82
+ data = config.data or {}
80
83
  for attempt in range(config.retry_attempts):
81
84
  try:
82
85
  response = self._session.request(
@@ -85,6 +88,8 @@ class HttpClient:
85
88
  headers=headers,
86
89
  params=params,
87
90
  json=json,
91
+ data=data,
92
+ files=files,
88
93
  timeout=timeout,
89
94
  ).result()
90
95
 
@@ -19,6 +19,8 @@ class RequestConfig(BaseModel):
19
19
  url: str
20
20
  headers: Optional[Dict[str, str]] = {}
21
21
  params: Optional[Dict[str, Any]] = {}
22
+ files: Optional[Dict[str, Any]] = {}
23
+ data: Optional[Dict[str, Any]] = {}
22
24
  json: Optional[Dict[str, Any]] = {}
23
25
  timeout: Optional[int] = None
24
26
  retry_attempts: int = 3
@@ -0,0 +1,4 @@
1
+ from fi.datasets.client import DatasetClient
2
+ from fi.datasets.types import DatasetConfig, HuggingfaceDatasetConfig
3
+
4
+ __all__ = ["DatasetClient", "HuggingfaceDatasetConfig", "DatasetConfig"]
@@ -0,0 +1,442 @@
1
+ import os
2
+ from typing import List, Optional, Union
3
+
4
+ import pandas as pd
5
+ from requests import Response
6
+ from tqdm import tqdm
7
+
8
+ from fi.api.auth import APIKeyAuth, ResponseHandler
9
+ from fi.api.types import HttpMethod, RequestConfig
10
+ from fi.datasets.types import (
11
+ Cell,
12
+ Column,
13
+ DatasetConfig,
14
+ DatasetTable,
15
+ HuggingfaceDatasetConfig,
16
+ Row,
17
+ )
18
+ from fi.utils.constants import (
19
+ DATASET_TEMP_FILE_PREFIX,
20
+ DATASET_TEMP_FILE_SUFFIX,
21
+ PAGE_SIZE,
22
+ )
23
+ from fi.utils.errors import InvalidAuthError
24
+ from fi.utils.routes import Routes
25
+ from fi.utils.utils import get_tempfile_path
26
+
27
+
28
+ class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
29
+
30
+ """Handles responses for dataset requests"""
31
+
32
+ @classmethod
33
+ def _parse_success(cls, response: Response) -> Union[DatasetConfig, DatasetTable]:
34
+ """Parse successful response into DatasetResponse"""
35
+ data = response.json()
36
+ if response.url.endswith(Routes.dataset_names.value):
37
+ datasets = data["result"]["datasets"]
38
+ if not datasets:
39
+ raise ValueError("No dataset found")
40
+ if len(datasets) > 1:
41
+ raise ValueError(
42
+ "Multiple datasets found. Please specify a dataset name."
43
+ )
44
+ return DatasetConfig(
45
+ id=datasets[0]["datasetId"],
46
+ name=datasets[0]["name"],
47
+ model_type=datasets[0]["modelType"],
48
+ )
49
+ elif Routes.dataset_table.value.split("/")[-2] in response.url:
50
+ id = response.url.split("/")[-3]
51
+ columns = [
52
+ Column(
53
+ id=column["id"],
54
+ name=column["name"],
55
+ data_type=column["dataType"],
56
+ source=column["originType"],
57
+ source_id=column["sourceId"],
58
+ is_frozen=column["isFrozen"]["isFrozen"]
59
+ if column["isFrozen"] is not None
60
+ else False,
61
+ is_visible=column["isVisible"],
62
+ eval_tags=column["evalTag"],
63
+ average_score=column["averageScore"],
64
+ order_index=column["orderIndex"],
65
+ )
66
+ for column in data["result"]["columnConfig"]
67
+ ]
68
+ rows = []
69
+ for row in data["result"]["table"]:
70
+ cells = []
71
+ row_id = row.pop("rowId")
72
+ order = row.pop("order")
73
+ for column_id, value in row.items():
74
+ cells.append(
75
+ Cell(
76
+ column_id=column_id,
77
+ row_id=row_id,
78
+ value=value.get("cellValue"),
79
+ value_infos=[value.get("valueInfos")]
80
+ if value.get("valueInfos")
81
+ else None,
82
+ metadata=value.get("metadata"),
83
+ status=value.get("status"),
84
+ failure_reason=value.get("failureReason"),
85
+ )
86
+ )
87
+ rows.append(Row(id=row_id, order=order, cells=cells))
88
+ metadata = data["result"]["metadata"]
89
+ return DatasetTable(id=id, columns=columns, rows=rows, metadata=metadata)
90
+ elif response.url.endswith(Routes.dataset_empty.value):
91
+ return DatasetConfig(
92
+ id=data["result"]["datasetId"],
93
+ name=data["result"]["datasetName"],
94
+ model_type=data["result"]["datasetModelType"],
95
+ )
96
+ elif response.url.endswith(Routes.dataset_local.value):
97
+ return DatasetConfig(
98
+ id=data["result"]["datasetId"],
99
+ name=data["result"]["datasetName"],
100
+ model_type=data["result"]["datasetModelType"],
101
+ )
102
+ elif response.url.endswith(Routes.dataset_huggingface.value):
103
+ return DatasetConfig(
104
+ id=data["result"]["datasetId"],
105
+ name=data["result"]["datasetName"],
106
+ model_type=data["result"]["datasetModelType"],
107
+ )
108
+ else:
109
+ return data
110
+
111
+ @classmethod
112
+ def _handle_error(cls, response: Response) -> None:
113
+ if response.status_code == 400:
114
+ response.raise_for_status()
115
+ if response.status_code == 403:
116
+ raise InvalidAuthError()
117
+
118
+
119
+ class DatasetClient(APIKeyAuth):
120
+ """Manager class for handling datasets
121
+
122
+ A client for managing datasets including creation, deletion, downloading and metadata operations.
123
+ Provides functionality to work with empty datasets, local files, and Hugging Face datasets.
124
+
125
+ Attributes:
126
+ _dataset_id_cache (dict): Class-level cache for storing dataset IDs
127
+ dataset_config (DatasetConfig): Configuration for the current dataset
128
+ """
129
+
130
+ _dataset_id_cache = {}
131
+
132
+ def __init__(
133
+ self,
134
+ fi_api_key: Optional[str] = None,
135
+ fi_secret_key: Optional[str] = None,
136
+ fi_base_url: Optional[str] = None,
137
+ **kwargs,
138
+ ):
139
+ """Initialize the dataset manager
140
+
141
+ Args:
142
+ fi_api_key (str, optional): API key for authentication
143
+ fi_secret_key (str, optional): Secret key for authentication
144
+ fi_base_url (str, optional): Base URL of the API
145
+
146
+ Kwargs:
147
+ dataset_config (DatasetConfig): Dataset configuration
148
+
149
+ Returns:
150
+ DatasetClient: Instance of DatasetClient
151
+ """
152
+ super().__init__(
153
+ fi_api_key=fi_api_key,
154
+ fi_secret_key=fi_secret_key,
155
+ fi_base_url=fi_base_url,
156
+ **kwargs,
157
+ )
158
+ self.dataset_config = None
159
+ if kwargs.get("dataset_config"):
160
+ self.dataset_config = kwargs.get("dataset_config")
161
+
162
+ @classmethod
163
+ def _get_instance(
164
+ cls,
165
+ fi_api_key: Optional[str] = None,
166
+ fi_secret_key: Optional[str] = None,
167
+ fi_base_url: Optional[str] = None,
168
+ **kwargs,
169
+ ) -> "DatasetClient":
170
+ """Create and return an instance of DatasetClient
171
+
172
+ Factory method to create a new DatasetClient instance or return existing one.
173
+
174
+ Args:
175
+ fi_api_key (str, optional): API key for authentication
176
+ fi_secret_key (str, optional): Secret key for authentication
177
+ fi_base_url (str, optional): Base URL of the API
178
+
179
+ Kwargs:
180
+ dataset_config (DatasetConfig): Dataset configuration
181
+
182
+ Returns:
183
+ DatasetClient: New or existing instance of DatasetClient
184
+ """
185
+
186
+ if isinstance(cls, type):
187
+ # Called on class
188
+ return cls(
189
+ fi_api_key=fi_api_key,
190
+ fi_secret_key=fi_secret_key,
191
+ fi_base_url=fi_base_url,
192
+ **kwargs,
193
+ )
194
+ else:
195
+ # Called on instance
196
+ return cls
197
+
198
+ @classmethod
199
+ def create_dataset(
200
+ cls,
201
+ dataset_config: Optional[DatasetConfig] = None,
202
+ source: Optional[Union[str, HuggingfaceDatasetConfig]] = None,
203
+ **kwargs,
204
+ ) -> "DatasetClient":
205
+ """Create a new dataset
206
+
207
+ Creates a new dataset either empty, from a local file, or from Hugging Face.
208
+
209
+ Args:
210
+ dataset_config (DatasetConfig): Configuration for the new dataset
211
+ source (Optional[Union[str, HuggingfaceDatasetConfig]]): Source for dataset creation.
212
+ - If None: Creates empty dataset
213
+ - If str: Path to local file (.csv, .xlsx, .json, or .jsonl)
214
+ - If HuggingfaceDatasetConfig: Configuration for importing from Hugging Face
215
+
216
+ Returns:
217
+ DatasetClient: Instance of DatasetClient with the created dataset
218
+
219
+ Raises:
220
+ ValueError: If file format is not supported or file cannot be processed
221
+ InvalidAuthError: If authentication fails
222
+ """
223
+ instance = cls._get_instance(dataset_config=dataset_config, **kwargs)
224
+
225
+ if source is None:
226
+ # Create empty dataset
227
+ payload = {
228
+ "new_dataset_name": instance.dataset_config.name,
229
+ "model_type": instance.dataset_config.model_type.value,
230
+ }
231
+ url = instance._base_url + "/" + Routes.dataset_empty.value
232
+ response = instance.request(
233
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
234
+ response_handler=DatasetResponseHandler,
235
+ )
236
+
237
+ elif isinstance(source, str):
238
+ # Create from local file
239
+ supported_extensions = [".csv", ".xlsx", ".xls", ".json", ".jsonl"]
240
+ file_ext = os.path.splitext(source)[1].lower()
241
+ if file_ext not in supported_extensions:
242
+ raise ValueError(
243
+ f"Unsupported file format. File must have one of these extensions: {', '.join(supported_extensions)}"
244
+ )
245
+
246
+ files = {"file": (os.path.basename(source), open(source, "rb").read())}
247
+ data = {}
248
+ if instance.dataset_config.model_type:
249
+ data["model_type"] = instance.dataset_config.model_type.value
250
+ if instance.dataset_config.name:
251
+ data["new_dataset_name"] = instance.dataset_config.name
252
+ url = instance._base_url + "/" + Routes.dataset_local.value
253
+
254
+ response = instance.request(
255
+ config=RequestConfig(
256
+ method=HttpMethod.POST, url=url, data=data, files=files
257
+ ),
258
+ response_handler=DatasetResponseHandler,
259
+ )
260
+
261
+ elif isinstance(source, HuggingfaceDatasetConfig):
262
+ # Create from Hugging Face dataset
263
+ data = {
264
+ "new_dataset_name": instance.dataset_config.name,
265
+ "huggingface_dataset_name": source.name,
266
+ }
267
+ if instance.dataset_config.model_type:
268
+ data["model_type"] = instance.dataset_config.model_type.value
269
+ if source.split:
270
+ data["huggingface_dataset_split"] = source.split
271
+ if source.num_rows:
272
+ data["num_rows"] = source.num_rows
273
+ url = instance._base_url + "/" + Routes.dataset_huggingface.value
274
+
275
+ response = instance.request(
276
+ config=RequestConfig(method=HttpMethod.POST, url=url, data=data),
277
+ response_handler=DatasetResponseHandler,
278
+ )
279
+
280
+ # Update the dataset config with the new dataset ID
281
+ instance.dataset_config.id = response.id
282
+ return instance
283
+
284
+ @classmethod
285
+ def download_dataset(
286
+ cls,
287
+ dataset_name: Optional[str] = None,
288
+ file_path: Optional[str] = None,
289
+ load_to_pandas: Optional[bool] = False,
290
+ ) -> Union[str, pd.DataFrame]:
291
+ """Download dataset to file
292
+
293
+ Downloads a dataset to a local file or pandas DataFrame.
294
+
295
+ Args:
296
+ dataset_name (str, optional): Name of the dataset
297
+ file_path (str, optional): File path to save the dataset
298
+ load_to_pandas (bool, optional): Whether to return as pandas DataFrame
299
+
300
+ Returns:
301
+ Union[str, pd.DataFrame]: File path or pandas DataFrame containing the dataset
302
+
303
+ Raises:
304
+ ValueError: If file format is not supported
305
+ InvalidAuthError: If authentication fails
306
+ """
307
+ # Get the dataset metadata
308
+ instance = cls.get_dataset(dataset_name)
309
+
310
+ # Prepare request and data
311
+ url = (
312
+ instance._base_url
313
+ + "/"
314
+ + Routes.dataset_table.value.format(
315
+ dataset_id=str(instance.dataset_config.id)
316
+ )
317
+ )
318
+ data = {}
319
+ data["page_size"] = PAGE_SIZE
320
+ data["current_page_index"] = 0
321
+
322
+ # Prepare file path
323
+ if file_path is None:
324
+ file_path = get_tempfile_path(
325
+ DATASET_TEMP_FILE_PREFIX, DATASET_TEMP_FILE_SUFFIX
326
+ )
327
+
328
+ # Make the request
329
+ with tqdm(desc="Downloading dataset") as pbar:
330
+ while True:
331
+ pbar.set_postfix({"page": data["current_page_index"] + 1})
332
+ dataset_table = instance.request(
333
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=data),
334
+ response_handler=DatasetResponseHandler,
335
+ )
336
+ _ = dataset_table.to_file(file_path)
337
+ data["current_page_index"] += 1
338
+ if (
339
+ dataset_table.metadata.get("totalPages")
340
+ == data["current_page_index"]
341
+ ):
342
+ pbar.update(1)
343
+ break
344
+
345
+ # Load the dataset to pandas if requested
346
+ if load_to_pandas:
347
+ if file_path.endswith(".csv"):
348
+ return pd.read_csv(file_path)
349
+ elif file_path.endswith(".json"):
350
+ return pd.read_json(file_path)
351
+ else:
352
+ raise ValueError(f"Unsupported file format: {file_path}")
353
+ else:
354
+ return instance
355
+
356
+ @classmethod
357
+ def get_dataset(
358
+ cls,
359
+ dataset_name: str,
360
+ excluded_datasets: Optional[List[str]] = None,
361
+ **kwargs,
362
+ ) -> DatasetConfig:
363
+ """Get the metadata of a dataset
364
+
365
+ Retrieves metadata for a dataset, with caching support.
366
+
367
+ Args:
368
+ dataset_name (str): Name of the dataset
369
+ excluded_datasets (List[str], optional): List of dataset IDs to exclude
370
+
371
+ Kwargs:
372
+ fi_api_key (str): API key for authentication
373
+ fi_secret_key (str): Secret key for authentication
374
+ fi_base_url (str): Base URL of the API
375
+
376
+ Returns:
377
+ DatasetConfig: Metadata of the dataset
378
+
379
+ Raises:
380
+ ValueError: If no dataset or multiple datasets found
381
+ InvalidAuthError: If authentication fails
382
+ """
383
+ # Check cache and return instance if found
384
+ cache_key = f"{dataset_name}_{str(excluded_datasets)}"
385
+ if cache_key in cls._dataset_id_cache:
386
+ return cls._dataset_id_cache[cache_key]
387
+
388
+ # Get the instance of the dataset client
389
+ instance = cls._get_instance(**kwargs)
390
+
391
+ # Prepare request and data
392
+ payload = {"search_text": dataset_name}
393
+ if excluded_datasets:
394
+ payload["excluded_datasets"] = excluded_datasets
395
+ url = instance._base_url + "/" + Routes.dataset_names
396
+
397
+ # Make the request
398
+ dataset_config = instance.request(
399
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
400
+ response_handler=DatasetResponseHandler,
401
+ )
402
+ instance.dataset_config = dataset_config
403
+
404
+ # Store in cache before returning
405
+ cls._dataset_id_cache[cache_key] = instance
406
+ return instance
407
+
408
+ @classmethod
409
+ def delete_dataset(
410
+ cls, dataset_name: Optional[str] = None, **kwargs
411
+ ) -> "DatasetClient":
412
+ """Delete a dataset
413
+
414
+ Deletes a dataset by name.
415
+
416
+ Args:
417
+ dataset_name (str): Name of the dataset to delete
418
+
419
+ Kwargs:
420
+ fi_api_key (str): API key for authentication
421
+ fi_secret_key (str): Secret key for authentication
422
+ fi_base_url (str): Base URL of the API
423
+
424
+ Returns:
425
+ None
426
+
427
+ Raises:
428
+ InvalidAuthError: If authentication fails
429
+ """
430
+ # Get the dataset metadata
431
+ instance = cls.get_dataset(dataset_name, **kwargs)
432
+
433
+ # Prepare request and data
434
+ url = instance._base_url + "/" + Routes.dataset_delete.value
435
+ payload = {"dataset_ids": [str(instance.dataset_config.id)]}
436
+
437
+ # Make the request
438
+ instance.request(
439
+ config=RequestConfig(method=HttpMethod.DELETE, url=url, json=payload),
440
+ response_handler=DatasetResponseHandler,
441
+ )
442
+ return instance