futureagi 0.3.3__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of futureagi might be problematic. Click here for more details.
- {futureagi-0.3.3 → futureagi-0.3.4}/PKG-INFO +1 -1
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/auth.py +8 -3
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/types.py +2 -0
- futureagi-0.3.4/fi/datasets/__init__.py +4 -0
- futureagi-0.3.4/fi/datasets/client.py +442 -0
- futureagi-0.3.4/fi/datasets/types.py +284 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/__init__.py +28 -12
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/evaluator.py +45 -2
- futureagi-0.3.4/fi/evals/templates.py +334 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/conversational_test_case.py +0 -1
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/constants.py +7 -1
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/errors.py +3 -3
- futureagi-0.3.4/fi/utils/routes.py +17 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/types.py +6 -23
- futureagi-0.3.4/fi/utils/utils.py +29 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/pyproject.toml +1 -1
- futureagi-0.3.3/fi/evals/templates.py +0 -1060
- futureagi-0.3.3/fi/utils/routes.py +0 -16
- futureagi-0.3.3/fi/utils/utils.py +0 -10
- {futureagi-0.3.3 → futureagi-0.3.4}/LICENSE.md +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/README.md +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/api/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/client.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/evals/types.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/general.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/testcases/utils.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/__init__.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/executor.py +0 -0
- {futureagi-0.3.3 → futureagi-0.3.4}/fi/utils/logging.py +0 -0
|
@@ -20,13 +20,14 @@ from fi.utils.errors import MissingAuthError
|
|
|
20
20
|
from fi.utils.executor import BoundedExecutor
|
|
21
21
|
|
|
22
22
|
T = TypeVar("T")
|
|
23
|
+
U = TypeVar("U", default=None)
|
|
23
24
|
|
|
24
25
|
|
|
25
|
-
class ResponseHandler(Generic[T], ABC):
|
|
26
|
+
class ResponseHandler(Generic[T, U], ABC):
|
|
26
27
|
"""Handles response parsing and validation"""
|
|
27
28
|
|
|
28
29
|
@classmethod
|
|
29
|
-
def parse(cls, response: Response) -> T:
|
|
30
|
+
def parse(cls, response: Response) -> Union[T, U]:
|
|
30
31
|
"""Parse the response into the expected type"""
|
|
31
32
|
if not response.ok:
|
|
32
33
|
cls._handle_error(response)
|
|
@@ -34,7 +35,7 @@ class ResponseHandler(Generic[T], ABC):
|
|
|
34
35
|
|
|
35
36
|
@classmethod
|
|
36
37
|
@abstractmethod
|
|
37
|
-
def _parse_success(cls, response: Response) -> T:
|
|
38
|
+
def _parse_success(cls, response: Response) -> Union[T, U]:
|
|
38
39
|
"""Parse successful response"""
|
|
39
40
|
pass
|
|
40
41
|
|
|
@@ -77,6 +78,8 @@ class HttpClient:
|
|
|
77
78
|
params = config.params or {}
|
|
78
79
|
json = config.json or {}
|
|
79
80
|
timeout = config.timeout or self._default_timeout
|
|
81
|
+
files = config.files or {}
|
|
82
|
+
data = config.data or {}
|
|
80
83
|
for attempt in range(config.retry_attempts):
|
|
81
84
|
try:
|
|
82
85
|
response = self._session.request(
|
|
@@ -85,6 +88,8 @@ class HttpClient:
|
|
|
85
88
|
headers=headers,
|
|
86
89
|
params=params,
|
|
87
90
|
json=json,
|
|
91
|
+
data=data,
|
|
92
|
+
files=files,
|
|
88
93
|
timeout=timeout,
|
|
89
94
|
).result()
|
|
90
95
|
|
|
@@ -19,6 +19,8 @@ class RequestConfig(BaseModel):
|
|
|
19
19
|
url: str
|
|
20
20
|
headers: Optional[Dict[str, str]] = {}
|
|
21
21
|
params: Optional[Dict[str, Any]] = {}
|
|
22
|
+
files: Optional[Dict[str, Any]] = {}
|
|
23
|
+
data: Optional[Dict[str, Any]] = {}
|
|
22
24
|
json: Optional[Dict[str, Any]] = {}
|
|
23
25
|
timeout: Optional[int] = None
|
|
24
26
|
retry_attempts: int = 3
|
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
import os
|
|
2
|
+
from typing import List, Optional, Union
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from requests import Response
|
|
6
|
+
from tqdm import tqdm
|
|
7
|
+
|
|
8
|
+
from fi.api.auth import APIKeyAuth, ResponseHandler
|
|
9
|
+
from fi.api.types import HttpMethod, RequestConfig
|
|
10
|
+
from fi.datasets.types import (
|
|
11
|
+
Cell,
|
|
12
|
+
Column,
|
|
13
|
+
DatasetConfig,
|
|
14
|
+
DatasetTable,
|
|
15
|
+
HuggingfaceDatasetConfig,
|
|
16
|
+
Row,
|
|
17
|
+
)
|
|
18
|
+
from fi.utils.constants import (
|
|
19
|
+
DATASET_TEMP_FILE_PREFIX,
|
|
20
|
+
DATASET_TEMP_FILE_SUFFIX,
|
|
21
|
+
PAGE_SIZE,
|
|
22
|
+
)
|
|
23
|
+
from fi.utils.errors import InvalidAuthError
|
|
24
|
+
from fi.utils.routes import Routes
|
|
25
|
+
from fi.utils.utils import get_tempfile_path
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
|
|
29
|
+
|
|
30
|
+
"""Handles responses for dataset requests"""
|
|
31
|
+
|
|
32
|
+
@classmethod
|
|
33
|
+
def _parse_success(cls, response: Response) -> Union[DatasetConfig, DatasetTable]:
|
|
34
|
+
"""Parse successful response into DatasetResponse"""
|
|
35
|
+
data = response.json()
|
|
36
|
+
if response.url.endswith(Routes.dataset_names.value):
|
|
37
|
+
datasets = data["result"]["datasets"]
|
|
38
|
+
if not datasets:
|
|
39
|
+
raise ValueError("No dataset found")
|
|
40
|
+
if len(datasets) > 1:
|
|
41
|
+
raise ValueError(
|
|
42
|
+
"Multiple datasets found. Please specify a dataset name."
|
|
43
|
+
)
|
|
44
|
+
return DatasetConfig(
|
|
45
|
+
id=datasets[0]["datasetId"],
|
|
46
|
+
name=datasets[0]["name"],
|
|
47
|
+
model_type=datasets[0]["modelType"],
|
|
48
|
+
)
|
|
49
|
+
elif Routes.dataset_table.value.split("/")[-2] in response.url:
|
|
50
|
+
id = response.url.split("/")[-3]
|
|
51
|
+
columns = [
|
|
52
|
+
Column(
|
|
53
|
+
id=column["id"],
|
|
54
|
+
name=column["name"],
|
|
55
|
+
data_type=column["dataType"],
|
|
56
|
+
source=column["originType"],
|
|
57
|
+
source_id=column["sourceId"],
|
|
58
|
+
is_frozen=column["isFrozen"]["isFrozen"]
|
|
59
|
+
if column["isFrozen"] is not None
|
|
60
|
+
else False,
|
|
61
|
+
is_visible=column["isVisible"],
|
|
62
|
+
eval_tags=column["evalTag"],
|
|
63
|
+
average_score=column["averageScore"],
|
|
64
|
+
order_index=column["orderIndex"],
|
|
65
|
+
)
|
|
66
|
+
for column in data["result"]["columnConfig"]
|
|
67
|
+
]
|
|
68
|
+
rows = []
|
|
69
|
+
for row in data["result"]["table"]:
|
|
70
|
+
cells = []
|
|
71
|
+
row_id = row.pop("rowId")
|
|
72
|
+
order = row.pop("order")
|
|
73
|
+
for column_id, value in row.items():
|
|
74
|
+
cells.append(
|
|
75
|
+
Cell(
|
|
76
|
+
column_id=column_id,
|
|
77
|
+
row_id=row_id,
|
|
78
|
+
value=value.get("cellValue"),
|
|
79
|
+
value_infos=[value.get("valueInfos")]
|
|
80
|
+
if value.get("valueInfos")
|
|
81
|
+
else None,
|
|
82
|
+
metadata=value.get("metadata"),
|
|
83
|
+
status=value.get("status"),
|
|
84
|
+
failure_reason=value.get("failureReason"),
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
rows.append(Row(id=row_id, order=order, cells=cells))
|
|
88
|
+
metadata = data["result"]["metadata"]
|
|
89
|
+
return DatasetTable(id=id, columns=columns, rows=rows, metadata=metadata)
|
|
90
|
+
elif response.url.endswith(Routes.dataset_empty.value):
|
|
91
|
+
return DatasetConfig(
|
|
92
|
+
id=data["result"]["datasetId"],
|
|
93
|
+
name=data["result"]["datasetName"],
|
|
94
|
+
model_type=data["result"]["datasetModelType"],
|
|
95
|
+
)
|
|
96
|
+
elif response.url.endswith(Routes.dataset_local.value):
|
|
97
|
+
return DatasetConfig(
|
|
98
|
+
id=data["result"]["datasetId"],
|
|
99
|
+
name=data["result"]["datasetName"],
|
|
100
|
+
model_type=data["result"]["datasetModelType"],
|
|
101
|
+
)
|
|
102
|
+
elif response.url.endswith(Routes.dataset_huggingface.value):
|
|
103
|
+
return DatasetConfig(
|
|
104
|
+
id=data["result"]["datasetId"],
|
|
105
|
+
name=data["result"]["datasetName"],
|
|
106
|
+
model_type=data["result"]["datasetModelType"],
|
|
107
|
+
)
|
|
108
|
+
else:
|
|
109
|
+
return data
|
|
110
|
+
|
|
111
|
+
@classmethod
|
|
112
|
+
def _handle_error(cls, response: Response) -> None:
|
|
113
|
+
if response.status_code == 400:
|
|
114
|
+
response.raise_for_status()
|
|
115
|
+
if response.status_code == 403:
|
|
116
|
+
raise InvalidAuthError()
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
class DatasetClient(APIKeyAuth):
|
|
120
|
+
"""Manager class for handling datasets
|
|
121
|
+
|
|
122
|
+
A client for managing datasets including creation, deletion, downloading and metadata operations.
|
|
123
|
+
Provides functionality to work with empty datasets, local files, and Hugging Face datasets.
|
|
124
|
+
|
|
125
|
+
Attributes:
|
|
126
|
+
_dataset_id_cache (dict): Class-level cache for storing dataset IDs
|
|
127
|
+
dataset_config (DatasetConfig): Configuration for the current dataset
|
|
128
|
+
"""
|
|
129
|
+
|
|
130
|
+
_dataset_id_cache = {}
|
|
131
|
+
|
|
132
|
+
def __init__(
|
|
133
|
+
self,
|
|
134
|
+
fi_api_key: Optional[str] = None,
|
|
135
|
+
fi_secret_key: Optional[str] = None,
|
|
136
|
+
fi_base_url: Optional[str] = None,
|
|
137
|
+
**kwargs,
|
|
138
|
+
):
|
|
139
|
+
"""Initialize the dataset manager
|
|
140
|
+
|
|
141
|
+
Args:
|
|
142
|
+
fi_api_key (str, optional): API key for authentication
|
|
143
|
+
fi_secret_key (str, optional): Secret key for authentication
|
|
144
|
+
fi_base_url (str, optional): Base URL of the API
|
|
145
|
+
|
|
146
|
+
Kwargs:
|
|
147
|
+
dataset_config (DatasetConfig): Dataset configuration
|
|
148
|
+
|
|
149
|
+
Returns:
|
|
150
|
+
DatasetClient: Instance of DatasetClient
|
|
151
|
+
"""
|
|
152
|
+
super().__init__(
|
|
153
|
+
fi_api_key=fi_api_key,
|
|
154
|
+
fi_secret_key=fi_secret_key,
|
|
155
|
+
fi_base_url=fi_base_url,
|
|
156
|
+
**kwargs,
|
|
157
|
+
)
|
|
158
|
+
self.dataset_config = None
|
|
159
|
+
if kwargs.get("dataset_config"):
|
|
160
|
+
self.dataset_config = kwargs.get("dataset_config")
|
|
161
|
+
|
|
162
|
+
@classmethod
|
|
163
|
+
def _get_instance(
|
|
164
|
+
cls,
|
|
165
|
+
fi_api_key: Optional[str] = None,
|
|
166
|
+
fi_secret_key: Optional[str] = None,
|
|
167
|
+
fi_base_url: Optional[str] = None,
|
|
168
|
+
**kwargs,
|
|
169
|
+
) -> "DatasetClient":
|
|
170
|
+
"""Create and return an instance of DatasetClient
|
|
171
|
+
|
|
172
|
+
Factory method to create a new DatasetClient instance or return existing one.
|
|
173
|
+
|
|
174
|
+
Args:
|
|
175
|
+
fi_api_key (str, optional): API key for authentication
|
|
176
|
+
fi_secret_key (str, optional): Secret key for authentication
|
|
177
|
+
fi_base_url (str, optional): Base URL of the API
|
|
178
|
+
|
|
179
|
+
Kwargs:
|
|
180
|
+
dataset_config (DatasetConfig): Dataset configuration
|
|
181
|
+
|
|
182
|
+
Returns:
|
|
183
|
+
DatasetClient: New or existing instance of DatasetClient
|
|
184
|
+
"""
|
|
185
|
+
|
|
186
|
+
if isinstance(cls, type):
|
|
187
|
+
# Called on class
|
|
188
|
+
return cls(
|
|
189
|
+
fi_api_key=fi_api_key,
|
|
190
|
+
fi_secret_key=fi_secret_key,
|
|
191
|
+
fi_base_url=fi_base_url,
|
|
192
|
+
**kwargs,
|
|
193
|
+
)
|
|
194
|
+
else:
|
|
195
|
+
# Called on instance
|
|
196
|
+
return cls
|
|
197
|
+
|
|
198
|
+
@classmethod
|
|
199
|
+
def create_dataset(
|
|
200
|
+
cls,
|
|
201
|
+
dataset_config: Optional[DatasetConfig] = None,
|
|
202
|
+
source: Optional[Union[str, HuggingfaceDatasetConfig]] = None,
|
|
203
|
+
**kwargs,
|
|
204
|
+
) -> "DatasetClient":
|
|
205
|
+
"""Create a new dataset
|
|
206
|
+
|
|
207
|
+
Creates a new dataset either empty, from a local file, or from Hugging Face.
|
|
208
|
+
|
|
209
|
+
Args:
|
|
210
|
+
dataset_config (DatasetConfig): Configuration for the new dataset
|
|
211
|
+
source (Optional[Union[str, HuggingfaceDatasetConfig]]): Source for dataset creation.
|
|
212
|
+
- If None: Creates empty dataset
|
|
213
|
+
- If str: Path to local file (.csv, .xlsx, .json, or .jsonl)
|
|
214
|
+
- If HuggingfaceDatasetConfig: Configuration for importing from Hugging Face
|
|
215
|
+
|
|
216
|
+
Returns:
|
|
217
|
+
DatasetClient: Instance of DatasetClient with the created dataset
|
|
218
|
+
|
|
219
|
+
Raises:
|
|
220
|
+
ValueError: If file format is not supported or file cannot be processed
|
|
221
|
+
InvalidAuthError: If authentication fails
|
|
222
|
+
"""
|
|
223
|
+
instance = cls._get_instance(dataset_config=dataset_config, **kwargs)
|
|
224
|
+
|
|
225
|
+
if source is None:
|
|
226
|
+
# Create empty dataset
|
|
227
|
+
payload = {
|
|
228
|
+
"new_dataset_name": instance.dataset_config.name,
|
|
229
|
+
"model_type": instance.dataset_config.model_type.value,
|
|
230
|
+
}
|
|
231
|
+
url = instance._base_url + "/" + Routes.dataset_empty.value
|
|
232
|
+
response = instance.request(
|
|
233
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
|
|
234
|
+
response_handler=DatasetResponseHandler,
|
|
235
|
+
)
|
|
236
|
+
|
|
237
|
+
elif isinstance(source, str):
|
|
238
|
+
# Create from local file
|
|
239
|
+
supported_extensions = [".csv", ".xlsx", ".xls", ".json", ".jsonl"]
|
|
240
|
+
file_ext = os.path.splitext(source)[1].lower()
|
|
241
|
+
if file_ext not in supported_extensions:
|
|
242
|
+
raise ValueError(
|
|
243
|
+
f"Unsupported file format. File must have one of these extensions: {', '.join(supported_extensions)}"
|
|
244
|
+
)
|
|
245
|
+
|
|
246
|
+
files = {"file": (os.path.basename(source), open(source, "rb").read())}
|
|
247
|
+
data = {}
|
|
248
|
+
if instance.dataset_config.model_type:
|
|
249
|
+
data["model_type"] = instance.dataset_config.model_type.value
|
|
250
|
+
if instance.dataset_config.name:
|
|
251
|
+
data["new_dataset_name"] = instance.dataset_config.name
|
|
252
|
+
url = instance._base_url + "/" + Routes.dataset_local.value
|
|
253
|
+
|
|
254
|
+
response = instance.request(
|
|
255
|
+
config=RequestConfig(
|
|
256
|
+
method=HttpMethod.POST, url=url, data=data, files=files
|
|
257
|
+
),
|
|
258
|
+
response_handler=DatasetResponseHandler,
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
elif isinstance(source, HuggingfaceDatasetConfig):
|
|
262
|
+
# Create from Hugging Face dataset
|
|
263
|
+
data = {
|
|
264
|
+
"new_dataset_name": instance.dataset_config.name,
|
|
265
|
+
"huggingface_dataset_name": source.name,
|
|
266
|
+
}
|
|
267
|
+
if instance.dataset_config.model_type:
|
|
268
|
+
data["model_type"] = instance.dataset_config.model_type.value
|
|
269
|
+
if source.split:
|
|
270
|
+
data["huggingface_dataset_split"] = source.split
|
|
271
|
+
if source.num_rows:
|
|
272
|
+
data["num_rows"] = source.num_rows
|
|
273
|
+
url = instance._base_url + "/" + Routes.dataset_huggingface.value
|
|
274
|
+
|
|
275
|
+
response = instance.request(
|
|
276
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, data=data),
|
|
277
|
+
response_handler=DatasetResponseHandler,
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
# Update the dataset config with the new dataset ID
|
|
281
|
+
instance.dataset_config.id = response.id
|
|
282
|
+
return instance
|
|
283
|
+
|
|
284
|
+
@classmethod
|
|
285
|
+
def download_dataset(
|
|
286
|
+
cls,
|
|
287
|
+
dataset_name: Optional[str] = None,
|
|
288
|
+
file_path: Optional[str] = None,
|
|
289
|
+
load_to_pandas: Optional[bool] = False,
|
|
290
|
+
) -> Union[str, pd.DataFrame]:
|
|
291
|
+
"""Download dataset to file
|
|
292
|
+
|
|
293
|
+
Downloads a dataset to a local file or pandas DataFrame.
|
|
294
|
+
|
|
295
|
+
Args:
|
|
296
|
+
dataset_name (str, optional): Name of the dataset
|
|
297
|
+
file_path (str, optional): File path to save the dataset
|
|
298
|
+
load_to_pandas (bool, optional): Whether to return as pandas DataFrame
|
|
299
|
+
|
|
300
|
+
Returns:
|
|
301
|
+
Union[str, pd.DataFrame]: File path or pandas DataFrame containing the dataset
|
|
302
|
+
|
|
303
|
+
Raises:
|
|
304
|
+
ValueError: If file format is not supported
|
|
305
|
+
InvalidAuthError: If authentication fails
|
|
306
|
+
"""
|
|
307
|
+
# Get the dataset metadata
|
|
308
|
+
instance = cls.get_dataset(dataset_name)
|
|
309
|
+
|
|
310
|
+
# Prepare request and data
|
|
311
|
+
url = (
|
|
312
|
+
instance._base_url
|
|
313
|
+
+ "/"
|
|
314
|
+
+ Routes.dataset_table.value.format(
|
|
315
|
+
dataset_id=str(instance.dataset_config.id)
|
|
316
|
+
)
|
|
317
|
+
)
|
|
318
|
+
data = {}
|
|
319
|
+
data["page_size"] = PAGE_SIZE
|
|
320
|
+
data["current_page_index"] = 0
|
|
321
|
+
|
|
322
|
+
# Prepare file path
|
|
323
|
+
if file_path is None:
|
|
324
|
+
file_path = get_tempfile_path(
|
|
325
|
+
DATASET_TEMP_FILE_PREFIX, DATASET_TEMP_FILE_SUFFIX
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
# Make the request
|
|
329
|
+
with tqdm(desc="Downloading dataset") as pbar:
|
|
330
|
+
while True:
|
|
331
|
+
pbar.set_postfix({"page": data["current_page_index"] + 1})
|
|
332
|
+
dataset_table = instance.request(
|
|
333
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, json=data),
|
|
334
|
+
response_handler=DatasetResponseHandler,
|
|
335
|
+
)
|
|
336
|
+
_ = dataset_table.to_file(file_path)
|
|
337
|
+
data["current_page_index"] += 1
|
|
338
|
+
if (
|
|
339
|
+
dataset_table.metadata.get("totalPages")
|
|
340
|
+
== data["current_page_index"]
|
|
341
|
+
):
|
|
342
|
+
pbar.update(1)
|
|
343
|
+
break
|
|
344
|
+
|
|
345
|
+
# Load the dataset to pandas if requested
|
|
346
|
+
if load_to_pandas:
|
|
347
|
+
if file_path.endswith(".csv"):
|
|
348
|
+
return pd.read_csv(file_path)
|
|
349
|
+
elif file_path.endswith(".json"):
|
|
350
|
+
return pd.read_json(file_path)
|
|
351
|
+
else:
|
|
352
|
+
raise ValueError(f"Unsupported file format: {file_path}")
|
|
353
|
+
else:
|
|
354
|
+
return instance
|
|
355
|
+
|
|
356
|
+
@classmethod
|
|
357
|
+
def get_dataset(
|
|
358
|
+
cls,
|
|
359
|
+
dataset_name: str,
|
|
360
|
+
excluded_datasets: Optional[List[str]] = None,
|
|
361
|
+
**kwargs,
|
|
362
|
+
) -> DatasetConfig:
|
|
363
|
+
"""Get the metadata of a dataset
|
|
364
|
+
|
|
365
|
+
Retrieves metadata for a dataset, with caching support.
|
|
366
|
+
|
|
367
|
+
Args:
|
|
368
|
+
dataset_name (str): Name of the dataset
|
|
369
|
+
excluded_datasets (List[str], optional): List of dataset IDs to exclude
|
|
370
|
+
|
|
371
|
+
Kwargs:
|
|
372
|
+
fi_api_key (str): API key for authentication
|
|
373
|
+
fi_secret_key (str): Secret key for authentication
|
|
374
|
+
fi_base_url (str): Base URL of the API
|
|
375
|
+
|
|
376
|
+
Returns:
|
|
377
|
+
DatasetConfig: Metadata of the dataset
|
|
378
|
+
|
|
379
|
+
Raises:
|
|
380
|
+
ValueError: If no dataset or multiple datasets found
|
|
381
|
+
InvalidAuthError: If authentication fails
|
|
382
|
+
"""
|
|
383
|
+
# Check cache and return instance if found
|
|
384
|
+
cache_key = f"{dataset_name}_{str(excluded_datasets)}"
|
|
385
|
+
if cache_key in cls._dataset_id_cache:
|
|
386
|
+
return cls._dataset_id_cache[cache_key]
|
|
387
|
+
|
|
388
|
+
# Get the instance of the dataset client
|
|
389
|
+
instance = cls._get_instance(**kwargs)
|
|
390
|
+
|
|
391
|
+
# Prepare request and data
|
|
392
|
+
payload = {"search_text": dataset_name}
|
|
393
|
+
if excluded_datasets:
|
|
394
|
+
payload["excluded_datasets"] = excluded_datasets
|
|
395
|
+
url = instance._base_url + "/" + Routes.dataset_names
|
|
396
|
+
|
|
397
|
+
# Make the request
|
|
398
|
+
dataset_config = instance.request(
|
|
399
|
+
config=RequestConfig(method=HttpMethod.POST, url=url, json=payload),
|
|
400
|
+
response_handler=DatasetResponseHandler,
|
|
401
|
+
)
|
|
402
|
+
instance.dataset_config = dataset_config
|
|
403
|
+
|
|
404
|
+
# Store in cache before returning
|
|
405
|
+
cls._dataset_id_cache[cache_key] = instance
|
|
406
|
+
return instance
|
|
407
|
+
|
|
408
|
+
@classmethod
|
|
409
|
+
def delete_dataset(
|
|
410
|
+
cls, dataset_name: Optional[str] = None, **kwargs
|
|
411
|
+
) -> "DatasetClient":
|
|
412
|
+
"""Delete a dataset
|
|
413
|
+
|
|
414
|
+
Deletes a dataset by name.
|
|
415
|
+
|
|
416
|
+
Args:
|
|
417
|
+
dataset_name (str): Name of the dataset to delete
|
|
418
|
+
|
|
419
|
+
Kwargs:
|
|
420
|
+
fi_api_key (str): API key for authentication
|
|
421
|
+
fi_secret_key (str): Secret key for authentication
|
|
422
|
+
fi_base_url (str): Base URL of the API
|
|
423
|
+
|
|
424
|
+
Returns:
|
|
425
|
+
None
|
|
426
|
+
|
|
427
|
+
Raises:
|
|
428
|
+
InvalidAuthError: If authentication fails
|
|
429
|
+
"""
|
|
430
|
+
# Get the dataset metadata
|
|
431
|
+
instance = cls.get_dataset(dataset_name, **kwargs)
|
|
432
|
+
|
|
433
|
+
# Prepare request and data
|
|
434
|
+
url = instance._base_url + "/" + Routes.dataset_delete.value
|
|
435
|
+
payload = {"dataset_ids": [str(instance.dataset_config.id)]}
|
|
436
|
+
|
|
437
|
+
# Make the request
|
|
438
|
+
instance.request(
|
|
439
|
+
config=RequestConfig(method=HttpMethod.DELETE, url=url, json=payload),
|
|
440
|
+
response_handler=DatasetResponseHandler,
|
|
441
|
+
)
|
|
442
|
+
return instance
|