futureagi 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of futureagi might be problematic. Click here for more details.
- docs/__init__.py +0 -0
- fi/__init__.py +1 -0
- fi/bounded_executor.py +33 -0
- fi/client.py +311 -0
- fi/examples/__init__.py +0 -0
- fi/examples/rag_file.py +29 -0
- fi/utils/__init__.py +1 -0
- fi/utils/constants.py +27 -0
- fi/utils/errors.py +132 -0
- fi/utils/logging.py +47 -0
- fi/utils/types.py +215 -0
- fi/utils/utils.py +9 -0
- futureagi-0.0.1.dist-info/LICENSE.md +16 -0
- futureagi-0.0.1.dist-info/METADATA +248 -0
- futureagi-0.0.1.dist-info/RECORD +17 -0
- futureagi-0.0.1.dist-info/WHEEL +5 -0
- futureagi-0.0.1.dist-info/top_level.txt +2 -0
docs/__init__.py
ADDED
|
File without changes
|
fi/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.1"
|
fi/bounded_executor.py
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
from concurrent.futures import ThreadPoolExecutor
|
|
2
|
+
from threading import BoundedSemaphore
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class BoundedExecutor:
|
|
6
|
+
"""BoundedExecutor behaves as a ThreadPoolExecutor which will block on
|
|
7
|
+
calls to submit() once the limit given as "bound" work items are queued for
|
|
8
|
+
execution.
|
|
9
|
+
:param bound: Integer - the maximum number of items in the work queue
|
|
10
|
+
:param max_workers: Integer - the size of the thread pool
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
def __init__(self, bound, max_workers):
|
|
14
|
+
self.executor = ThreadPoolExecutor(max_workers=max_workers)
|
|
15
|
+
self.semaphore = BoundedSemaphore(bound + max_workers)
|
|
16
|
+
|
|
17
|
+
"""See concurrent.futures.Executor#submit"""
|
|
18
|
+
|
|
19
|
+
def submit(self, fn, *args, **kwargs):
|
|
20
|
+
self.semaphore.acquire()
|
|
21
|
+
try:
|
|
22
|
+
future = self.executor.submit(fn, *args, **kwargs)
|
|
23
|
+
except Exception:
|
|
24
|
+
self.semaphore.release()
|
|
25
|
+
raise
|
|
26
|
+
else:
|
|
27
|
+
future.add_done_callback(lambda x: self.semaphore.release())
|
|
28
|
+
return future
|
|
29
|
+
|
|
30
|
+
"""See concurrent.futures.Executor#shutdown"""
|
|
31
|
+
|
|
32
|
+
def shutdown(self, wait=True):
|
|
33
|
+
self.executor.shutdown(wait)
|
fi/client.py
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
1
|
+
import concurrent.futures as cf
|
|
2
|
+
import os
|
|
3
|
+
import time
|
|
4
|
+
from typing import Dict, Optional, Union, List
|
|
5
|
+
|
|
6
|
+
from requests_futures.sessions import FuturesSession
|
|
7
|
+
|
|
8
|
+
from fi.bounded_executor import BoundedExecutor
|
|
9
|
+
from fi.utils.constants import (
|
|
10
|
+
API_KEY_ENVVAR_NAME,
|
|
11
|
+
MAX_FUTURE_YEARS_FROM_CURRENT_TIME,
|
|
12
|
+
MAX_NUMBER_OF_EMBEDDINGS,
|
|
13
|
+
MAX_PAST_YEARS_FROM_CURRENT_TIME,
|
|
14
|
+
SECRET_KEY_ENVVAR_NAME,
|
|
15
|
+
MAX_EMBEDDING_DIMENSIONALITY,
|
|
16
|
+
)
|
|
17
|
+
from fi.utils.errors import (
|
|
18
|
+
AuthError,
|
|
19
|
+
InvalidAdditionalHeaders,
|
|
20
|
+
InvalidNumberOfEmbeddings,
|
|
21
|
+
InvalidValueType,
|
|
22
|
+
InvalidSupportedType,
|
|
23
|
+
MissingRequiredKey,
|
|
24
|
+
InvalidVectorLength,
|
|
25
|
+
)
|
|
26
|
+
from fi.utils.types import Embedding, Environments, ModelTypes
|
|
27
|
+
from fi.utils.utils import is_timestamp_in_range
|
|
28
|
+
|
|
29
|
+
from fi.utils.logging import logger
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class Client:
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
api_key,
|
|
36
|
+
secret_key,
|
|
37
|
+
uri="http://localhost:8000",
|
|
38
|
+
max_workers=8,
|
|
39
|
+
max_queue_bound=5000,
|
|
40
|
+
timeout=200,
|
|
41
|
+
additional_headers=None,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""
|
|
44
|
+
Initializes the Fi Client
|
|
45
|
+
:param api_key: provided API key associated with your account.
|
|
46
|
+
:param secret_key:provided identifier to connect records to spaces.
|
|
47
|
+
:param uri: RI to send your records to Fi AI..
|
|
48
|
+
:param max_workers: maximum number of concurrent requests to Fi. Defaults
|
|
49
|
+
to 8.
|
|
50
|
+
:param max_queue_bound: maximum number of concurrent future objects
|
|
51
|
+
generated for publishing to Fi. Defaults to 5000.
|
|
52
|
+
:param timeout: how long to wait for the server to send data before giving
|
|
53
|
+
up. Defaults to 200.
|
|
54
|
+
:param additional_headers: Dictionary of additional headers to
|
|
55
|
+
append to request
|
|
56
|
+
"""
|
|
57
|
+
api_key = api_key or os.getenv(API_KEY_ENVVAR_NAME)
|
|
58
|
+
secret_key = secret_key or os.getenv(SECRET_KEY_ENVVAR_NAME)
|
|
59
|
+
if api_key is None or secret_key is None:
|
|
60
|
+
raise AuthError(api_key, secret_key)
|
|
61
|
+
self._uri_event = f"{uri}/log/event/"
|
|
62
|
+
self._uri_model = f"{uri}/log/model/"
|
|
63
|
+
self._timeout = timeout
|
|
64
|
+
self._session = FuturesSession(
|
|
65
|
+
executor=BoundedExecutor(max_queue_bound, max_workers)
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
self._headers = {
|
|
69
|
+
"X-Api-Key": api_key,
|
|
70
|
+
"X-Secret-Key": secret_key,
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if additional_headers is not None:
|
|
74
|
+
conflicting_keys = self._headers.keys() & additional_headers.keys()
|
|
75
|
+
if conflicting_keys:
|
|
76
|
+
raise InvalidAdditionalHeaders(conflicting_keys)
|
|
77
|
+
self._headers.update(additional_headers)
|
|
78
|
+
|
|
79
|
+
def _now(self):
|
|
80
|
+
return time.time()
|
|
81
|
+
|
|
82
|
+
def __validate_params(
|
|
83
|
+
self,
|
|
84
|
+
model_id: str,
|
|
85
|
+
model_type: ModelTypes,
|
|
86
|
+
environment: Environments,
|
|
87
|
+
model_version: Optional[str] = None,
|
|
88
|
+
prediction_timestamp: Optional[int] = None,
|
|
89
|
+
conversation: Optional[Dict[str, Union[str, bool, float, int]]] = None,
|
|
90
|
+
tags: Optional[Dict[str, Union[str, bool, float, int]]] = None,
|
|
91
|
+
):
|
|
92
|
+
"""
|
|
93
|
+
Validates input parameters against specified standards.
|
|
94
|
+
:param model_id:
|
|
95
|
+
:param model_type:
|
|
96
|
+
:param environment:
|
|
97
|
+
:param model_version:
|
|
98
|
+
:param prediction_timestamp:
|
|
99
|
+
:param conversation:
|
|
100
|
+
:param tags:
|
|
101
|
+
"""
|
|
102
|
+
# Validate model id
|
|
103
|
+
if not isinstance(model_id, str):
|
|
104
|
+
raise InvalidValueType("model_id", model_id, "str")
|
|
105
|
+
|
|
106
|
+
# Validate model type
|
|
107
|
+
if not isinstance(model_type, ModelTypes):
|
|
108
|
+
raise InvalidValueType(
|
|
109
|
+
"model_type", model_type, "fi.utils.types.ModelTypes"
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
# Validate supported model types
|
|
113
|
+
if model_type not in [ModelTypes.GENERATIVE_LLM, ModelTypes.GENERATIVE_IMAGE]:
|
|
114
|
+
raise InvalidSupportedType(
|
|
115
|
+
"model_type",
|
|
116
|
+
model_type,
|
|
117
|
+
"ModelTypes.GENERATIVE_LLM,ModelTypes.GENERATIVE_IMAGE",
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
# Validate environment
|
|
121
|
+
if not isinstance(environment, Environments):
|
|
122
|
+
raise InvalidValueType(
|
|
123
|
+
"environment", environment, "fi.utils.types.Environments"
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
# Validate model_version
|
|
127
|
+
if model_version:
|
|
128
|
+
if not isinstance(model_version, str):
|
|
129
|
+
raise InvalidValueType("model_version", model_version, "str")
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# Validate feature types
|
|
133
|
+
if conversation:
|
|
134
|
+
if isinstance(conversation, dict):
|
|
135
|
+
if "chat_history" not in conversation and "chat_graph" not in conversation:
|
|
136
|
+
raise MissingRequiredKey("conversation", "[chat_history, chat_graph]")
|
|
137
|
+
|
|
138
|
+
if "chat_history" in conversation:
|
|
139
|
+
chat_history = conversation["chat_history"]
|
|
140
|
+
if not isinstance(chat_history, list):
|
|
141
|
+
raise InvalidValueType(
|
|
142
|
+
"conversation['chat_history']", chat_history, "list"
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
for item in chat_history:
|
|
146
|
+
if not isinstance(item, dict):
|
|
147
|
+
raise InvalidValueType(
|
|
148
|
+
"conversation['chat_history'] item", item, "dict"
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
required_keys = ["role", "content"]
|
|
152
|
+
for key in required_keys:
|
|
153
|
+
if key not in item:
|
|
154
|
+
raise MissingRequiredKey(
|
|
155
|
+
"conversation['chat_history'] item", key
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
if not isinstance(item["role"], str):
|
|
159
|
+
raise InvalidValueType(
|
|
160
|
+
"conversation['chat_history']['role']", item["role"], "str"
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
if "chat_graph" in conversation:
|
|
164
|
+
feature = conversation["chat_graph"]
|
|
165
|
+
required_keys = ["conversation_id", "nodes"]
|
|
166
|
+
# required_keys = ["conversation_id", "title", "root_node", "metadata", "nodes"]
|
|
167
|
+
for key in required_keys:
|
|
168
|
+
if key not in feature:
|
|
169
|
+
raise MissingRequiredKey("conversation item", key)
|
|
170
|
+
|
|
171
|
+
if not isinstance(feature["nodes"], list):
|
|
172
|
+
raise InvalidValueType(
|
|
173
|
+
"conversation['nodes']", feature["nodes"], "list"
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
for node in feature["nodes"]:
|
|
177
|
+
node_required_keys = ["message"]
|
|
178
|
+
# node_required_keys = ["parent_node", "child_node", "message"]
|
|
179
|
+
for key in node_required_keys:
|
|
180
|
+
if key not in node:
|
|
181
|
+
raise MissingRequiredKey("conversation['nodes'] item", key)
|
|
182
|
+
|
|
183
|
+
if not isinstance(node["message"], dict):
|
|
184
|
+
raise InvalidValueType(
|
|
185
|
+
"conversation['nodes']['message']", node["message"], "dict"
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
message_required_keys = ["id", "author", "content", "context"]
|
|
189
|
+
for key in message_required_keys:
|
|
190
|
+
if key not in node["message"]:
|
|
191
|
+
raise MissingRequiredKey(
|
|
192
|
+
"conversation['nodes']['message']", key
|
|
193
|
+
)
|
|
194
|
+
|
|
195
|
+
author_required_keys = ["role", "metadata"]
|
|
196
|
+
for key in author_required_keys:
|
|
197
|
+
if key not in node["message"]["author"]:
|
|
198
|
+
raise MissingRequiredKey(
|
|
199
|
+
"conversation['nodes']['message']['author']", key
|
|
200
|
+
)
|
|
201
|
+
|
|
202
|
+
if node["message"]["author"]["role"] not in [
|
|
203
|
+
"assistant",
|
|
204
|
+
"user",
|
|
205
|
+
"system",
|
|
206
|
+
]:
|
|
207
|
+
raise InvalidValueType(
|
|
208
|
+
"conversation['nodes']['message']['author']['role']",
|
|
209
|
+
node["message"]["author"]["role"],
|
|
210
|
+
"str",
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
content_required_keys = ["content_type", "parts"]
|
|
214
|
+
for key in content_required_keys:
|
|
215
|
+
if key not in node["message"]["content"]:
|
|
216
|
+
raise MissingRequiredKey(
|
|
217
|
+
"conversation['nodes']['message']['content']", key
|
|
218
|
+
)
|
|
219
|
+
|
|
220
|
+
if not isinstance(node["message"]["content"]["parts"], list):
|
|
221
|
+
raise InvalidValueType(
|
|
222
|
+
"conversation['nodes']['message']['content']['parts']",
|
|
223
|
+
node["message"]["content"]["parts"],
|
|
224
|
+
"list",
|
|
225
|
+
)
|
|
226
|
+
else:
|
|
227
|
+
raise InvalidValueType("conversation", conversation, "dict or list")
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
# Validate tags type
|
|
231
|
+
if tags:
|
|
232
|
+
if not isinstance(tags, dict):
|
|
233
|
+
raise InvalidValueType("tags", tags, "dict")
|
|
234
|
+
for key, value in tags.items():
|
|
235
|
+
if not isinstance(key, str):
|
|
236
|
+
raise InvalidValueType("tags key", key, "str")
|
|
237
|
+
|
|
238
|
+
# Check the timestamp present on the event
|
|
239
|
+
if prediction_timestamp is not None:
|
|
240
|
+
if not isinstance(prediction_timestamp, int):
|
|
241
|
+
raise InvalidValueType(
|
|
242
|
+
"prediction_timestamp", prediction_timestamp, "int"
|
|
243
|
+
)
|
|
244
|
+
# Send warning if prediction is sent with future timestamp
|
|
245
|
+
now = int(time.time())
|
|
246
|
+
if prediction_timestamp > now:
|
|
247
|
+
logger.warning(
|
|
248
|
+
"Caution when sending a prediction with future timestamp."
|
|
249
|
+
"fi only stores 2 years worth of data. For example, if you sent a prediction "
|
|
250
|
+
"to fi from 1.5 years ago, and now send a prediction with timestamp of a year in "
|
|
251
|
+
"the future, the oldest 0.5 years will be dropped to maintain the 2 years worth of data "
|
|
252
|
+
"requirement."
|
|
253
|
+
)
|
|
254
|
+
if not is_timestamp_in_range(now, prediction_timestamp):
|
|
255
|
+
raise ValueError(
|
|
256
|
+
f"prediction_timestamp: {prediction_timestamp} is out of range."
|
|
257
|
+
f"Prediction timestamps must be within {MAX_FUTURE_YEARS_FROM_CURRENT_TIME} year in the "
|
|
258
|
+
f"future and {MAX_PAST_YEARS_FROM_CURRENT_TIME} years in the past from the current time."
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
def log(
|
|
262
|
+
self,
|
|
263
|
+
model_id: str,
|
|
264
|
+
model_type: ModelTypes,
|
|
265
|
+
environment: Environments,
|
|
266
|
+
model_version: Optional[str] = None,
|
|
267
|
+
prediction_timestamp: Optional[int] = None,
|
|
268
|
+
conversation: Optional[Dict[str, Union[str, bool, float, int]]] = None,
|
|
269
|
+
tags: Optional[Dict[str, Union[str, bool, float, int]]] = None,
|
|
270
|
+
) -> cf.Future:
|
|
271
|
+
"""
|
|
272
|
+
Check `readme.md` for details on these parameters.
|
|
273
|
+
|
|
274
|
+
:param model_id:
|
|
275
|
+
:param model_type:
|
|
276
|
+
:param environment:
|
|
277
|
+
:param model_version:
|
|
278
|
+
:param prediction_timestamp:
|
|
279
|
+
:param conversation:
|
|
280
|
+
:param tags:
|
|
281
|
+
:return:
|
|
282
|
+
"""
|
|
283
|
+
self.__validate_params(
|
|
284
|
+
model_id=model_id,
|
|
285
|
+
model_type=model_type,
|
|
286
|
+
environment=environment,
|
|
287
|
+
model_version=model_version,
|
|
288
|
+
prediction_timestamp=prediction_timestamp,
|
|
289
|
+
conversation=conversation,
|
|
290
|
+
tags=tags,
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
record = {
|
|
294
|
+
"model_id": model_id,
|
|
295
|
+
"model_type": model_type.value,
|
|
296
|
+
"environment": environment.value,
|
|
297
|
+
"model_version": model_version,
|
|
298
|
+
"prediction_timestamp": prediction_timestamp,
|
|
299
|
+
"conversation": conversation,
|
|
300
|
+
"tags": tags,
|
|
301
|
+
}
|
|
302
|
+
return self._post(record=record, uri=self._uri_model)
|
|
303
|
+
|
|
304
|
+
def _post(self, record, uri):
|
|
305
|
+
resp = self._session.post(
|
|
306
|
+
uri,
|
|
307
|
+
headers=self._headers,
|
|
308
|
+
timeout=self._timeout,
|
|
309
|
+
json=record,
|
|
310
|
+
)
|
|
311
|
+
return resp
|
fi/examples/__init__.py
ADDED
|
File without changes
|
fi/examples/rag_file.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from fi.client import Client
|
|
3
|
+
|
|
4
|
+
from fi.client import ModelTypes, Environments
|
|
5
|
+
|
|
6
|
+
fi_client = Client(
|
|
7
|
+
api_key="5ffab106598343a79720824d380632ce",
|
|
8
|
+
secret_key="faf78735c36c459f8f6be81823950f9d"
|
|
9
|
+
)
|
|
10
|
+
|
|
11
|
+
response = fi_client.log(
|
|
12
|
+
"sZZTSGXIFi",
|
|
13
|
+
ModelTypes.GENERATIVE_LLM,
|
|
14
|
+
Environments.PRODUCTION,
|
|
15
|
+
"1.2",
|
|
16
|
+
int(time.time()),
|
|
17
|
+
{
|
|
18
|
+
"chat_history": [
|
|
19
|
+
{"role": "user", "content": "What is the capital of France?", "context": [
|
|
20
|
+
["France is a country in Europe.", " The capital of France is Paris."],
|
|
21
|
+
["Germany is a country in Europe.", " The capital of Germany is Berlin."],
|
|
22
|
+
["Italy is a country in Europe.", " The capital of Italy is Rome."]
|
|
23
|
+
]},
|
|
24
|
+
{"role": "assistant",
|
|
25
|
+
"content": "Paris"}
|
|
26
|
+
]
|
|
27
|
+
}
|
|
28
|
+
).result()
|
|
29
|
+
print(response)
|
fi/utils/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.0.1"
|
fi/utils/constants.py
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
|
|
2
|
+
|
|
3
|
+
MIN_PREDICTION_ID_LEN = 1
|
|
4
|
+
MAX_PREDICTION_ID_LEN = 128
|
|
5
|
+
|
|
6
|
+
# The maximum number of character for tag values
|
|
7
|
+
MAX_TAG_LENGTH = 20_000
|
|
8
|
+
MAX_TAG_LENGTH_TRUNCATION = 1_000
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
MAX_PAST_YEARS_FROM_CURRENT_TIME = 5
|
|
12
|
+
MAX_FUTURE_YEARS_FROM_CURRENT_TIME = 1
|
|
13
|
+
|
|
14
|
+
MAX_RAW_DATA_CHARACTERS = 50_000
|
|
15
|
+
MAX_RAW_DATA_CHARACTERS_TRUNCATION = 5_000
|
|
16
|
+
|
|
17
|
+
# The maximum number of embeddings
|
|
18
|
+
MAX_NUMBER_OF_EMBEDDINGS = 30
|
|
19
|
+
MAX_EMBEDDING_DIMENSIONALITY = 20_000
|
|
20
|
+
|
|
21
|
+
RESERVED_TAG_COLS = [
|
|
22
|
+
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
# Authentication via environment variables
|
|
26
|
+
SECRET_KEY_ENVVAR_NAME = "TFC_SECRET_KEY"
|
|
27
|
+
API_KEY_ENVVAR_NAME = "TFC_API_KEY"
|
fi/utils/errors.py
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
from typing import Optional, Union
|
|
2
|
+
from abc import ABC, abstractmethod
|
|
3
|
+
from collections.abc import Iterable
|
|
4
|
+
|
|
5
|
+
from .constants import API_KEY_ENVVAR_NAME, MAX_NUMBER_OF_EMBEDDINGS, SECRET_KEY_ENVVAR_NAME
|
|
6
|
+
from .types import ModelTypes, Environments
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ValidationError(Exception, ABC):
|
|
10
|
+
def __str__(self) -> str:
|
|
11
|
+
return self.error_message()
|
|
12
|
+
|
|
13
|
+
@abstractmethod
|
|
14
|
+
def __repr__(self) -> str:
|
|
15
|
+
pass
|
|
16
|
+
|
|
17
|
+
@abstractmethod
|
|
18
|
+
def error_message(self) -> str:
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class AuthError(Exception):
|
|
23
|
+
def __init__(self, api_key: Optional[str], space_key: Optional[str]) -> None:
|
|
24
|
+
self.missing_api_key = api_key is None
|
|
25
|
+
self.missing_space_key = space_key is None
|
|
26
|
+
|
|
27
|
+
def __repr__(self) -> str:
|
|
28
|
+
return "Invalid_TFC_Client_Authentication"
|
|
29
|
+
|
|
30
|
+
def __str__(self) -> str:
|
|
31
|
+
return self.error_message()
|
|
32
|
+
|
|
33
|
+
def error_message(self) -> str:
|
|
34
|
+
missing_list = ["api_key"] if self.missing_api_key else []
|
|
35
|
+
if self.missing_space_key:
|
|
36
|
+
missing_list.append("space_key")
|
|
37
|
+
|
|
38
|
+
return (
|
|
39
|
+
"TFC Client could not obtain credentials. You can pass your api_key and space_key "
|
|
40
|
+
"directly to the TFC Client, or you can set environment variables which will be read if the "
|
|
41
|
+
"keys are not directly passed. "
|
|
42
|
+
"To set the environment variables use the following variable names: \n"
|
|
43
|
+
f" - {API_KEY_ENVVAR_NAME} for the api key\n"
|
|
44
|
+
f" - {SECRET_KEY_ENVVAR_NAME} for the secret key\n"
|
|
45
|
+
f"Missing: {missing_list}"
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class InvalidAdditionalHeaders(ValidationError):
|
|
50
|
+
def __repr__(self) -> str:
|
|
51
|
+
return "Invalid_Additional_Headers"
|
|
52
|
+
|
|
53
|
+
def __init__(self, invalid_headers: Iterable) -> None:
|
|
54
|
+
self.invalid_header_names = invalid_headers
|
|
55
|
+
|
|
56
|
+
def error_message(self) -> str:
|
|
57
|
+
return (
|
|
58
|
+
"Found invalid additional header, cannot use reserved headers named: "
|
|
59
|
+
f"{', '.join(map(str, self.invalid_header_names))}."
|
|
60
|
+
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class InvalidNumberOfEmbeddings(ValidationError):
|
|
64
|
+
def __repr__(self) -> str:
|
|
65
|
+
return "Invalid_Number_Of_Embeddings"
|
|
66
|
+
|
|
67
|
+
def __init__(self, number_of_embeddings: int) -> None:
|
|
68
|
+
self.number_of_embeddings = number_of_embeddings
|
|
69
|
+
|
|
70
|
+
def error_message(self) -> str:
|
|
71
|
+
return (
|
|
72
|
+
f"The schema contains {self.number_of_embeddings} different embeddings when a maximum of "
|
|
73
|
+
f"{MAX_NUMBER_OF_EMBEDDINGS} is allowed."
|
|
74
|
+
)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class InvalidValueType(Exception):
|
|
78
|
+
def __init__(
|
|
79
|
+
self,
|
|
80
|
+
value_name: str,
|
|
81
|
+
value: Union[bool, int, float, str],
|
|
82
|
+
correct_type: str,
|
|
83
|
+
) -> None:
|
|
84
|
+
self.value_name = value_name
|
|
85
|
+
self.value = value
|
|
86
|
+
self.correct_type = correct_type
|
|
87
|
+
|
|
88
|
+
def __repr__(self) -> str:
|
|
89
|
+
return "Invalid_Value_Type"
|
|
90
|
+
|
|
91
|
+
def __str__(self) -> str:
|
|
92
|
+
return self.error_message()
|
|
93
|
+
|
|
94
|
+
def error_message(self) -> str:
|
|
95
|
+
return (
|
|
96
|
+
f"{self.value_name} with value {self.value} is of type {type(self.value).__name__}, "
|
|
97
|
+
f"but expected From {self.correct_type}"
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class InvalidSupportedType(Exception):
|
|
102
|
+
def __init__(
|
|
103
|
+
self,
|
|
104
|
+
value_name: str,
|
|
105
|
+
value: Union[ModelTypes, Environments],
|
|
106
|
+
correct_type: str,
|
|
107
|
+
) -> None:
|
|
108
|
+
self.value_name = value_name
|
|
109
|
+
self.value = value
|
|
110
|
+
self.correct_type = correct_type
|
|
111
|
+
|
|
112
|
+
def __repr__(self) -> str:
|
|
113
|
+
return "Invalid_Value_Type"
|
|
114
|
+
|
|
115
|
+
def __str__(self) -> str:
|
|
116
|
+
return self.error_message()
|
|
117
|
+
|
|
118
|
+
def error_message(self) -> str:
|
|
119
|
+
return (
|
|
120
|
+
f"{self.value_name} with value {self.value} is noy supported as of now, "
|
|
121
|
+
f"supported model types are {self.correct_type}"
|
|
122
|
+
)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class MissingRequiredKey(Exception):
|
|
126
|
+
def __init__(self, field_name, missing_key):
|
|
127
|
+
super().__init__(f"Missing required key '{missing_key}' in {field_name}.")
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class InvalidVectorLength(Exception):
|
|
131
|
+
def __init__(self, field_name, length, max_length):
|
|
132
|
+
super().__init__(f"Invalid vector length for {field_name}: maximum length is {max_length}, got {length}.")
|
fi/utils/logging.py
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import sys
|
|
3
|
+
|
|
4
|
+
class CustomLogFormatter(logging.Formatter):
|
|
5
|
+
grey = "\x1b[38;21m"
|
|
6
|
+
blue = "\x1b[38;5;39m"
|
|
7
|
+
yellow = "\x1b[33m"
|
|
8
|
+
red = "\x1b[38;5;196m"
|
|
9
|
+
bold_red = "\x1b[31;1m"
|
|
10
|
+
reset = "\x1b[0m"
|
|
11
|
+
|
|
12
|
+
def __init__(self, fmt):
|
|
13
|
+
super().__init__()
|
|
14
|
+
self.fmt = fmt
|
|
15
|
+
self.FORMATS = {
|
|
16
|
+
logging.DEBUG: self.blue + self.fmt + self.reset,
|
|
17
|
+
logging.INFO: self.grey + self.fmt + self.reset,
|
|
18
|
+
logging.WARNING: self.yellow + self.fmt + self.reset,
|
|
19
|
+
logging.ERROR: self.red + self.fmt + self.reset,
|
|
20
|
+
logging.CRITICAL: self.bold_red + self.fmt + self.reset,
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
def format(self, record):
|
|
24
|
+
log_fmt = self.FORMATS.get(record.levelno)
|
|
25
|
+
formatter = logging.Formatter(log_fmt)
|
|
26
|
+
return formatter.format(record)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
logger = logging.getLogger(__name__)
|
|
30
|
+
logger.propagate = False
|
|
31
|
+
if logger.hasHandlers():
|
|
32
|
+
logger.handlers.clear()
|
|
33
|
+
logger.setLevel(logging.INFO)
|
|
34
|
+
fmt = " %(name)s | %(levelname)s | %(message)s"
|
|
35
|
+
if hasattr(sys, "ps1"): # for python interactive mode
|
|
36
|
+
handler = logging.StreamHandler(sys.stdout)
|
|
37
|
+
handler.setLevel(logging.INFO)
|
|
38
|
+
handler.setFormatter(CustomLogFormatter(fmt))
|
|
39
|
+
logger.addHandler(handler)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def get_truncation_warning_message(instance, limit) -> str:
|
|
43
|
+
return (
|
|
44
|
+
f"Attention: {instance} exceeding the {limit} character limit will be "
|
|
45
|
+
"automatically truncated upon ingestion into the fi platform. Should you require "
|
|
46
|
+
"a higher limit, please reach out to our support team at support@fi.com"
|
|
47
|
+
)
|
fi/utils/types.py
ADDED
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
from enum import Enum, unique
|
|
2
|
+
from typing import List, NamedTuple, Optional, Sequence, TypeVar, Union, Type
|
|
3
|
+
|
|
4
|
+
import pandas as pd
|
|
5
|
+
from fi.utils.logging import get_truncation_warning_message, logger
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from fi.utils.constants import MAX_RAW_DATA_CHARACTERS, MAX_RAW_DATA_CHARACTERS_TRUNCATION
|
|
10
|
+
|
|
11
|
+
@unique
|
|
12
|
+
class ModelTypes(Enum):
|
|
13
|
+
NUMERIC = 1
|
|
14
|
+
SCORE_CATEGORICAL = 2 # Multi class classification, nlp, image classification
|
|
15
|
+
RANKING = 3
|
|
16
|
+
BINARY_CLASSIFICATION = 4 # single class classification
|
|
17
|
+
REGRESSION = 5 # regression, timeseries
|
|
18
|
+
OBJECT_DETECTION = 6
|
|
19
|
+
SEGMENTATION = 7
|
|
20
|
+
GENERATIVE_LLM = 8
|
|
21
|
+
GENERATIVE_IMAGE = 9
|
|
22
|
+
GENERATIVE_VIDEO = 10
|
|
23
|
+
TTS = 11
|
|
24
|
+
STT = 12
|
|
25
|
+
MULTI_MODAL = 13
|
|
26
|
+
|
|
27
|
+
@classmethod
|
|
28
|
+
def list_types(cls):
|
|
29
|
+
return [t.name for t in cls]
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
NUMERIC_MODEL_TYPES = [ModelTypes.NUMERIC, ModelTypes.REGRESSION]
|
|
33
|
+
CATEGORICAL_MODEL_TYPES = [
|
|
34
|
+
ModelTypes.SCORE_CATEGORICAL,
|
|
35
|
+
ModelTypes.BINARY_CLASSIFICATION,
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@unique
|
|
40
|
+
class Environments(Enum):
|
|
41
|
+
TRAINING = 1
|
|
42
|
+
VALIDATION = 2
|
|
43
|
+
PRODUCTION = 3
|
|
44
|
+
CORPUS = 4
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Embedding(NamedTuple):
|
|
48
|
+
vector: List[float]
|
|
49
|
+
data: Optional[Union[str, List[str]]] = None
|
|
50
|
+
link_to_data: Optional[str] = None
|
|
51
|
+
|
|
52
|
+
def validate(self, emb_name: Union[str, int, float]) -> None:
|
|
53
|
+
"""
|
|
54
|
+
Validates that the embedding object passed is of the correct format. That is, validations must
|
|
55
|
+
be passed for vector, data & link_to_data.
|
|
56
|
+
|
|
57
|
+
Arguments:
|
|
58
|
+
----------
|
|
59
|
+
emb_name (str, int, float): Name of the embedding feature the vector belongs to
|
|
60
|
+
|
|
61
|
+
Raises:
|
|
62
|
+
-------
|
|
63
|
+
TypeError: If the embedding fields are of the wrong type
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
if self.vector is not None:
|
|
67
|
+
self._validate_embedding_vector(emb_name)
|
|
68
|
+
|
|
69
|
+
# Validate embedding raw data, if present
|
|
70
|
+
if self.data is not None:
|
|
71
|
+
self._validate_embedding_data(emb_name, self.data)
|
|
72
|
+
|
|
73
|
+
# Validate embedding link to data, if present
|
|
74
|
+
if self.link_to_data is not None:
|
|
75
|
+
self._validate_embedding_link_to_data(emb_name, self.link_to_data)
|
|
76
|
+
|
|
77
|
+
return None
|
|
78
|
+
|
|
79
|
+
def _validate_embedding_vector(
|
|
80
|
+
self,
|
|
81
|
+
emb_name: Union[str, int, float],
|
|
82
|
+
) -> None:
|
|
83
|
+
"""
|
|
84
|
+
Validates that the embedding vector passed is of the correct format. That is:
|
|
85
|
+
1. Type must be list or convertible to list (like numpy arrays, pandas Series)
|
|
86
|
+
2. List must not be empty
|
|
87
|
+
3. Elements in list must be floats
|
|
88
|
+
|
|
89
|
+
Arguments:
|
|
90
|
+
----------
|
|
91
|
+
emb_name (str, int, float): Name of the embedding feature the vector belongs to
|
|
92
|
+
|
|
93
|
+
Raises:
|
|
94
|
+
-------
|
|
95
|
+
TypeError: If the embedding does not satisfy requirements above
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
if not Embedding._is_valid_iterable(self.vector):
|
|
99
|
+
raise TypeError(
|
|
100
|
+
f'Embedding feature "{emb_name}" has vector type {type(self.vector)}. Must be '
|
|
101
|
+
f"list, "
|
|
102
|
+
f"np.ndarray or pd.Series"
|
|
103
|
+
)
|
|
104
|
+
# Fail if not all elements in list are floats
|
|
105
|
+
allowed_types = (int, float, np.int16, np.int32, np.float16, np.float32)
|
|
106
|
+
if not all(isinstance(val, allowed_types) for val in self.vector): # type: ignore
|
|
107
|
+
raise TypeError(
|
|
108
|
+
f"Embedding vector must be a vector of integers and/or floats. Got "
|
|
109
|
+
f"{emb_name}.vector = {self.vector}"
|
|
110
|
+
)
|
|
111
|
+
# Fail if the length of the vector is 1
|
|
112
|
+
if len(self.vector) == 1:
|
|
113
|
+
raise ValueError("Embedding vector must not have a size of 1")
|
|
114
|
+
|
|
115
|
+
@staticmethod
|
|
116
|
+
def _validate_embedding_data(
|
|
117
|
+
emb_name: Union[str, int, float], data: Union[str, List[str]]
|
|
118
|
+
) -> None:
|
|
119
|
+
"""
|
|
120
|
+
Validates that the embedding raw data field is of the correct format. That is:
|
|
121
|
+
1. Must be string or list of strings (NLP case)
|
|
122
|
+
|
|
123
|
+
Arguments:
|
|
124
|
+
----------
|
|
125
|
+
emb_name (str, int, float): Name of the embedding feature the vector belongs to
|
|
126
|
+
data (str, int, float): Raw data associated with the embedding feature. Typically raw text.
|
|
127
|
+
|
|
128
|
+
Raises:
|
|
129
|
+
-------
|
|
130
|
+
TypeError: If the embedding does not satisfy requirements above
|
|
131
|
+
"""
|
|
132
|
+
# Validate that data is a string or iterable of strings
|
|
133
|
+
is_string = isinstance(data, str)
|
|
134
|
+
is_allowed_iterable = not is_string and Embedding._is_valid_iterable(data)
|
|
135
|
+
if not (is_string or is_allowed_iterable):
|
|
136
|
+
raise TypeError(
|
|
137
|
+
f'Embedding feature "{emb_name}" data field must be str, list, np.ndarray or '
|
|
138
|
+
f"pd.Series"
|
|
139
|
+
)
|
|
140
|
+
|
|
141
|
+
if is_allowed_iterable:
|
|
142
|
+
# Fail if not all elements in iterable are strings
|
|
143
|
+
if not all(isinstance(val, str) for val in data):
|
|
144
|
+
raise TypeError("Embedding data field must contain strings")
|
|
145
|
+
|
|
146
|
+
character_count = count_characters_raw_data(data)
|
|
147
|
+
if character_count > MAX_RAW_DATA_CHARACTERS:
|
|
148
|
+
raise ValueError(
|
|
149
|
+
f"Embedding data field must not contain more than {MAX_RAW_DATA_CHARACTERS} characters. "
|
|
150
|
+
f"Found {character_count}."
|
|
151
|
+
)
|
|
152
|
+
elif character_count > MAX_RAW_DATA_CHARACTERS_TRUNCATION:
|
|
153
|
+
logger.warning(
|
|
154
|
+
get_truncation_warning_message(
|
|
155
|
+
"Embedding raw data fields", MAX_RAW_DATA_CHARACTERS_TRUNCATION
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def _validate_embedding_link_to_data(
|
|
161
|
+
emb_name: Union[str, int, float], link_to_data: str
|
|
162
|
+
) -> None:
|
|
163
|
+
"""
|
|
164
|
+
Validates that the embedding link to data field is of the correct format. That is:
|
|
165
|
+
1. Must be string
|
|
166
|
+
|
|
167
|
+
Arguments:
|
|
168
|
+
----------
|
|
169
|
+
emb_name (str, int, float): Name of the embedding feature the vector belongs to
|
|
170
|
+
link_to_data (str): Link to source data of embedding feature, typically an image file on
|
|
171
|
+
cloud storage
|
|
172
|
+
|
|
173
|
+
Raises:
|
|
174
|
+
-------
|
|
175
|
+
TypeError: If the embedding does not satisfy requirements above
|
|
176
|
+
"""
|
|
177
|
+
if not isinstance(link_to_data, str):
|
|
178
|
+
raise TypeError(
|
|
179
|
+
f'Embedding feature "{emb_name}" link_to_data field must be str and got '
|
|
180
|
+
f"{type(link_to_data)}"
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
@staticmethod
|
|
184
|
+
def _is_valid_iterable(data: Union[str, List[str], List[float], np.ndarray, pd.Series]) -> bool:
|
|
185
|
+
"""
|
|
186
|
+
Validates that the input data field is of the correct iterable type. That is:
|
|
187
|
+
1. List or
|
|
188
|
+
2. numpy array or
|
|
189
|
+
3. pandas Series
|
|
190
|
+
|
|
191
|
+
Arguments:
|
|
192
|
+
----------
|
|
193
|
+
data: input iterable
|
|
194
|
+
|
|
195
|
+
Returns:
|
|
196
|
+
--------
|
|
197
|
+
True if the data type is one of the accepted iterable types, false otherwise
|
|
198
|
+
"""
|
|
199
|
+
return any(isinstance(data, t) for t in (list, np.ndarray, pd.Series))
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
T = TypeVar("T")
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def is_list_of(lst: Sequence[object], tp: Type[T]) -> bool:
|
|
206
|
+
return isinstance(lst, list) and all(isinstance(x, tp) for x in lst)
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def count_characters_raw_data(data: Union[str, List[str]]) -> int:
|
|
210
|
+
character_count = 0
|
|
211
|
+
if isinstance(data, str):
|
|
212
|
+
return len(data)
|
|
213
|
+
for string in data:
|
|
214
|
+
character_count += len(string)
|
|
215
|
+
return character_count
|
fi/utils/utils.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
from fi.utils.constants import MAX_FUTURE_YEARS_FROM_CURRENT_TIME, MAX_PAST_YEARS_FROM_CURRENT_TIME
|
|
2
|
+
|
|
3
|
+
def is_timestamp_in_range(now: int, ts: int):
|
|
4
|
+
max_time = now + (MAX_FUTURE_YEARS_FROM_CURRENT_TIME * 365 * 24 * 60 * 60)
|
|
5
|
+
min_time = now - (MAX_PAST_YEARS_FROM_CURRENT_TIME * 365 * 24 * 60 * 60)
|
|
6
|
+
return min_time <= ts <= max_time
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
Copyright (c) 2020,The Future Company(tfc)
|
|
2
|
+
All rights reserved.
|
|
3
|
+
|
|
4
|
+
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
|
|
5
|
+
|
|
6
|
+
1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
|
|
7
|
+
|
|
8
|
+
2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
|
|
9
|
+
|
|
10
|
+
3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
|
14
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
|
|
15
|
+
IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
|
|
16
|
+
HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: futureagi
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Empowering GenAI teams to maintain peak model accuracy in production environments.
|
|
5
|
+
Home-page: https://github.com/future-agi/client
|
|
6
|
+
Author: Future AGI
|
|
7
|
+
Author-email: noreply@mail.futureagi.com
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Requires-Python: >=3.6
|
|
12
|
+
Description-Content-Type: text/markdown
|
|
13
|
+
License-File: LICENSE.md
|
|
14
|
+
Requires-Dist: certifi ==2024.6.2
|
|
15
|
+
Requires-Dist: charset-normalizer ==3.3.2
|
|
16
|
+
Requires-Dist: idna ==3.7
|
|
17
|
+
Requires-Dist: numpy ==1.26.4
|
|
18
|
+
Requires-Dist: pandas ==2.2.2
|
|
19
|
+
Requires-Dist: python-dateutil ==2.9.0.post0
|
|
20
|
+
Requires-Dist: pytz ==2024.1
|
|
21
|
+
Requires-Dist: requests ==2.32.3
|
|
22
|
+
Requires-Dist: requests-futures ==1.0.1
|
|
23
|
+
Requires-Dist: six ==1.16.0
|
|
24
|
+
Requires-Dist: tzdata ==2024.1
|
|
25
|
+
Requires-Dist: urllib3 ==2.2.2
|
|
26
|
+
|
|
27
|
+
# Future AGI
|
|
28
|
+
|
|
29
|
+

|
|
30
|
+
|
|
31
|
+
Welcome to FutureAGI - Empowering GenAI Teams with Advanced Performance Management
|
|
32
|
+
|
|
33
|
+
# Overview
|
|
34
|
+
|
|
35
|
+
FutureAGI provides a cutting-edge platform designed to help GenAI teams maintain peak model accuracy in production environments.
|
|
36
|
+
Our solution is purpose-built, scalable, and delivers results 10x faster than traditional methods.
|
|
37
|
+
|
|
38
|
+
**Key Features**
|
|
39
|
+
|
|
40
|
+
* **_Simplified GenAI Performance Management_**: Streamline your workflow and focus on developing cutting-edge AI models.
|
|
41
|
+
* **_Instant Evaluation_**: Score outputs without human-in-the-loop or ground truth, increasing QA team efficiency by up to 10x.
|
|
42
|
+
* **_Advanced Error Analytics_**: Gain ready-to-use insights with comprehensive error tagging and segmentation.
|
|
43
|
+
* **_Configurable Metrics_**: Define custom metrics tailored to your specific use case for precise model evaluation.
|
|
44
|
+
|
|
45
|
+
# Quickstart
|
|
46
|
+
**Installation**
|
|
47
|
+
|
|
48
|
+
To install the client, you can clone the repository or install the library:
|
|
49
|
+
|
|
50
|
+
Install the library in an environment using Python >= 3.6.
|
|
51
|
+
```
|
|
52
|
+
$ pip3 install futureagi
|
|
53
|
+
```
|
|
54
|
+
Or clone the repo:
|
|
55
|
+
|
|
56
|
+
```
|
|
57
|
+
$ git clone https://github.com/future-agi/client
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
**Initialisation**
|
|
61
|
+
|
|
62
|
+
To initialise the Future agi Client, you need to provide your api_key and secret_key, which are associated with your Future agi account.
|
|
63
|
+
|
|
64
|
+
Get your service API key When you create an account, we generate a service API key. You will need this API Key and your Space Key for logging authentication.
|
|
65
|
+
Instrument your code Python Client If you are using the Future Agi python client, add a few lines to your code to log your data. Logs are sent to us asynchronously.
|
|
66
|
+
|
|
67
|
+
```
|
|
68
|
+
from fi.client import Client
|
|
69
|
+
|
|
70
|
+
api_key = os.environ["FI_API_KEY"]
|
|
71
|
+
secret_key = os.environ["FI_SECRET_KEY"]
|
|
72
|
+
base_url = os.environ["FI_API_URL"]
|
|
73
|
+
|
|
74
|
+
client = Client(api_key=api_key, secret_key=secret_key,
|
|
75
|
+
uri=base_url,
|
|
76
|
+
max_workers=8,
|
|
77
|
+
max_queue_bound=5000,
|
|
78
|
+
timeout=200,
|
|
79
|
+
additional_headers=None,
|
|
80
|
+
)
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
**Initializes the Futureagi Client**
|
|
84
|
+
* _api_key_: provided API key associated with your account.
|
|
85
|
+
* _secret_key_:provided identifier to connect records to spaces.
|
|
86
|
+
* _uri_: URI to send your records to Futureagi client.
|
|
87
|
+
* _max_workers_: maximum number of concurrent requests to Futureagi. Defaults to 8.
|
|
88
|
+
* _max_queue_bound_: maximum number of concurrent future objects generated for publishing to Futureagi. Defaults to 5000.
|
|
89
|
+
* _timeout_: how long to wait for the server to send data before giving up. Defaults to 200.
|
|
90
|
+
* _additional_headers_: Dictionary of additional headers to append to request
|
|
91
|
+
|
|
92
|
+
You can also set these keys as environment variables:
|
|
93
|
+
```
|
|
94
|
+
export FI_API_KEY=your_api_key
|
|
95
|
+
export FI_SECRET_KEY=your_secret_key
|
|
96
|
+
```
|
|
97
|
+
And then initialise the client without passing the keys directly:
|
|
98
|
+
|
|
99
|
+
```
|
|
100
|
+
from fi.utils.types import ModelTypes, Environments
|
|
101
|
+
|
|
102
|
+
client.log(
|
|
103
|
+
model_id="your_model_id",
|
|
104
|
+
model_type=ModelTypes.GENERATIVE_LLM,
|
|
105
|
+
environment=Environments.PRODUCTION,
|
|
106
|
+
model_version="1.0.0",
|
|
107
|
+
prediction_timestamp=1625216400,
|
|
108
|
+
conversation={
|
|
109
|
+
"chat_history": [
|
|
110
|
+
{"role": "user", "content": "How do I implement a neural network in Python?"}
|
|
111
|
+
]
|
|
112
|
+
},
|
|
113
|
+
tags={"project": "AI project"}
|
|
114
|
+
)
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
**Parameters**
|
|
118
|
+
* _model_id_: The ID of the model. Must be a string.
|
|
119
|
+
* _model_type_: The type of the model. Must be an instance of ModelTypes.
|
|
120
|
+
* _environment_: The environment in which the model is running. Must be an instance of Environments.
|
|
121
|
+
* _model_version_: The version of the model. Must be a string.
|
|
122
|
+
* _prediction_timestamp_: (Optional) The timestamp of the prediction. Must be an integer.
|
|
123
|
+
* _conversation_: The conversation data. Must be a dictionary containing either chat_history or chat_graph.
|
|
124
|
+
* _tags_: (Optional) Additional tags for the event. Must be a dictionary.
|
|
125
|
+
|
|
126
|
+
**_[For full details, see our docs.](https://docs.futureagi.com/)_**
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
**Conversation Format**
|
|
130
|
+
|
|
131
|
+
**Chat History**
|
|
132
|
+
The chat_history must be a list of dictionaries with the following keys:
|
|
133
|
+
* _role_: The role of the participant (e.g., “user”, “assistant”). Must be a string.
|
|
134
|
+
* _content_: The content of the message. Must be a string.
|
|
135
|
+
* _context_: (Optional) The context of the message. Must be a list of pairs of strings in the format [["", ""]...].
|
|
136
|
+
|
|
137
|
+
**Chat History with conversation ID**
|
|
138
|
+
The chat_history must be a list of dictionaries with the following keys:
|
|
139
|
+
* _conversation_id_: The ID of the conversation. Must be a string.
|
|
140
|
+
* _role_: The role of the participant (e.g., “user”, “assistant”). Must be a string.
|
|
141
|
+
* _content_: The content of the message. Must be a string.
|
|
142
|
+
* _context_: (Optional) The context of the message. Must be a list of pairs of strings in the format [["", ""]...].
|
|
143
|
+
|
|
144
|
+
**Chat Graph**
|
|
145
|
+
The chat_graph must be a dictionary with the following keys:
|
|
146
|
+
* _conversation_id_: The ID of the conversation. Must be a string.
|
|
147
|
+
* _nodes_: A list of nodes, each containing:
|
|
148
|
+
* _message_: A dictionary with the message details.
|
|
149
|
+
* _node_id_: The ID of the node. Must be a string.
|
|
150
|
+
* _parent_id_: The ID of the parent node. Must be a string.
|
|
151
|
+
* _timestamp_: The timestamp of the node. Must be an integer.
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
1. **Logging data individually:** For example, "chat_history" may include a list of dictionaries where each dictionary represents a message with attributes like "role" (str) and "content" (str) .
|
|
155
|
+
|
|
156
|
+
```
|
|
157
|
+
{
|
|
158
|
+
"chat_history": [
|
|
159
|
+
{
|
|
160
|
+
"role": "user",
|
|
161
|
+
"content": "Who won the world series in 2020?"
|
|
162
|
+
},
|
|
163
|
+
{
|
|
164
|
+
"role": "assistant",
|
|
165
|
+
"content": "The Los Angeles Dodgers won the World Series in 2020."
|
|
166
|
+
}
|
|
167
|
+
]
|
|
168
|
+
}
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
1. **Logging data all at once:** This involves logging structured conversations in a unified format:
|
|
172
|
+
|
|
173
|
+
```
|
|
174
|
+
[{
|
|
175
|
+
"conversation_id": "",
|
|
176
|
+
"title": "",
|
|
177
|
+
"root_node": "",
|
|
178
|
+
"metadata": {},
|
|
179
|
+
"nodes": [{
|
|
180
|
+
"parent_node": "",
|
|
181
|
+
"child_node": "",
|
|
182
|
+
"message": {
|
|
183
|
+
"id": "",
|
|
184
|
+
"author": {
|
|
185
|
+
"role": "assistant",
|
|
186
|
+
"metadata": {}
|
|
187
|
+
},
|
|
188
|
+
"content": {
|
|
189
|
+
"content_type": "text",
|
|
190
|
+
"parts": [
|
|
191
|
+
"The user is interested to do this task..."
|
|
192
|
+
]
|
|
193
|
+
}
|
|
194
|
+
"context": ""
|
|
195
|
+
}
|
|
196
|
+
}]
|
|
197
|
+
}]
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
**Error Handling**
|
|
201
|
+
The client raises specific exceptions for different types of errors:
|
|
202
|
+
* _AuthError_: Raised if the API key or secret key is missing.
|
|
203
|
+
* _InvalidAdditionalHeaders_: Raised if there are conflicting additional headers.
|
|
204
|
+
* _InvalidValueType_: Raised if a parameter has an invalid type.
|
|
205
|
+
* _InvalidSupportedType_: Raised if a model type is not supported.
|
|
206
|
+
* _MissingRequiredKey_: Raised if a required key is missing.
|
|
207
|
+
* _InvalidVectorLength_: Raised if the vector length is invalid.
|
|
208
|
+
|
|
209
|
+
# FAQ’s:
|
|
210
|
+
|
|
211
|
+
1. Q: How do you give a performance score without human in the loop?
|
|
212
|
+
|
|
213
|
+
Our secret Sauce is a Critique AI agent that cana deliver powerful evaluation framework without need for human in the loop. What’s more is that it is 100% configurable as per new evolving use cases. Now anything that you can imagine your AI system should deliver - you can configure our platform to manage it.
|
|
214
|
+
|
|
215
|
+
2. Q: What all inputs FutureAGI platform needs?
|
|
216
|
+
|
|
217
|
+
We would need only the input-output database, training dataset if available, and User-analytics. We do not need to understand the model and how it is taking decisions.
|
|
218
|
+
|
|
219
|
+
3. Q: I don't want to share data with Future AGI, can I still use it?
|
|
220
|
+
|
|
221
|
+
Yes, you can now install our SDK in your private cloud and take advantage of our strong platform to align your Ai system to your users.
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
4. Q: My use case is unique, would you provide service to customise your platform as per my use case?
|
|
225
|
+
|
|
226
|
+
Our platform if 100% customisable and easy to configure for all types of models and modalities by the AI teams. However, our customer-success engineer would be happy to assist you for figuring out solutions to your unique use cases.
|
|
227
|
+
|
|
228
|
+
5. Q: My app uses multiple models with multiple modalities, can you work with images and videos also?
|
|
229
|
+
|
|
230
|
+
Yes we can.
|
|
231
|
+
|
|
232
|
+
6. Q: How much time does it take to integrate the Future AGI platform? How much bandwidth would be required?
|
|
233
|
+
|
|
234
|
+
It takes just 2 minutes to integrate a few lines of code and your data starts showing on our platform. Try it today.
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
# Resources
|
|
238
|
+
|
|
239
|
+
**Website**: https://www.futureagi.com/
|
|
240
|
+
|
|
241
|
+
**Documentation**: https://docs.futureagi.com/
|
|
242
|
+
|
|
243
|
+
**PyPI** : https://pypi.org/project/futureagi/
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
2
|
+
fi/__init__.py,sha256=ugyuFliEqtAwQmH4sTlc16YXKYbFWDmfyk87fErB8-8,21
|
|
3
|
+
fi/bounded_executor.py,sha256=6KzRarfVqFfpvh6n0ots74zuiaayoqvynP6b9kMaGTs,1125
|
|
4
|
+
fi/client.py,sha256=sTkxKKhub8w2zfIctWm7itAZ_GS4HO0IrZ-tOpozO2A,12446
|
|
5
|
+
fi/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
6
|
+
fi/examples/rag_file.py,sha256=LyfRTgDAhzpc2f5A2WgJLFdp2yzbmi80z_o54ydICSE,856
|
|
7
|
+
fi/utils/__init__.py,sha256=ugyuFliEqtAwQmH4sTlc16YXKYbFWDmfyk87fErB8-8,21
|
|
8
|
+
fi/utils/constants.py,sha256=jBsGYfWNTuh6nt91-FPurSOm5QvJ819DhoDfFJHB_G8,570
|
|
9
|
+
fi/utils/errors.py,sha256=BO4OASST4DJxko_QNgo5fEcf_Q4VqSZzkJ0UwWAtTwo,4158
|
|
10
|
+
fi/utils/logging.py,sha256=f_xhmmNEczGkgqS3c6YF4rCuBMx8EU5jcmQAVVKC7WQ,1575
|
|
11
|
+
fi/utils/types.py,sha256=GvK00f9cf78d0UeUuu3clIkaNOEq5tEExcPr4Dvgiik,7335
|
|
12
|
+
fi/utils/utils.py,sha256=PkapFGJWuGKQPxwrQDibS1m5O9SBoQGWr2SZRa7wzcQ,344
|
|
13
|
+
futureagi-0.0.1.dist-info/LICENSE.md,sha256=hvUTP78ShDjfpU0QfvZN2owOUNzjP3gJvWqM4GYxr-s,1493
|
|
14
|
+
futureagi-0.0.1.dist-info/METADATA,sha256=NCCa3gMCOcKIvaoF2jQGYhVHBRyTa_cFj86Y3qpr9ls,9447
|
|
15
|
+
futureagi-0.0.1.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
|
|
16
|
+
futureagi-0.0.1.dist-info/top_level.txt,sha256=C02pR7GwZeKIkBiG9mnos33e1KovTNh6dVOxAqr--Mo,8
|
|
17
|
+
futureagi-0.0.1.dist-info/RECORD,,
|