futureagi 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of futureagi might be problematic. Click here for more details.

docs/__init__.py ADDED
File without changes
fi/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.0.1"
fi/bounded_executor.py ADDED
@@ -0,0 +1,33 @@
1
+ from concurrent.futures import ThreadPoolExecutor
2
+ from threading import BoundedSemaphore
3
+
4
+
5
+ class BoundedExecutor:
6
+ """BoundedExecutor behaves as a ThreadPoolExecutor which will block on
7
+ calls to submit() once the limit given as "bound" work items are queued for
8
+ execution.
9
+ :param bound: Integer - the maximum number of items in the work queue
10
+ :param max_workers: Integer - the size of the thread pool
11
+ """
12
+
13
+ def __init__(self, bound, max_workers):
14
+ self.executor = ThreadPoolExecutor(max_workers=max_workers)
15
+ self.semaphore = BoundedSemaphore(bound + max_workers)
16
+
17
+ """See concurrent.futures.Executor#submit"""
18
+
19
+ def submit(self, fn, *args, **kwargs):
20
+ self.semaphore.acquire()
21
+ try:
22
+ future = self.executor.submit(fn, *args, **kwargs)
23
+ except Exception:
24
+ self.semaphore.release()
25
+ raise
26
+ else:
27
+ future.add_done_callback(lambda x: self.semaphore.release())
28
+ return future
29
+
30
+ """See concurrent.futures.Executor#shutdown"""
31
+
32
+ def shutdown(self, wait=True):
33
+ self.executor.shutdown(wait)
fi/client.py ADDED
@@ -0,0 +1,311 @@
1
+ import concurrent.futures as cf
2
+ import os
3
+ import time
4
+ from typing import Dict, Optional, Union, List
5
+
6
+ from requests_futures.sessions import FuturesSession
7
+
8
+ from fi.bounded_executor import BoundedExecutor
9
+ from fi.utils.constants import (
10
+ API_KEY_ENVVAR_NAME,
11
+ MAX_FUTURE_YEARS_FROM_CURRENT_TIME,
12
+ MAX_NUMBER_OF_EMBEDDINGS,
13
+ MAX_PAST_YEARS_FROM_CURRENT_TIME,
14
+ SECRET_KEY_ENVVAR_NAME,
15
+ MAX_EMBEDDING_DIMENSIONALITY,
16
+ )
17
+ from fi.utils.errors import (
18
+ AuthError,
19
+ InvalidAdditionalHeaders,
20
+ InvalidNumberOfEmbeddings,
21
+ InvalidValueType,
22
+ InvalidSupportedType,
23
+ MissingRequiredKey,
24
+ InvalidVectorLength,
25
+ )
26
+ from fi.utils.types import Embedding, Environments, ModelTypes
27
+ from fi.utils.utils import is_timestamp_in_range
28
+
29
+ from fi.utils.logging import logger
30
+
31
+
32
+ class Client:
33
+ def __init__(
34
+ self,
35
+ api_key,
36
+ secret_key,
37
+ uri="http://localhost:8000",
38
+ max_workers=8,
39
+ max_queue_bound=5000,
40
+ timeout=200,
41
+ additional_headers=None,
42
+ ) -> None:
43
+ """
44
+ Initializes the Fi Client
45
+ :param api_key: provided API key associated with your account.
46
+ :param secret_key:provided identifier to connect records to spaces.
47
+ :param uri: RI to send your records to Fi AI..
48
+ :param max_workers: maximum number of concurrent requests to Fi. Defaults
49
+ to 8.
50
+ :param max_queue_bound: maximum number of concurrent future objects
51
+ generated for publishing to Fi. Defaults to 5000.
52
+ :param timeout: how long to wait for the server to send data before giving
53
+ up. Defaults to 200.
54
+ :param additional_headers: Dictionary of additional headers to
55
+ append to request
56
+ """
57
+ api_key = api_key or os.getenv(API_KEY_ENVVAR_NAME)
58
+ secret_key = secret_key or os.getenv(SECRET_KEY_ENVVAR_NAME)
59
+ if api_key is None or secret_key is None:
60
+ raise AuthError(api_key, secret_key)
61
+ self._uri_event = f"{uri}/log/event/"
62
+ self._uri_model = f"{uri}/log/model/"
63
+ self._timeout = timeout
64
+ self._session = FuturesSession(
65
+ executor=BoundedExecutor(max_queue_bound, max_workers)
66
+ )
67
+
68
+ self._headers = {
69
+ "X-Api-Key": api_key,
70
+ "X-Secret-Key": secret_key,
71
+ }
72
+
73
+ if additional_headers is not None:
74
+ conflicting_keys = self._headers.keys() & additional_headers.keys()
75
+ if conflicting_keys:
76
+ raise InvalidAdditionalHeaders(conflicting_keys)
77
+ self._headers.update(additional_headers)
78
+
79
+ def _now(self):
80
+ return time.time()
81
+
82
+ def __validate_params(
83
+ self,
84
+ model_id: str,
85
+ model_type: ModelTypes,
86
+ environment: Environments,
87
+ model_version: Optional[str] = None,
88
+ prediction_timestamp: Optional[int] = None,
89
+ conversation: Optional[Dict[str, Union[str, bool, float, int]]] = None,
90
+ tags: Optional[Dict[str, Union[str, bool, float, int]]] = None,
91
+ ):
92
+ """
93
+ Validates input parameters against specified standards.
94
+ :param model_id:
95
+ :param model_type:
96
+ :param environment:
97
+ :param model_version:
98
+ :param prediction_timestamp:
99
+ :param conversation:
100
+ :param tags:
101
+ """
102
+ # Validate model id
103
+ if not isinstance(model_id, str):
104
+ raise InvalidValueType("model_id", model_id, "str")
105
+
106
+ # Validate model type
107
+ if not isinstance(model_type, ModelTypes):
108
+ raise InvalidValueType(
109
+ "model_type", model_type, "fi.utils.types.ModelTypes"
110
+ )
111
+
112
+ # Validate supported model types
113
+ if model_type not in [ModelTypes.GENERATIVE_LLM, ModelTypes.GENERATIVE_IMAGE]:
114
+ raise InvalidSupportedType(
115
+ "model_type",
116
+ model_type,
117
+ "ModelTypes.GENERATIVE_LLM,ModelTypes.GENERATIVE_IMAGE",
118
+ )
119
+
120
+ # Validate environment
121
+ if not isinstance(environment, Environments):
122
+ raise InvalidValueType(
123
+ "environment", environment, "fi.utils.types.Environments"
124
+ )
125
+
126
+ # Validate model_version
127
+ if model_version:
128
+ if not isinstance(model_version, str):
129
+ raise InvalidValueType("model_version", model_version, "str")
130
+
131
+
132
+ # Validate feature types
133
+ if conversation:
134
+ if isinstance(conversation, dict):
135
+ if "chat_history" not in conversation and "chat_graph" not in conversation:
136
+ raise MissingRequiredKey("conversation", "[chat_history, chat_graph]")
137
+
138
+ if "chat_history" in conversation:
139
+ chat_history = conversation["chat_history"]
140
+ if not isinstance(chat_history, list):
141
+ raise InvalidValueType(
142
+ "conversation['chat_history']", chat_history, "list"
143
+ )
144
+
145
+ for item in chat_history:
146
+ if not isinstance(item, dict):
147
+ raise InvalidValueType(
148
+ "conversation['chat_history'] item", item, "dict"
149
+ )
150
+
151
+ required_keys = ["role", "content"]
152
+ for key in required_keys:
153
+ if key not in item:
154
+ raise MissingRequiredKey(
155
+ "conversation['chat_history'] item", key
156
+ )
157
+
158
+ if not isinstance(item["role"], str):
159
+ raise InvalidValueType(
160
+ "conversation['chat_history']['role']", item["role"], "str"
161
+ )
162
+
163
+ if "chat_graph" in conversation:
164
+ feature = conversation["chat_graph"]
165
+ required_keys = ["conversation_id", "nodes"]
166
+ # required_keys = ["conversation_id", "title", "root_node", "metadata", "nodes"]
167
+ for key in required_keys:
168
+ if key not in feature:
169
+ raise MissingRequiredKey("conversation item", key)
170
+
171
+ if not isinstance(feature["nodes"], list):
172
+ raise InvalidValueType(
173
+ "conversation['nodes']", feature["nodes"], "list"
174
+ )
175
+
176
+ for node in feature["nodes"]:
177
+ node_required_keys = ["message"]
178
+ # node_required_keys = ["parent_node", "child_node", "message"]
179
+ for key in node_required_keys:
180
+ if key not in node:
181
+ raise MissingRequiredKey("conversation['nodes'] item", key)
182
+
183
+ if not isinstance(node["message"], dict):
184
+ raise InvalidValueType(
185
+ "conversation['nodes']['message']", node["message"], "dict"
186
+ )
187
+
188
+ message_required_keys = ["id", "author", "content", "context"]
189
+ for key in message_required_keys:
190
+ if key not in node["message"]:
191
+ raise MissingRequiredKey(
192
+ "conversation['nodes']['message']", key
193
+ )
194
+
195
+ author_required_keys = ["role", "metadata"]
196
+ for key in author_required_keys:
197
+ if key not in node["message"]["author"]:
198
+ raise MissingRequiredKey(
199
+ "conversation['nodes']['message']['author']", key
200
+ )
201
+
202
+ if node["message"]["author"]["role"] not in [
203
+ "assistant",
204
+ "user",
205
+ "system",
206
+ ]:
207
+ raise InvalidValueType(
208
+ "conversation['nodes']['message']['author']['role']",
209
+ node["message"]["author"]["role"],
210
+ "str",
211
+ )
212
+
213
+ content_required_keys = ["content_type", "parts"]
214
+ for key in content_required_keys:
215
+ if key not in node["message"]["content"]:
216
+ raise MissingRequiredKey(
217
+ "conversation['nodes']['message']['content']", key
218
+ )
219
+
220
+ if not isinstance(node["message"]["content"]["parts"], list):
221
+ raise InvalidValueType(
222
+ "conversation['nodes']['message']['content']['parts']",
223
+ node["message"]["content"]["parts"],
224
+ "list",
225
+ )
226
+ else:
227
+ raise InvalidValueType("conversation", conversation, "dict or list")
228
+
229
+
230
+ # Validate tags type
231
+ if tags:
232
+ if not isinstance(tags, dict):
233
+ raise InvalidValueType("tags", tags, "dict")
234
+ for key, value in tags.items():
235
+ if not isinstance(key, str):
236
+ raise InvalidValueType("tags key", key, "str")
237
+
238
+ # Check the timestamp present on the event
239
+ if prediction_timestamp is not None:
240
+ if not isinstance(prediction_timestamp, int):
241
+ raise InvalidValueType(
242
+ "prediction_timestamp", prediction_timestamp, "int"
243
+ )
244
+ # Send warning if prediction is sent with future timestamp
245
+ now = int(time.time())
246
+ if prediction_timestamp > now:
247
+ logger.warning(
248
+ "Caution when sending a prediction with future timestamp."
249
+ "fi only stores 2 years worth of data. For example, if you sent a prediction "
250
+ "to fi from 1.5 years ago, and now send a prediction with timestamp of a year in "
251
+ "the future, the oldest 0.5 years will be dropped to maintain the 2 years worth of data "
252
+ "requirement."
253
+ )
254
+ if not is_timestamp_in_range(now, prediction_timestamp):
255
+ raise ValueError(
256
+ f"prediction_timestamp: {prediction_timestamp} is out of range."
257
+ f"Prediction timestamps must be within {MAX_FUTURE_YEARS_FROM_CURRENT_TIME} year in the "
258
+ f"future and {MAX_PAST_YEARS_FROM_CURRENT_TIME} years in the past from the current time."
259
+ )
260
+
261
+ def log(
262
+ self,
263
+ model_id: str,
264
+ model_type: ModelTypes,
265
+ environment: Environments,
266
+ model_version: Optional[str] = None,
267
+ prediction_timestamp: Optional[int] = None,
268
+ conversation: Optional[Dict[str, Union[str, bool, float, int]]] = None,
269
+ tags: Optional[Dict[str, Union[str, bool, float, int]]] = None,
270
+ ) -> cf.Future:
271
+ """
272
+ Check `readme.md` for details on these parameters.
273
+
274
+ :param model_id:
275
+ :param model_type:
276
+ :param environment:
277
+ :param model_version:
278
+ :param prediction_timestamp:
279
+ :param conversation:
280
+ :param tags:
281
+ :return:
282
+ """
283
+ self.__validate_params(
284
+ model_id=model_id,
285
+ model_type=model_type,
286
+ environment=environment,
287
+ model_version=model_version,
288
+ prediction_timestamp=prediction_timestamp,
289
+ conversation=conversation,
290
+ tags=tags,
291
+ )
292
+
293
+ record = {
294
+ "model_id": model_id,
295
+ "model_type": model_type.value,
296
+ "environment": environment.value,
297
+ "model_version": model_version,
298
+ "prediction_timestamp": prediction_timestamp,
299
+ "conversation": conversation,
300
+ "tags": tags,
301
+ }
302
+ return self._post(record=record, uri=self._uri_model)
303
+
304
+ def _post(self, record, uri):
305
+ resp = self._session.post(
306
+ uri,
307
+ headers=self._headers,
308
+ timeout=self._timeout,
309
+ json=record,
310
+ )
311
+ return resp
File without changes
@@ -0,0 +1,29 @@
1
+ import time
2
+ from fi.client import Client
3
+
4
+ from fi.client import ModelTypes, Environments
5
+
6
+ fi_client = Client(
7
+ api_key="5ffab106598343a79720824d380632ce",
8
+ secret_key="faf78735c36c459f8f6be81823950f9d"
9
+ )
10
+
11
+ response = fi_client.log(
12
+ "sZZTSGXIFi",
13
+ ModelTypes.GENERATIVE_LLM,
14
+ Environments.PRODUCTION,
15
+ "1.2",
16
+ int(time.time()),
17
+ {
18
+ "chat_history": [
19
+ {"role": "user", "content": "What is the capital of France?", "context": [
20
+ ["France is a country in Europe.", " The capital of France is Paris."],
21
+ ["Germany is a country in Europe.", " The capital of Germany is Berlin."],
22
+ ["Italy is a country in Europe.", " The capital of Italy is Rome."]
23
+ ]},
24
+ {"role": "assistant",
25
+ "content": "Paris"}
26
+ ]
27
+ }
28
+ ).result()
29
+ print(response)
fi/utils/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __version__ = "0.0.1"
fi/utils/constants.py ADDED
@@ -0,0 +1,27 @@
1
+
2
+
3
+ MIN_PREDICTION_ID_LEN = 1
4
+ MAX_PREDICTION_ID_LEN = 128
5
+
6
+ # The maximum number of character for tag values
7
+ MAX_TAG_LENGTH = 20_000
8
+ MAX_TAG_LENGTH_TRUNCATION = 1_000
9
+
10
+
11
+ MAX_PAST_YEARS_FROM_CURRENT_TIME = 5
12
+ MAX_FUTURE_YEARS_FROM_CURRENT_TIME = 1
13
+
14
+ MAX_RAW_DATA_CHARACTERS = 50_000
15
+ MAX_RAW_DATA_CHARACTERS_TRUNCATION = 5_000
16
+
17
+ # The maximum number of embeddings
18
+ MAX_NUMBER_OF_EMBEDDINGS = 30
19
+ MAX_EMBEDDING_DIMENSIONALITY = 20_000
20
+
21
+ RESERVED_TAG_COLS = [
22
+
23
+ ]
24
+
25
+ # Authentication via environment variables
26
+ SECRET_KEY_ENVVAR_NAME = "TFC_SECRET_KEY"
27
+ API_KEY_ENVVAR_NAME = "TFC_API_KEY"
fi/utils/errors.py ADDED
@@ -0,0 +1,132 @@
1
+ from typing import Optional, Union
2
+ from abc import ABC, abstractmethod
3
+ from collections.abc import Iterable
4
+
5
+ from .constants import API_KEY_ENVVAR_NAME, MAX_NUMBER_OF_EMBEDDINGS, SECRET_KEY_ENVVAR_NAME
6
+ from .types import ModelTypes, Environments
7
+
8
+
9
+ class ValidationError(Exception, ABC):
10
+ def __str__(self) -> str:
11
+ return self.error_message()
12
+
13
+ @abstractmethod
14
+ def __repr__(self) -> str:
15
+ pass
16
+
17
+ @abstractmethod
18
+ def error_message(self) -> str:
19
+ pass
20
+
21
+
22
+ class AuthError(Exception):
23
+ def __init__(self, api_key: Optional[str], space_key: Optional[str]) -> None:
24
+ self.missing_api_key = api_key is None
25
+ self.missing_space_key = space_key is None
26
+
27
+ def __repr__(self) -> str:
28
+ return "Invalid_TFC_Client_Authentication"
29
+
30
+ def __str__(self) -> str:
31
+ return self.error_message()
32
+
33
+ def error_message(self) -> str:
34
+ missing_list = ["api_key"] if self.missing_api_key else []
35
+ if self.missing_space_key:
36
+ missing_list.append("space_key")
37
+
38
+ return (
39
+ "TFC Client could not obtain credentials. You can pass your api_key and space_key "
40
+ "directly to the TFC Client, or you can set environment variables which will be read if the "
41
+ "keys are not directly passed. "
42
+ "To set the environment variables use the following variable names: \n"
43
+ f" - {API_KEY_ENVVAR_NAME} for the api key\n"
44
+ f" - {SECRET_KEY_ENVVAR_NAME} for the secret key\n"
45
+ f"Missing: {missing_list}"
46
+ )
47
+
48
+
49
+ class InvalidAdditionalHeaders(ValidationError):
50
+ def __repr__(self) -> str:
51
+ return "Invalid_Additional_Headers"
52
+
53
+ def __init__(self, invalid_headers: Iterable) -> None:
54
+ self.invalid_header_names = invalid_headers
55
+
56
+ def error_message(self) -> str:
57
+ return (
58
+ "Found invalid additional header, cannot use reserved headers named: "
59
+ f"{', '.join(map(str, self.invalid_header_names))}."
60
+ )
61
+
62
+
63
+ class InvalidNumberOfEmbeddings(ValidationError):
64
+ def __repr__(self) -> str:
65
+ return "Invalid_Number_Of_Embeddings"
66
+
67
+ def __init__(self, number_of_embeddings: int) -> None:
68
+ self.number_of_embeddings = number_of_embeddings
69
+
70
+ def error_message(self) -> str:
71
+ return (
72
+ f"The schema contains {self.number_of_embeddings} different embeddings when a maximum of "
73
+ f"{MAX_NUMBER_OF_EMBEDDINGS} is allowed."
74
+ )
75
+
76
+
77
+ class InvalidValueType(Exception):
78
+ def __init__(
79
+ self,
80
+ value_name: str,
81
+ value: Union[bool, int, float, str],
82
+ correct_type: str,
83
+ ) -> None:
84
+ self.value_name = value_name
85
+ self.value = value
86
+ self.correct_type = correct_type
87
+
88
+ def __repr__(self) -> str:
89
+ return "Invalid_Value_Type"
90
+
91
+ def __str__(self) -> str:
92
+ return self.error_message()
93
+
94
+ def error_message(self) -> str:
95
+ return (
96
+ f"{self.value_name} with value {self.value} is of type {type(self.value).__name__}, "
97
+ f"but expected From {self.correct_type}"
98
+ )
99
+
100
+
101
+ class InvalidSupportedType(Exception):
102
+ def __init__(
103
+ self,
104
+ value_name: str,
105
+ value: Union[ModelTypes, Environments],
106
+ correct_type: str,
107
+ ) -> None:
108
+ self.value_name = value_name
109
+ self.value = value
110
+ self.correct_type = correct_type
111
+
112
+ def __repr__(self) -> str:
113
+ return "Invalid_Value_Type"
114
+
115
+ def __str__(self) -> str:
116
+ return self.error_message()
117
+
118
+ def error_message(self) -> str:
119
+ return (
120
+ f"{self.value_name} with value {self.value} is noy supported as of now, "
121
+ f"supported model types are {self.correct_type}"
122
+ )
123
+
124
+
125
+ class MissingRequiredKey(Exception):
126
+ def __init__(self, field_name, missing_key):
127
+ super().__init__(f"Missing required key '{missing_key}' in {field_name}.")
128
+
129
+
130
+ class InvalidVectorLength(Exception):
131
+ def __init__(self, field_name, length, max_length):
132
+ super().__init__(f"Invalid vector length for {field_name}: maximum length is {max_length}, got {length}.")
fi/utils/logging.py ADDED
@@ -0,0 +1,47 @@
1
+ import logging
2
+ import sys
3
+
4
+ class CustomLogFormatter(logging.Formatter):
5
+ grey = "\x1b[38;21m"
6
+ blue = "\x1b[38;5;39m"
7
+ yellow = "\x1b[33m"
8
+ red = "\x1b[38;5;196m"
9
+ bold_red = "\x1b[31;1m"
10
+ reset = "\x1b[0m"
11
+
12
+ def __init__(self, fmt):
13
+ super().__init__()
14
+ self.fmt = fmt
15
+ self.FORMATS = {
16
+ logging.DEBUG: self.blue + self.fmt + self.reset,
17
+ logging.INFO: self.grey + self.fmt + self.reset,
18
+ logging.WARNING: self.yellow + self.fmt + self.reset,
19
+ logging.ERROR: self.red + self.fmt + self.reset,
20
+ logging.CRITICAL: self.bold_red + self.fmt + self.reset,
21
+ }
22
+
23
+ def format(self, record):
24
+ log_fmt = self.FORMATS.get(record.levelno)
25
+ formatter = logging.Formatter(log_fmt)
26
+ return formatter.format(record)
27
+
28
+
29
+ logger = logging.getLogger(__name__)
30
+ logger.propagate = False
31
+ if logger.hasHandlers():
32
+ logger.handlers.clear()
33
+ logger.setLevel(logging.INFO)
34
+ fmt = " %(name)s | %(levelname)s | %(message)s"
35
+ if hasattr(sys, "ps1"): # for python interactive mode
36
+ handler = logging.StreamHandler(sys.stdout)
37
+ handler.setLevel(logging.INFO)
38
+ handler.setFormatter(CustomLogFormatter(fmt))
39
+ logger.addHandler(handler)
40
+
41
+
42
+ def get_truncation_warning_message(instance, limit) -> str:
43
+ return (
44
+ f"Attention: {instance} exceeding the {limit} character limit will be "
45
+ "automatically truncated upon ingestion into the fi platform. Should you require "
46
+ "a higher limit, please reach out to our support team at support@fi.com"
47
+ )
fi/utils/types.py ADDED
@@ -0,0 +1,215 @@
1
+ from enum import Enum, unique
2
+ from typing import List, NamedTuple, Optional, Sequence, TypeVar, Union, Type
3
+
4
+ import pandas as pd
5
+ from fi.utils.logging import get_truncation_warning_message, logger
6
+
7
+ import numpy as np
8
+
9
+ from fi.utils.constants import MAX_RAW_DATA_CHARACTERS, MAX_RAW_DATA_CHARACTERS_TRUNCATION
10
+
11
+ @unique
12
+ class ModelTypes(Enum):
13
+ NUMERIC = 1
14
+ SCORE_CATEGORICAL = 2 # Multi class classification, nlp, image classification
15
+ RANKING = 3
16
+ BINARY_CLASSIFICATION = 4 # single class classification
17
+ REGRESSION = 5 # regression, timeseries
18
+ OBJECT_DETECTION = 6
19
+ SEGMENTATION = 7
20
+ GENERATIVE_LLM = 8
21
+ GENERATIVE_IMAGE = 9
22
+ GENERATIVE_VIDEO = 10
23
+ TTS = 11
24
+ STT = 12
25
+ MULTI_MODAL = 13
26
+
27
+ @classmethod
28
+ def list_types(cls):
29
+ return [t.name for t in cls]
30
+
31
+
32
+ NUMERIC_MODEL_TYPES = [ModelTypes.NUMERIC, ModelTypes.REGRESSION]
33
+ CATEGORICAL_MODEL_TYPES = [
34
+ ModelTypes.SCORE_CATEGORICAL,
35
+ ModelTypes.BINARY_CLASSIFICATION,
36
+ ]
37
+
38
+
39
+ @unique
40
+ class Environments(Enum):
41
+ TRAINING = 1
42
+ VALIDATION = 2
43
+ PRODUCTION = 3
44
+ CORPUS = 4
45
+
46
+
47
+ class Embedding(NamedTuple):
48
+ vector: List[float]
49
+ data: Optional[Union[str, List[str]]] = None
50
+ link_to_data: Optional[str] = None
51
+
52
+ def validate(self, emb_name: Union[str, int, float]) -> None:
53
+ """
54
+ Validates that the embedding object passed is of the correct format. That is, validations must
55
+ be passed for vector, data & link_to_data.
56
+
57
+ Arguments:
58
+ ----------
59
+ emb_name (str, int, float): Name of the embedding feature the vector belongs to
60
+
61
+ Raises:
62
+ -------
63
+ TypeError: If the embedding fields are of the wrong type
64
+ """
65
+
66
+ if self.vector is not None:
67
+ self._validate_embedding_vector(emb_name)
68
+
69
+ # Validate embedding raw data, if present
70
+ if self.data is not None:
71
+ self._validate_embedding_data(emb_name, self.data)
72
+
73
+ # Validate embedding link to data, if present
74
+ if self.link_to_data is not None:
75
+ self._validate_embedding_link_to_data(emb_name, self.link_to_data)
76
+
77
+ return None
78
+
79
+ def _validate_embedding_vector(
80
+ self,
81
+ emb_name: Union[str, int, float],
82
+ ) -> None:
83
+ """
84
+ Validates that the embedding vector passed is of the correct format. That is:
85
+ 1. Type must be list or convertible to list (like numpy arrays, pandas Series)
86
+ 2. List must not be empty
87
+ 3. Elements in list must be floats
88
+
89
+ Arguments:
90
+ ----------
91
+ emb_name (str, int, float): Name of the embedding feature the vector belongs to
92
+
93
+ Raises:
94
+ -------
95
+ TypeError: If the embedding does not satisfy requirements above
96
+ """
97
+
98
+ if not Embedding._is_valid_iterable(self.vector):
99
+ raise TypeError(
100
+ f'Embedding feature "{emb_name}" has vector type {type(self.vector)}. Must be '
101
+ f"list, "
102
+ f"np.ndarray or pd.Series"
103
+ )
104
+ # Fail if not all elements in list are floats
105
+ allowed_types = (int, float, np.int16, np.int32, np.float16, np.float32)
106
+ if not all(isinstance(val, allowed_types) for val in self.vector): # type: ignore
107
+ raise TypeError(
108
+ f"Embedding vector must be a vector of integers and/or floats. Got "
109
+ f"{emb_name}.vector = {self.vector}"
110
+ )
111
+ # Fail if the length of the vector is 1
112
+ if len(self.vector) == 1:
113
+ raise ValueError("Embedding vector must not have a size of 1")
114
+
115
+ @staticmethod
116
+ def _validate_embedding_data(
117
+ emb_name: Union[str, int, float], data: Union[str, List[str]]
118
+ ) -> None:
119
+ """
120
+ Validates that the embedding raw data field is of the correct format. That is:
121
+ 1. Must be string or list of strings (NLP case)
122
+
123
+ Arguments:
124
+ ----------
125
+ emb_name (str, int, float): Name of the embedding feature the vector belongs to
126
+ data (str, int, float): Raw data associated with the embedding feature. Typically raw text.
127
+
128
+ Raises:
129
+ -------
130
+ TypeError: If the embedding does not satisfy requirements above
131
+ """
132
+ # Validate that data is a string or iterable of strings
133
+ is_string = isinstance(data, str)
134
+ is_allowed_iterable = not is_string and Embedding._is_valid_iterable(data)
135
+ if not (is_string or is_allowed_iterable):
136
+ raise TypeError(
137
+ f'Embedding feature "{emb_name}" data field must be str, list, np.ndarray or '
138
+ f"pd.Series"
139
+ )
140
+
141
+ if is_allowed_iterable:
142
+ # Fail if not all elements in iterable are strings
143
+ if not all(isinstance(val, str) for val in data):
144
+ raise TypeError("Embedding data field must contain strings")
145
+
146
+ character_count = count_characters_raw_data(data)
147
+ if character_count > MAX_RAW_DATA_CHARACTERS:
148
+ raise ValueError(
149
+ f"Embedding data field must not contain more than {MAX_RAW_DATA_CHARACTERS} characters. "
150
+ f"Found {character_count}."
151
+ )
152
+ elif character_count > MAX_RAW_DATA_CHARACTERS_TRUNCATION:
153
+ logger.warning(
154
+ get_truncation_warning_message(
155
+ "Embedding raw data fields", MAX_RAW_DATA_CHARACTERS_TRUNCATION
156
+ )
157
+ )
158
+
159
+ @staticmethod
160
+ def _validate_embedding_link_to_data(
161
+ emb_name: Union[str, int, float], link_to_data: str
162
+ ) -> None:
163
+ """
164
+ Validates that the embedding link to data field is of the correct format. That is:
165
+ 1. Must be string
166
+
167
+ Arguments:
168
+ ----------
169
+ emb_name (str, int, float): Name of the embedding feature the vector belongs to
170
+ link_to_data (str): Link to source data of embedding feature, typically an image file on
171
+ cloud storage
172
+
173
+ Raises:
174
+ -------
175
+ TypeError: If the embedding does not satisfy requirements above
176
+ """
177
+ if not isinstance(link_to_data, str):
178
+ raise TypeError(
179
+ f'Embedding feature "{emb_name}" link_to_data field must be str and got '
180
+ f"{type(link_to_data)}"
181
+ )
182
+
183
+ @staticmethod
184
+ def _is_valid_iterable(data: Union[str, List[str], List[float], np.ndarray, pd.Series]) -> bool:
185
+ """
186
+ Validates that the input data field is of the correct iterable type. That is:
187
+ 1. List or
188
+ 2. numpy array or
189
+ 3. pandas Series
190
+
191
+ Arguments:
192
+ ----------
193
+ data: input iterable
194
+
195
+ Returns:
196
+ --------
197
+ True if the data type is one of the accepted iterable types, false otherwise
198
+ """
199
+ return any(isinstance(data, t) for t in (list, np.ndarray, pd.Series))
200
+
201
+
202
+ T = TypeVar("T")
203
+
204
+
205
+ def is_list_of(lst: Sequence[object], tp: Type[T]) -> bool:
206
+ return isinstance(lst, list) and all(isinstance(x, tp) for x in lst)
207
+
208
+
209
+ def count_characters_raw_data(data: Union[str, List[str]]) -> int:
210
+ character_count = 0
211
+ if isinstance(data, str):
212
+ return len(data)
213
+ for string in data:
214
+ character_count += len(string)
215
+ return character_count
fi/utils/utils.py ADDED
@@ -0,0 +1,9 @@
1
+ from fi.utils.constants import MAX_FUTURE_YEARS_FROM_CURRENT_TIME, MAX_PAST_YEARS_FROM_CURRENT_TIME
2
+
3
+ def is_timestamp_in_range(now: int, ts: int):
4
+ max_time = now + (MAX_FUTURE_YEARS_FROM_CURRENT_TIME * 365 * 24 * 60 * 60)
5
+ min_time = now - (MAX_PAST_YEARS_FROM_CURRENT_TIME * 365 * 24 * 60 * 60)
6
+ return min_time <= ts <= max_time
7
+
8
+
9
+
@@ -0,0 +1,16 @@
1
+ Copyright (c) 2020,The Future Company(tfc)
2
+ All rights reserved.
3
+
4
+ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
5
+
6
+ 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
7
+
8
+ 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
9
+
10
+ 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
11
+
12
+
13
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
14
+ THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
15
+ IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
16
+ HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@@ -0,0 +1,248 @@
1
+ Metadata-Version: 2.1
2
+ Name: futureagi
3
+ Version: 0.0.1
4
+ Summary: Empowering GenAI teams to maintain peak model accuracy in production environments.
5
+ Home-page: https://github.com/future-agi/client
6
+ Author: Future AGI
7
+ Author-email: noreply@mail.futureagi.com
8
+ Classifier: Programming Language :: Python :: 3
9
+ Classifier: License :: OSI Approved :: MIT License
10
+ Classifier: Operating System :: OS Independent
11
+ Requires-Python: >=3.6
12
+ Description-Content-Type: text/markdown
13
+ License-File: LICENSE.md
14
+ Requires-Dist: certifi ==2024.6.2
15
+ Requires-Dist: charset-normalizer ==3.3.2
16
+ Requires-Dist: idna ==3.7
17
+ Requires-Dist: numpy ==1.26.4
18
+ Requires-Dist: pandas ==2.2.2
19
+ Requires-Dist: python-dateutil ==2.9.0.post0
20
+ Requires-Dist: pytz ==2024.1
21
+ Requires-Dist: requests ==2.32.3
22
+ Requires-Dist: requests-futures ==1.0.1
23
+ Requires-Dist: six ==1.16.0
24
+ Requires-Dist: tzdata ==2024.1
25
+ Requires-Dist: urllib3 ==2.2.2
26
+
27
+ # Future AGI
28
+
29
+ ![Company Logo](https://github.com/future-agi/client/blob/main/Logo.png)
30
+
31
+ Welcome to FutureAGI - Empowering GenAI Teams with Advanced Performance Management
32
+
33
+ # Overview
34
+
35
+ FutureAGI provides a cutting-edge platform designed to help GenAI teams maintain peak model accuracy in production environments.
36
+ Our solution is purpose-built, scalable, and delivers results 10x faster than traditional methods.
37
+
38
+ **Key Features**
39
+
40
+ * **_Simplified GenAI Performance Management_**: Streamline your workflow and focus on developing cutting-edge AI models.
41
+ * **_Instant Evaluation_**: Score outputs without human-in-the-loop or ground truth, increasing QA team efficiency by up to 10x.
42
+ * **_Advanced Error Analytics_**: Gain ready-to-use insights with comprehensive error tagging and segmentation.
43
+ * **_Configurable Metrics_**: Define custom metrics tailored to your specific use case for precise model evaluation.
44
+
45
+ # Quickstart
46
+ **Installation**
47
+
48
+ To install the client, you can clone the repository or install the library:
49
+
50
+ Install the library in an environment using Python >= 3.6.
51
+ ```
52
+ $ pip3 install futureagi
53
+ ```
54
+ Or clone the repo:
55
+
56
+ ```
57
+ $ git clone https://github.com/future-agi/client
58
+ ```
59
+
60
+ **Initialisation**
61
+
62
+ To initialise the Future agi Client, you need to provide your api_key and secret_key, which are associated with your Future agi account.
63
+
64
+ Get your service API key When you create an account, we generate a service API key. You will need this API Key and your Space Key for logging authentication.
65
+ Instrument your code Python Client If you are using the Future Agi python client, add a few lines to your code to log your data. Logs are sent to us asynchronously.
66
+
67
+ ```
68
+ from fi.client import Client
69
+
70
+ api_key = os.environ["FI_API_KEY"]
71
+ secret_key = os.environ["FI_SECRET_KEY"]
72
+ base_url = os.environ["FI_API_URL"]
73
+
74
+ client = Client(api_key=api_key, secret_key=secret_key,
75
+ uri=base_url,
76
+ max_workers=8,
77
+ max_queue_bound=5000,
78
+ timeout=200,
79
+ additional_headers=None,
80
+ )
81
+ ```
82
+
83
+ **Initializes the Futureagi Client**
84
+ * _api_key_: provided API key associated with your account.
85
+ * _secret_key_:provided identifier to connect records to spaces.
86
+ * _uri_: URI to send your records to Futureagi client.
87
+ * _max_workers_: maximum number of concurrent requests to Futureagi. Defaults to 8.
88
+ * _max_queue_bound_: maximum number of concurrent future objects generated for publishing to Futureagi. Defaults to 5000.
89
+ * _timeout_: how long to wait for the server to send data before giving up. Defaults to 200.
90
+ * _additional_headers_: Dictionary of additional headers to append to request
91
+
92
+ You can also set these keys as environment variables:
93
+ ```
94
+ export FI_API_KEY=your_api_key
95
+ export FI_SECRET_KEY=your_secret_key
96
+ ```
97
+ And then initialise the client without passing the keys directly:
98
+
99
+ ```
100
+ from fi.utils.types import ModelTypes, Environments
101
+
102
+ client.log(
103
+ model_id="your_model_id",
104
+ model_type=ModelTypes.GENERATIVE_LLM,
105
+ environment=Environments.PRODUCTION,
106
+ model_version="1.0.0",
107
+ prediction_timestamp=1625216400,
108
+ conversation={
109
+ "chat_history": [
110
+ {"role": "user", "content": "How do I implement a neural network in Python?"}
111
+ ]
112
+ },
113
+ tags={"project": "AI project"}
114
+ )
115
+ ```
116
+
117
+ **Parameters**
118
+ * _model_id_: The ID of the model. Must be a string.
119
+ * _model_type_: The type of the model. Must be an instance of ModelTypes.
120
+ * _environment_: The environment in which the model is running. Must be an instance of Environments.
121
+ * _model_version_: The version of the model. Must be a string.
122
+ * _prediction_timestamp_: (Optional) The timestamp of the prediction. Must be an integer.
123
+ * _conversation_: The conversation data. Must be a dictionary containing either chat_history or chat_graph.
124
+ * _tags_: (Optional) Additional tags for the event. Must be a dictionary.
125
+
126
+ **_[For full details, see our docs.](https://docs.futureagi.com/)_**
127
+
128
+
129
+ **Conversation Format**
130
+
131
+ **Chat History**
132
+ The chat_history must be a list of dictionaries with the following keys:
133
+ * _role_: The role of the participant (e.g., “user”, “assistant”). Must be a string.
134
+ * _content_: The content of the message. Must be a string.
135
+ * _context_: (Optional) The context of the message. Must be a list of pairs of strings in the format [["", ""]...].
136
+
137
+ **Chat History with conversation ID**
138
+ The chat_history must be a list of dictionaries with the following keys:
139
+ * _conversation_id_: The ID of the conversation. Must be a string.
140
+ * _role_: The role of the participant (e.g., “user”, “assistant”). Must be a string.
141
+ * _content_: The content of the message. Must be a string.
142
+ * _context_: (Optional) The context of the message. Must be a list of pairs of strings in the format [["", ""]...].
143
+
144
+ **Chat Graph**
145
+ The chat_graph must be a dictionary with the following keys:
146
+ * _conversation_id_: The ID of the conversation. Must be a string.
147
+ * _nodes_: A list of nodes, each containing:
148
+ * _message_: A dictionary with the message details.
149
+ * _node_id_: The ID of the node. Must be a string.
150
+ * _parent_id_: The ID of the parent node. Must be a string.
151
+ * _timestamp_: The timestamp of the node. Must be an integer.
152
+
153
+
154
+ 1. **Logging data individually:** For example, "chat_history" may include a list of dictionaries where each dictionary represents a message with attributes like "role" (str) and "content" (str) .
155
+
156
+ ```
157
+ {
158
+ "chat_history": [
159
+ {
160
+ "role": "user",
161
+ "content": "Who won the world series in 2020?"
162
+ },
163
+ {
164
+ "role": "assistant",
165
+ "content": "The Los Angeles Dodgers won the World Series in 2020."
166
+ }
167
+ ]
168
+ }
169
+ ```
170
+
171
+ 1. **Logging data all at once:** This involves logging structured conversations in a unified format:
172
+
173
+ ```
174
+ [{
175
+ "conversation_id": "",
176
+ "title": "",
177
+ "root_node": "",
178
+ "metadata": {},
179
+ "nodes": [{
180
+ "parent_node": "",
181
+ "child_node": "",
182
+ "message": {
183
+ "id": "",
184
+ "author": {
185
+ "role": "assistant",
186
+ "metadata": {}
187
+ },
188
+ "content": {
189
+ "content_type": "text",
190
+ "parts": [
191
+ "The user is interested to do this task..."
192
+ ]
193
+ }
194
+ "context": ""
195
+ }
196
+ }]
197
+ }]
198
+
199
+ ```
200
+ **Error Handling**
201
+ The client raises specific exceptions for different types of errors:
202
+ * _AuthError_: Raised if the API key or secret key is missing.
203
+ * _InvalidAdditionalHeaders_: Raised if there are conflicting additional headers.
204
+ * _InvalidValueType_: Raised if a parameter has an invalid type.
205
+ * _InvalidSupportedType_: Raised if a model type is not supported.
206
+ * _MissingRequiredKey_: Raised if a required key is missing.
207
+ * _InvalidVectorLength_: Raised if the vector length is invalid.
208
+
209
+ # FAQ’s:
210
+
211
+ 1. Q: How do you give a performance score without human in the loop?
212
+
213
+ Our secret Sauce is a Critique AI agent that cana deliver powerful evaluation framework without need for human in the loop. What’s more is that it is 100% configurable as per new evolving use cases. Now anything that you can imagine your AI system should deliver - you can configure our platform to manage it.
214
+
215
+ 2. Q: What all inputs FutureAGI platform needs?
216
+
217
+ We would need only the input-output database, training dataset if available, and User-analytics. We do not need to understand the model and how it is taking decisions.
218
+
219
+ 3. Q: I don't want to share data with Future AGI, can I still use it?
220
+
221
+ Yes, you can now install our SDK in your private cloud and take advantage of our strong platform to align your Ai system to your users.
222
+
223
+
224
+ 4. Q: My use case is unique, would you provide service to customise your platform as per my use case?
225
+
226
+ Our platform if 100% customisable and easy to configure for all types of models and modalities by the AI teams. However, our customer-success engineer would be happy to assist you for figuring out solutions to your unique use cases.
227
+
228
+ 5. Q: My app uses multiple models with multiple modalities, can you work with images and videos also?
229
+
230
+ Yes we can.
231
+
232
+ 6. Q: How much time does it take to integrate the Future AGI platform? How much bandwidth would be required?
233
+
234
+ It takes just 2 minutes to integrate a few lines of code and your data starts showing on our platform. Try it today.
235
+
236
+
237
+ # Resources
238
+
239
+ **Website**: https://www.futureagi.com/
240
+
241
+ **Documentation**: https://docs.futureagi.com/
242
+
243
+ **PyPI** : https://pypi.org/project/futureagi/
244
+
245
+
246
+
247
+
248
+
@@ -0,0 +1,17 @@
1
+ docs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
2
+ fi/__init__.py,sha256=ugyuFliEqtAwQmH4sTlc16YXKYbFWDmfyk87fErB8-8,21
3
+ fi/bounded_executor.py,sha256=6KzRarfVqFfpvh6n0ots74zuiaayoqvynP6b9kMaGTs,1125
4
+ fi/client.py,sha256=sTkxKKhub8w2zfIctWm7itAZ_GS4HO0IrZ-tOpozO2A,12446
5
+ fi/examples/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
6
+ fi/examples/rag_file.py,sha256=LyfRTgDAhzpc2f5A2WgJLFdp2yzbmi80z_o54ydICSE,856
7
+ fi/utils/__init__.py,sha256=ugyuFliEqtAwQmH4sTlc16YXKYbFWDmfyk87fErB8-8,21
8
+ fi/utils/constants.py,sha256=jBsGYfWNTuh6nt91-FPurSOm5QvJ819DhoDfFJHB_G8,570
9
+ fi/utils/errors.py,sha256=BO4OASST4DJxko_QNgo5fEcf_Q4VqSZzkJ0UwWAtTwo,4158
10
+ fi/utils/logging.py,sha256=f_xhmmNEczGkgqS3c6YF4rCuBMx8EU5jcmQAVVKC7WQ,1575
11
+ fi/utils/types.py,sha256=GvK00f9cf78d0UeUuu3clIkaNOEq5tEExcPr4Dvgiik,7335
12
+ fi/utils/utils.py,sha256=PkapFGJWuGKQPxwrQDibS1m5O9SBoQGWr2SZRa7wzcQ,344
13
+ futureagi-0.0.1.dist-info/LICENSE.md,sha256=hvUTP78ShDjfpU0QfvZN2owOUNzjP3gJvWqM4GYxr-s,1493
14
+ futureagi-0.0.1.dist-info/METADATA,sha256=NCCa3gMCOcKIvaoF2jQGYhVHBRyTa_cFj86Y3qpr9ls,9447
15
+ futureagi-0.0.1.dist-info/WHEEL,sha256=y4mX-SOX4fYIkonsAGA5N0Oy-8_gI4FXw5HNI1xqvWg,91
16
+ futureagi-0.0.1.dist-info/top_level.txt,sha256=C02pR7GwZeKIkBiG9mnos33e1KovTNh6dVOxAqr--Mo,8
17
+ futureagi-0.0.1.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (70.2.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ docs
2
+ fi