skyflow-flowvault-python 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- common/.coveragerc +4 -0
- common/__init__.py +3 -0
- common/client/__init__.py +0 -0
- common/client/base_skyflow.py +345 -0
- common/client/utils/__init__.py +1 -0
- common/client/utils/_utils.py +127 -0
- common/errors/__init__.py +1 -0
- common/errors/_skyflow_error.py +17 -0
- common/generated/__init__.py +0 -0
- common/generated/rest/__init__.py +24 -0
- common/generated/rest/authentication/__init__.py +4 -0
- common/generated/rest/authentication/client.py +181 -0
- common/generated/rest/authentication/raw_client.py +241 -0
- common/generated/rest/client.py +153 -0
- common/generated/rest/core/__init__.py +52 -0
- common/generated/rest/core/api_error.py +23 -0
- common/generated/rest/core/client_wrapper.py +86 -0
- common/generated/rest/core/datetime_utils.py +28 -0
- common/generated/rest/core/file.py +67 -0
- common/generated/rest/core/force_multipart.py +16 -0
- common/generated/rest/core/http_client.py +543 -0
- common/generated/rest/core/http_response.py +55 -0
- common/generated/rest/core/jsonable_encoder.py +100 -0
- common/generated/rest/core/pydantic_utilities.py +255 -0
- common/generated/rest/core/query_encoder.py +58 -0
- common/generated/rest/core/remove_none_from_dict.py +11 -0
- common/generated/rest/core/request_options.py +35 -0
- common/generated/rest/core/serialization.py +276 -0
- common/generated/rest/environment.py +8 -0
- common/generated/rest/errors/__init__.py +9 -0
- common/generated/rest/errors/bad_request_error.py +14 -0
- common/generated/rest/errors/not_found_error.py +14 -0
- common/generated/rest/errors/unauthorized_error.py +14 -0
- common/generated/rest/py.typed +0 -0
- common/generated/rest/types/__init__.py +9 -0
- common/generated/rest/types/googlerpc_status.py +22 -0
- common/generated/rest/types/protobuf_any.py +21 -0
- common/generated/rest/types/v_1_get_auth_token_response.py +33 -0
- common/generated/rest/version.py +6 -0
- common/service_account/__init__.py +1 -0
- common/service_account/_utils.py +248 -0
- common/service_account/client/__init__.py +0 -0
- common/service_account/client/auth_client.py +13 -0
- common/utils/__init__.py +4 -0
- common/utils/_helpers.py +18 -0
- common/utils/_skyflow_messages.py +445 -0
- common/utils/_utils.py +50 -0
- common/utils/constants.py +291 -0
- common/utils/enums/__init__.py +12 -0
- common/utils/enums/content_types.py +9 -0
- common/utils/enums/detect_entities.py +73 -0
- common/utils/enums/detect_output_transcriptions.py +8 -0
- common/utils/enums/env.py +13 -0
- common/utils/enums/log_level.py +8 -0
- common/utils/enums/masking_method.py +5 -0
- common/utils/enums/redaction_type.py +7 -0
- common/utils/enums/request_method.py +8 -0
- common/utils/enums/token_mode.py +6 -0
- common/utils/enums/token_type.py +6 -0
- common/utils/logger/__init__.py +2 -0
- common/utils/logger/_log_helpers.py +47 -0
- common/utils/logger/_logger.py +50 -0
- common/utils/validations/__init__.py +11 -0
- common/utils/validations/_validations.py +267 -0
- common/vault/base_vault_client.py +122 -0
- common/vault/base_vault_controller.py +58 -0
- common/vault/data/__init__.py +2 -0
- common/vault/data/_base_insert_request.py +7 -0
- common/vault/data/_base_insert_response.py +11 -0
- skyflow/__init__.py +2 -0
- skyflow/client/__init__.py +1 -0
- skyflow/client/_http_config_builder.py +45 -0
- skyflow/client/skyflow.py +15 -0
- skyflow/error/__init__.py +3 -0
- skyflow/generated/__init__.py +0 -0
- skyflow/generated/rest/__init__.py +164 -0
- skyflow/generated/rest/_default_clients.py +32 -0
- skyflow/generated/rest/client.py +385 -0
- skyflow/generated/rest/core/__init__.py +127 -0
- skyflow/generated/rest/core/api_error.py +23 -0
- skyflow/generated/rest/core/client_wrapper.py +148 -0
- skyflow/generated/rest/core/datetime_utils.py +70 -0
- skyflow/generated/rest/core/file.py +67 -0
- skyflow/generated/rest/core/force_multipart.py +18 -0
- skyflow/generated/rest/core/http_client.py +940 -0
- skyflow/generated/rest/core/http_response.py +63 -0
- skyflow/generated/rest/core/http_sse/__init__.py +42 -0
- skyflow/generated/rest/core/http_sse/_api.py +455 -0
- skyflow/generated/rest/core/http_sse/_decoders.py +74 -0
- skyflow/generated/rest/core/http_sse/_exceptions.py +7 -0
- skyflow/generated/rest/core/http_sse/_models.py +17 -0
- skyflow/generated/rest/core/jsonable_encoder.py +133 -0
- skyflow/generated/rest/core/logging.py +107 -0
- skyflow/generated/rest/core/parse_error.py +36 -0
- skyflow/generated/rest/core/pydantic_utilities.py +486 -0
- skyflow/generated/rest/core/query_encoder.py +58 -0
- skyflow/generated/rest/core/remove_none_from_dict.py +11 -0
- skyflow/generated/rest/core/request_options.py +40 -0
- skyflow/generated/rest/core/serialization.py +347 -0
- skyflow/generated/rest/environment.py +8 -0
- skyflow/generated/rest/errors/__init__.py +53 -0
- skyflow/generated/rest/errors/bad_request_error.py +11 -0
- skyflow/generated/rest/errors/forbidden_error.py +11 -0
- skyflow/generated/rest/errors/internal_server_error.py +11 -0
- skyflow/generated/rest/errors/not_found_error.py +11 -0
- skyflow/generated/rest/errors/too_many_requests_error.py +11 -0
- skyflow/generated/rest/errors/unauthorized_error.py +11 -0
- skyflow/generated/rest/py.typed +0 -0
- skyflow/generated/rest/query/__init__.py +4 -0
- skyflow/generated/rest/query/client.py +139 -0
- skyflow/generated/rest/query/raw_client.py +239 -0
- skyflow/generated/rest/raw_client.py +86 -0
- skyflow/generated/rest/records/__init__.py +34 -0
- skyflow/generated/rest/records/client.py +685 -0
- skyflow/generated/rest/records/raw_client.py +988 -0
- skyflow/generated/rest/records/types/__init__.py +34 -0
- skyflow/generated/rest/records/types/update_request_update_type.py +5 -0
- skyflow/generated/rest/tokens/__init__.py +4 -0
- skyflow/generated/rest/tokens/client.py +246 -0
- skyflow/generated/rest/tokens/raw_client.py +507 -0
- skyflow/generated/rest/types/__init__.py +116 -0
- skyflow/generated/rest/types/column_redactions.py +33 -0
- skyflow/generated/rest/types/delete_response.py +23 -0
- skyflow/generated/rest/types/delete_response_object.py +42 -0
- skyflow/generated/rest/types/detokenize_response.py +23 -0
- skyflow/generated/rest/types/detokenize_response_object.py +54 -0
- skyflow/generated/rest/types/error_response.py +20 -0
- skyflow/generated/rest/types/error_response_error.py +28 -0
- skyflow/generated/rest/types/execute_query_record_response.py +22 -0
- skyflow/generated/rest/types/execute_query_response.py +26 -0
- skyflow/generated/rest/types/execute_query_response_metadata.py +26 -0
- skyflow/generated/rest/types/get_request_data.py +60 -0
- skyflow/generated/rest/types/get_response.py +23 -0
- skyflow/generated/rest/types/get_tokens_from_values_request_object.py +30 -0
- skyflow/generated/rest/types/get_tokens_from_values_response.py +23 -0
- skyflow/generated/rest/types/google_protobuf_value.py +8 -0
- skyflow/generated/rest/types/http_code.py +6 -0
- skyflow/generated/rest/types/insert_record_data.py +36 -0
- skyflow/generated/rest/types/insert_response.py +23 -0
- skyflow/generated/rest/types/record_response_object.py +70 -0
- skyflow/generated/rest/types/token_group_redactions.py +33 -0
- skyflow/generated/rest/types/tokenize_response_object.py +49 -0
- skyflow/generated/rest/types/unique_value.py +22 -0
- skyflow/generated/rest/types/update_record_data.py +52 -0
- skyflow/generated/rest/types/update_record_data_update_type.py +5 -0
- skyflow/generated/rest/types/update_response.py +23 -0
- skyflow/generated/rest/types/upsert.py +45 -0
- skyflow/generated/rest/types/upsert_update_type.py +5 -0
- skyflow/generated/rest/version.py +6 -0
- skyflow/service_account/__init__.py +15 -0
- skyflow/utils/__init__.py +9 -0
- skyflow/utils/_http_config.py +36 -0
- skyflow/utils/_response_parsing.py +66 -0
- skyflow/utils/_retry.py +97 -0
- skyflow/utils/_skyflow_messages.py +114 -0
- skyflow/utils/_utils.py +54 -0
- skyflow/utils/_version.py +1 -0
- skyflow/utils/enums/__init__.py +3 -0
- skyflow/utils/enums/_custom_header_key.py +10 -0
- skyflow/utils/enums/_env_urls.py +9 -0
- skyflow/utils/enums/_upsert_type.py +7 -0
- skyflow/utils/validations/__init__.py +9 -0
- skyflow/utils/validations/_validations.py +195 -0
- skyflow/vault/__init__.py +0 -0
- skyflow/vault/client/__init__.py +0 -0
- skyflow/vault/client/client.py +102 -0
- skyflow/vault/controller/__init__.py +1 -0
- skyflow/vault/controller/_vault.py +475 -0
- skyflow/vault/data/__init__.py +30 -0
- skyflow/vault/data/_column_redactions.py +4 -0
- skyflow/vault/data/_delete_options.py +8 -0
- skyflow/vault/data/_delete_request.py +5 -0
- skyflow/vault/data/_delete_response.py +9 -0
- skyflow/vault/data/_delete_response_record.py +9 -0
- skyflow/vault/data/_detokenize_options.py +8 -0
- skyflow/vault/data/_detokenize_request.py +9 -0
- skyflow/vault/data/_detokenize_response.py +9 -0
- skyflow/vault/data/_detokenize_response_record.py +16 -0
- skyflow/vault/data/_detokenize_response_record_metadata.py +10 -0
- skyflow/vault/data/_get_options.py +8 -0
- skyflow/vault/data/_get_request.py +17 -0
- skyflow/vault/data/_get_request_record.py +13 -0
- skyflow/vault/data/_get_response.py +9 -0
- skyflow/vault/data/_get_response_record.py +17 -0
- skyflow/vault/data/_insert_options.py +8 -0
- skyflow/vault/data/_insert_request.py +11 -0
- skyflow/vault/data/_insert_request_record.py +9 -0
- skyflow/vault/data/_insert_response.py +9 -0
- skyflow/vault/data/_insert_response_record.py +17 -0
- skyflow/vault/data/_request_context.py +16 -0
- skyflow/vault/data/_token.py +8 -0
- skyflow/vault/data/_token_group_redactions.py +4 -0
- skyflow/vault/data/_update_options.py +8 -0
- skyflow/vault/data/_update_request.py +11 -0
- skyflow/vault/data/_update_request_record.py +6 -0
- skyflow/vault/data/_update_response.py +9 -0
- skyflow/vault/data/_update_response_record.py +17 -0
- skyflow/vault/data/_upsert_options.py +7 -0
- skyflow_flowvault_python-1.0.0.dist-info/METADATA +896 -0
- skyflow_flowvault_python-1.0.0.dist-info/RECORD +202 -0
- skyflow_flowvault_python-1.0.0.dist-info/WHEEL +5 -0
- skyflow_flowvault_python-1.0.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,267 @@
|
|
|
1
|
+
from common.errors import SkyflowError
|
|
2
|
+
from common.service_account import is_expired
|
|
3
|
+
from common.service_account._utils import _validate_and_resolve_ctx
|
|
4
|
+
from common.utils import SkyflowMessages
|
|
5
|
+
from common.utils.constants import ApiKey, ConfigField, CredentialField, OptionField
|
|
6
|
+
from common.utils.enums import Env, LogLevel
|
|
7
|
+
from common.utils.logger import log_error_log, log_info
|
|
8
|
+
from common.utils._helpers import is_valid_url
|
|
9
|
+
|
|
10
|
+
invalid_input_error_code = SkyflowMessages.ErrorCodes.INVALID_INPUT.value
|
|
11
|
+
|
|
12
|
+
VALID_VAULT_CONFIG_KEYS = [
|
|
13
|
+
ConfigField.VAULT_ID,
|
|
14
|
+
ConfigField.CLUSTER_ID,
|
|
15
|
+
ConfigField.CREDENTIALS,
|
|
16
|
+
ConfigField.ENV,
|
|
17
|
+
]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def validate_required_field(logger, config, field_name, expected_type, empty_error, invalid_error, messages=None):
|
|
21
|
+
messages = messages or SkyflowMessages
|
|
22
|
+
field_value = config.get(field_name)
|
|
23
|
+
|
|
24
|
+
if field_name not in config or not isinstance(field_value, expected_type):
|
|
25
|
+
if field_name == ConfigField.VAULT_ID:
|
|
26
|
+
log_error_log(messages.ErrorLogs.VAULTID_IS_REQUIRED.value, logger)
|
|
27
|
+
if field_name == ConfigField.CLUSTER_ID:
|
|
28
|
+
log_error_log(messages.ErrorLogs.CLUSTER_ID_IS_REQUIRED.value, logger)
|
|
29
|
+
if field_name == OptionField.CONNECTION_ID:
|
|
30
|
+
log_error_log(messages.ErrorLogs.CONNECTION_ID_IS_REQUIRED.value, logger)
|
|
31
|
+
if field_name == OptionField.CONNECTION_URL:
|
|
32
|
+
log_error_log(messages.ErrorLogs.INVALID_CONNECTION_URL.value, logger)
|
|
33
|
+
raise SkyflowError(invalid_error, invalid_input_error_code)
|
|
34
|
+
|
|
35
|
+
if isinstance(field_value, str) and not field_value.strip():
|
|
36
|
+
if field_name == ConfigField.VAULT_ID:
|
|
37
|
+
log_error_log(messages.ErrorLogs.EMPTY_VAULTID.value, logger)
|
|
38
|
+
if field_name == ConfigField.CLUSTER_ID:
|
|
39
|
+
log_error_log(messages.ErrorLogs.EMPTY_CLUSTER_ID.value, logger)
|
|
40
|
+
if field_name == OptionField.CONNECTION_ID:
|
|
41
|
+
log_error_log(messages.ErrorLogs.EMPTY_CONNECTION_ID.value, logger)
|
|
42
|
+
if field_name == OptionField.CONNECTION_URL:
|
|
43
|
+
log_error_log(messages.ErrorLogs.EMPTY_CONNECTION_URL.value, logger)
|
|
44
|
+
if field_name == CredentialField.PATH:
|
|
45
|
+
log_error_log(messages.ErrorLogs.EMPTY_CREDENTIALS_PATH.value, logger)
|
|
46
|
+
if field_name == CredentialField.CREDENTIALS_STRING:
|
|
47
|
+
log_error_log(messages.ErrorLogs.EMPTY_CREDENTIALS_STRING.value, logger)
|
|
48
|
+
if field_name == CredentialField.TOKEN:
|
|
49
|
+
log_error_log(messages.ErrorLogs.EMPTY_TOKEN_VALUE.value, logger)
|
|
50
|
+
if field_name == CredentialField.API_KEY:
|
|
51
|
+
log_error_log(messages.ErrorLogs.EMPTY_API_KEY_VALUE.value, logger)
|
|
52
|
+
raise SkyflowError(empty_error, invalid_input_error_code)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def validate_api_key(api_key: str, logger=None, messages=None) -> bool:
|
|
56
|
+
messages = messages or SkyflowMessages
|
|
57
|
+
if not api_key.startswith(ApiKey.SKY_PREFIX):
|
|
58
|
+
log_error_log(messages.ErrorLogs.INVALID_API_KEY.value, logger=logger)
|
|
59
|
+
return False
|
|
60
|
+
|
|
61
|
+
if len(api_key) != ApiKey.LENGTH:
|
|
62
|
+
log_error_log(messages.ErrorLogs.INVALID_API_KEY.value, logger=logger)
|
|
63
|
+
return False
|
|
64
|
+
|
|
65
|
+
return True
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def validate_token_options(logger, credentials, config_id_type=None, config_id=None, messages=None):
|
|
69
|
+
messages = messages or SkyflowMessages
|
|
70
|
+
|
|
71
|
+
if CredentialField.ROLES in credentials:
|
|
72
|
+
empty_roles_error = (
|
|
73
|
+
messages.Error.EMPTY_ROLES_IN_CONFIG.value.format(config_id_type, config_id)
|
|
74
|
+
if config_id_type and config_id else messages.Error.EMPTY_ROLES.value
|
|
75
|
+
)
|
|
76
|
+
validate_required_field(
|
|
77
|
+
logger, credentials, CredentialField.ROLES, list,
|
|
78
|
+
empty_roles_error,
|
|
79
|
+
messages.Error.INVALID_ROLES_KEY_TYPE_IN_CONFIG.value.format(config_id_type, config_id)
|
|
80
|
+
if config_id_type and config_id else messages.Error.INVALID_ROLES_KEY_TYPE.value,
|
|
81
|
+
messages=messages)
|
|
82
|
+
if not credentials.get(CredentialField.ROLES):
|
|
83
|
+
raise SkyflowError(empty_roles_error, invalid_input_error_code)
|
|
84
|
+
|
|
85
|
+
invalid_role_element_error = (
|
|
86
|
+
messages.Error.INVALID_ROLE_ELEMENT_TYPE_IN_CONFIG.value.format(config_id_type, config_id)
|
|
87
|
+
if config_id_type and config_id else messages.Error.INVALID_ROLE_ELEMENT_TYPE.value
|
|
88
|
+
)
|
|
89
|
+
for role in credentials.get(CredentialField.ROLES):
|
|
90
|
+
if not isinstance(role, str) or not role.strip():
|
|
91
|
+
raise SkyflowError(invalid_role_element_error, invalid_input_error_code)
|
|
92
|
+
|
|
93
|
+
if CredentialField.CONTEXT in credentials:
|
|
94
|
+
empty_context_error = (
|
|
95
|
+
messages.Error.EMPTY_CONTEXT_IN_CONFIG.value.format(config_id_type, config_id)
|
|
96
|
+
if config_id_type and config_id else messages.Error.EMPTY_CONTEXT.value
|
|
97
|
+
)
|
|
98
|
+
validate_required_field(
|
|
99
|
+
logger, credentials, CredentialField.CONTEXT, (str, dict, bool, int, float),
|
|
100
|
+
empty_context_error,
|
|
101
|
+
messages.Error.INVALID_CONTEXT_IN_CONFIG.value.format(config_id_type, config_id)
|
|
102
|
+
if config_id_type and config_id else messages.Error.INVALID_CONTEXT.value,
|
|
103
|
+
messages=messages)
|
|
104
|
+
context = credentials.get(CredentialField.CONTEXT)
|
|
105
|
+
if isinstance(context, dict):
|
|
106
|
+
if not context:
|
|
107
|
+
raise SkyflowError(empty_context_error, invalid_input_error_code)
|
|
108
|
+
_validate_and_resolve_ctx(context, messages=messages)
|
|
109
|
+
|
|
110
|
+
def validate_credentials(logger, credentials, config_id_type=None, config_id=None, messages=None):
|
|
111
|
+
messages = messages or SkyflowMessages
|
|
112
|
+
key_present = [k for k in [CredentialField.PATH, CredentialField.TOKEN, CredentialField.CREDENTIALS_STRING, CredentialField.API_KEY] if credentials.get(k)]
|
|
113
|
+
|
|
114
|
+
if len(key_present) == 0:
|
|
115
|
+
error_message = (
|
|
116
|
+
messages.Error.INVALID_CREDENTIALS_IN_CONFIG.value.format(config_id_type, config_id)
|
|
117
|
+
if config_id_type and config_id else
|
|
118
|
+
messages.Error.INVALID_CREDENTIALS.value
|
|
119
|
+
)
|
|
120
|
+
log_error_log(error_message, logger)
|
|
121
|
+
raise SkyflowError(error_message, invalid_input_error_code)
|
|
122
|
+
elif len(key_present) > 1:
|
|
123
|
+
error_message = (
|
|
124
|
+
messages.Error.MULTIPLE_CREDENTIALS_PASSED_IN_CONFIG.value.format(config_id_type, config_id)
|
|
125
|
+
if config_id_type and config_id else
|
|
126
|
+
messages.Error.MULTIPLE_CREDENTIALS_PASSED.value
|
|
127
|
+
)
|
|
128
|
+
log_error_log(error_message, logger)
|
|
129
|
+
raise SkyflowError(error_message, invalid_input_error_code)
|
|
130
|
+
|
|
131
|
+
validate_token_options(logger, credentials, config_id_type, config_id, messages=messages)
|
|
132
|
+
|
|
133
|
+
if CredentialField.CREDENTIALS_STRING in credentials:
|
|
134
|
+
validate_required_field(
|
|
135
|
+
logger, credentials, CredentialField.CREDENTIALS_STRING, str,
|
|
136
|
+
messages.Error.EMPTY_CREDENTIALS_STRING_IN_CONFIG.value.format(config_id_type, config_id)
|
|
137
|
+
if config_id_type and config_id else messages.Error.EMPTY_CREDENTIALS_STRING.value,
|
|
138
|
+
messages.Error.INVALID_CREDENTIALS_STRING_IN_CONFIG.value.format(config_id_type, config_id)
|
|
139
|
+
if config_id_type and config_id else messages.Error.INVALID_CREDENTIALS_STRING.value,
|
|
140
|
+
messages=messages)
|
|
141
|
+
elif CredentialField.PATH in credentials:
|
|
142
|
+
validate_required_field(
|
|
143
|
+
logger, credentials, CredentialField.PATH, str,
|
|
144
|
+
messages.Error.EMPTY_CREDENTIAL_FILE_PATH_IN_CONFIG.value.format(config_id_type, config_id)
|
|
145
|
+
if config_id_type and config_id else messages.Error.EMPTY_CREDENTIAL_FILE_PATH.value,
|
|
146
|
+
messages.Error.INVALID_CREDENTIAL_FILE_PATH_IN_CONFIG.value.format(config_id_type, config_id)
|
|
147
|
+
if config_id_type and config_id else messages.Error.INVALID_CREDENTIAL_FILE_PATH.value,
|
|
148
|
+
messages=messages)
|
|
149
|
+
elif CredentialField.TOKEN in credentials:
|
|
150
|
+
validate_required_field(
|
|
151
|
+
logger, credentials, CredentialField.TOKEN, str,
|
|
152
|
+
messages.Error.EMPTY_CREDENTIALS_TOKEN.value.format(config_id_type, config_id)
|
|
153
|
+
if config_id_type and config_id else messages.Error.EMPTY_CREDENTIALS_TOKEN.value,
|
|
154
|
+
messages.Error.INVALID_CREDENTIALS_TOKEN.value.format(config_id_type, config_id)
|
|
155
|
+
if config_id_type and config_id else messages.Error.INVALID_CREDENTIALS_TOKEN.value,
|
|
156
|
+
messages=messages)
|
|
157
|
+
if is_expired(credentials.get(CredentialField.TOKEN), logger):
|
|
158
|
+
log_error_log(messages.ErrorLogs.INVALID_BEARER_TOKEN.value, logger)
|
|
159
|
+
raise SkyflowError(
|
|
160
|
+
messages.Error.EXPIRED_BEARER_TOKEN.value
|
|
161
|
+
if config_id_type and config_id else messages.Error.EXPIRED_BEARER_TOKEN.value,
|
|
162
|
+
invalid_input_error_code
|
|
163
|
+
)
|
|
164
|
+
elif CredentialField.API_KEY in credentials:
|
|
165
|
+
validate_required_field(
|
|
166
|
+
logger, credentials, CredentialField.API_KEY, str,
|
|
167
|
+
messages.Error.EMPTY_API_KEY.value.format(config_id_type, config_id)
|
|
168
|
+
if config_id_type and config_id else messages.Error.EMPTY_API_KEY.value,
|
|
169
|
+
messages.Error.INVALID_API_KEY.value.format(config_id_type, config_id)
|
|
170
|
+
if config_id_type and config_id else messages.Error.INVALID_API_KEY.value,
|
|
171
|
+
messages=messages)
|
|
172
|
+
if not validate_api_key(credentials.get(CredentialField.API_KEY), logger, messages=messages):
|
|
173
|
+
raise SkyflowError(messages.Error.INVALID_API_KEY.value.format(config_id_type, config_id)
|
|
174
|
+
if config_id_type and config_id else messages.Error.INVALID_API_KEY.value,
|
|
175
|
+
invalid_input_error_code)
|
|
176
|
+
|
|
177
|
+
if CredentialField.TOKEN_URI_OPTION in credentials:
|
|
178
|
+
token_uri = credentials.get(CredentialField.TOKEN_URI_OPTION)
|
|
179
|
+
if (
|
|
180
|
+
token_uri is None
|
|
181
|
+
or not isinstance(token_uri, str)
|
|
182
|
+
or not is_valid_url(token_uri)
|
|
183
|
+
):
|
|
184
|
+
log_error_log(messages.ErrorLogs.INVALID_TOKEN_URI.value, logger)
|
|
185
|
+
raise SkyflowError(messages.Error.INVALID_TOKEN_URI.value, invalid_input_error_code)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def validate_log_level(logger, log_level, messages=None):
|
|
189
|
+
messages = messages or SkyflowMessages
|
|
190
|
+
if not isinstance(log_level, LogLevel):
|
|
191
|
+
log_error_log(messages.ErrorLogs.INVALID_LOG_LEVEL.value, logger)
|
|
192
|
+
raise SkyflowError(messages.Error.INVALID_LOG_LEVEL.value, invalid_input_error_code)
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def validate_keys(logger, config, config_keys, messages=None):
|
|
196
|
+
messages = messages or SkyflowMessages
|
|
197
|
+
for key in config.keys():
|
|
198
|
+
if key not in config_keys:
|
|
199
|
+
log_error_log(messages.ErrorLogs.INVALID_KEY.value.format(key), logger)
|
|
200
|
+
raise SkyflowError(messages.Error.INVALID_KEY.value.format(key), invalid_input_error_code)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def validate_non_empty_string_list(logger, value, error):
|
|
204
|
+
if not isinstance(value, list) or not value or not all(isinstance(item, str) and item.strip() for item in value):
|
|
205
|
+
raise SkyflowError(error, invalid_input_error_code)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def validate_vault_config(logger, config, messages=None, allowed_keys=None):
|
|
209
|
+
messages = messages or SkyflowMessages
|
|
210
|
+
allowed_keys = allowed_keys or VALID_VAULT_CONFIG_KEYS
|
|
211
|
+
log_info(messages.Info.VALIDATING_VAULT_CONFIG.value, logger)
|
|
212
|
+
validate_keys(logger, config, allowed_keys, messages=messages)
|
|
213
|
+
|
|
214
|
+
validate_required_field(
|
|
215
|
+
logger, config, ConfigField.VAULT_ID, str,
|
|
216
|
+
messages.Error.EMPTY_VAULT_ID.value,
|
|
217
|
+
messages.Error.INVALID_VAULT_ID.value,
|
|
218
|
+
messages=messages
|
|
219
|
+
)
|
|
220
|
+
vault_id = config.get(ConfigField.VAULT_ID)
|
|
221
|
+
|
|
222
|
+
validate_required_field(
|
|
223
|
+
logger, config, ConfigField.CLUSTER_ID, str,
|
|
224
|
+
messages.Error.EMPTY_CLUSTER_ID.value.format(vault_id),
|
|
225
|
+
messages.Error.INVALID_CLUSTER_ID.value.format(vault_id),
|
|
226
|
+
messages=messages
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
if ConfigField.CREDENTIALS in config and not config.get(ConfigField.CREDENTIALS):
|
|
230
|
+
raise SkyflowError(messages.Error.EMPTY_CREDENTIALS.value.format("vault", vault_id), invalid_input_error_code)
|
|
231
|
+
|
|
232
|
+
if ConfigField.CREDENTIALS in config and config.get(ConfigField.CREDENTIALS):
|
|
233
|
+
validate_credentials(logger, config.get(ConfigField.CREDENTIALS), "vault", vault_id, messages=messages)
|
|
234
|
+
|
|
235
|
+
if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env:
|
|
236
|
+
log_error_log(messages.ErrorLogs.ENV_IS_REQUIRED.value, logger)
|
|
237
|
+
raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code)
|
|
238
|
+
|
|
239
|
+
return True
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def validate_update_vault_config(logger, config, messages=None, allowed_keys=None):
|
|
243
|
+
"""Credentials are required on update (unlike on initial add, where they're optional)."""
|
|
244
|
+
messages = messages or SkyflowMessages
|
|
245
|
+
allowed_keys = allowed_keys or VALID_VAULT_CONFIG_KEYS
|
|
246
|
+
validate_keys(logger, config, allowed_keys, messages=messages)
|
|
247
|
+
|
|
248
|
+
validate_required_field(
|
|
249
|
+
logger, config, ConfigField.VAULT_ID, str,
|
|
250
|
+
messages.Error.EMPTY_VAULT_ID.value,
|
|
251
|
+
messages.Error.INVALID_VAULT_ID.value,
|
|
252
|
+
messages=messages
|
|
253
|
+
)
|
|
254
|
+
vault_id = config.get(ConfigField.VAULT_ID)
|
|
255
|
+
|
|
256
|
+
if ConfigField.CLUSTER_ID in config and not config.get(ConfigField.CLUSTER_ID):
|
|
257
|
+
raise SkyflowError(messages.Error.INVALID_CLUSTER_ID.value.format(vault_id), invalid_input_error_code)
|
|
258
|
+
|
|
259
|
+
if ConfigField.ENV in config and config.get(ConfigField.ENV) not in Env:
|
|
260
|
+
raise SkyflowError(messages.Error.INVALID_ENV.value.format(vault_id), invalid_input_error_code)
|
|
261
|
+
|
|
262
|
+
if ConfigField.CREDENTIALS not in config:
|
|
263
|
+
raise SkyflowError(messages.Error.EMPTY_CREDENTIALS.value.format("vault", vault_id), invalid_input_error_code)
|
|
264
|
+
|
|
265
|
+
validate_credentials(logger, config.get(ConfigField.CREDENTIALS), "vault", vault_id, messages=messages)
|
|
266
|
+
|
|
267
|
+
return True
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
from common.service_account import generate_bearer_token, generate_bearer_token_from_creds, is_expired
|
|
4
|
+
from common.utils import get_credentials, SkyflowMessages
|
|
5
|
+
from common.utils.logger import log_info
|
|
6
|
+
from common.utils.constants import OptionField, CredentialField, ConfigField
|
|
7
|
+
from common.utils.validations import validate_token_options
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class IVaultClient(ABC):
|
|
11
|
+
@abstractmethod
|
|
12
|
+
def resolve_vault_url(self, cluster_id, env, vault_id, logger=None):
|
|
13
|
+
raise NotImplementedError
|
|
14
|
+
|
|
15
|
+
@abstractmethod
|
|
16
|
+
def initialize_api_client(self, vault_url, bearer_token):
|
|
17
|
+
raise NotImplementedError
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class BaseVaultClient(IVaultClient):
|
|
21
|
+
def __init__(self, config):
|
|
22
|
+
self._config = config
|
|
23
|
+
self._common_skyflow_credentials = None
|
|
24
|
+
self._log_level = None
|
|
25
|
+
self._api_client = None
|
|
26
|
+
self._logger = None
|
|
27
|
+
self._is_config_updated = False
|
|
28
|
+
self._bearer_token = None
|
|
29
|
+
self._credentials = None
|
|
30
|
+
self._vault_url = None
|
|
31
|
+
self._is_static_token = None
|
|
32
|
+
|
|
33
|
+
def set_common_skyflow_credentials(self, credentials):
|
|
34
|
+
self._common_skyflow_credentials = credentials
|
|
35
|
+
|
|
36
|
+
def set_logger(self, log_level, logger):
|
|
37
|
+
self._log_level = log_level
|
|
38
|
+
self._logger = logger
|
|
39
|
+
|
|
40
|
+
def initialize_client_configuration(self):
|
|
41
|
+
if self._api_client is not None and not self._is_config_updated:
|
|
42
|
+
if self._is_static_token:
|
|
43
|
+
return
|
|
44
|
+
if self._bearer_token is not None and not is_expired(self._bearer_token):
|
|
45
|
+
return
|
|
46
|
+
|
|
47
|
+
needs_reinit = self._api_client is None or self._is_config_updated
|
|
48
|
+
if needs_reinit:
|
|
49
|
+
self._credentials = get_credentials(self._config.get(ConfigField.CREDENTIALS), self._common_skyflow_credentials, logger=self._logger)
|
|
50
|
+
self._vault_url = self.resolve_vault_url(self._config.get(ConfigField.CLUSTER_ID),
|
|
51
|
+
self._config.get(ConfigField.ENV),
|
|
52
|
+
self._config.get(ConfigField.VAULT_ID),
|
|
53
|
+
logger=self._logger)
|
|
54
|
+
self._is_static_token = CredentialField.TOKEN in self._credentials or CredentialField.API_KEY in self._credentials
|
|
55
|
+
bearer_token = self.get_bearer_token(self._credentials)
|
|
56
|
+
self._bearer_token = bearer_token
|
|
57
|
+
if needs_reinit:
|
|
58
|
+
self.initialize_api_client(self._vault_url, bearer_token)
|
|
59
|
+
|
|
60
|
+
def get_current_bearer_token(self):
|
|
61
|
+
return self._bearer_token
|
|
62
|
+
|
|
63
|
+
def get_current_vault_url(self):
|
|
64
|
+
return self._vault_url
|
|
65
|
+
|
|
66
|
+
def get_vault_id(self):
|
|
67
|
+
return self._config.get(ConfigField.VAULT_ID)
|
|
68
|
+
|
|
69
|
+
def get_bearer_token(self, credentials):
|
|
70
|
+
if CredentialField.API_KEY in credentials:
|
|
71
|
+
return credentials.get(CredentialField.API_KEY)
|
|
72
|
+
elif CredentialField.TOKEN in credentials:
|
|
73
|
+
return credentials.get(CredentialField.TOKEN)
|
|
74
|
+
|
|
75
|
+
validate_token_options(self._logger, credentials)
|
|
76
|
+
|
|
77
|
+
options = {}
|
|
78
|
+
if CredentialField.ROLES in credentials:
|
|
79
|
+
options[OptionField.ROLE_IDS] = credentials.get(CredentialField.ROLES)
|
|
80
|
+
|
|
81
|
+
if CredentialField.CONTEXT in credentials:
|
|
82
|
+
options[OptionField.CTX] = credentials.get(CredentialField.CONTEXT)
|
|
83
|
+
|
|
84
|
+
if CredentialField.TOKEN_URI_OPTION in credentials and credentials.get(CredentialField.TOKEN_URI_OPTION):
|
|
85
|
+
options[CredentialField.TOKEN_URI_OPTION] = credentials.get(CredentialField.TOKEN_URI_OPTION)
|
|
86
|
+
|
|
87
|
+
if self._bearer_token is None or self._is_config_updated or is_expired(self._bearer_token):
|
|
88
|
+
if CredentialField.PATH in credentials:
|
|
89
|
+
self._bearer_token, _ = generate_bearer_token(
|
|
90
|
+
credentials.get(CredentialField.PATH),
|
|
91
|
+
options,
|
|
92
|
+
self._logger
|
|
93
|
+
)
|
|
94
|
+
else:
|
|
95
|
+
credentials_string = credentials.get(CredentialField.CREDENTIALS_STRING)
|
|
96
|
+
log_info(SkyflowMessages.Info.GENERATE_BEARER_TOKEN_FROM_CREDENTIALS_STRING_TRIGGERED.value, self._logger)
|
|
97
|
+
self._bearer_token, _ = generate_bearer_token_from_creds(
|
|
98
|
+
credentials_string,
|
|
99
|
+
options,
|
|
100
|
+
self._logger
|
|
101
|
+
)
|
|
102
|
+
self._is_config_updated = False
|
|
103
|
+
else:
|
|
104
|
+
log_info(SkyflowMessages.Info.REUSE_BEARER_TOKEN.value, self._logger)
|
|
105
|
+
|
|
106
|
+
return self._bearer_token
|
|
107
|
+
|
|
108
|
+
def update_config(self, config):
|
|
109
|
+
self._config.update(config)
|
|
110
|
+
self._is_config_updated = True
|
|
111
|
+
|
|
112
|
+
def get_config(self):
|
|
113
|
+
return self._config
|
|
114
|
+
|
|
115
|
+
def get_common_skyflow_credentials(self):
|
|
116
|
+
return self._common_skyflow_credentials
|
|
117
|
+
|
|
118
|
+
def get_log_level(self):
|
|
119
|
+
return self._log_level
|
|
120
|
+
|
|
121
|
+
def get_logger(self):
|
|
122
|
+
return self._logger
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
from abc import ABC, abstractmethod
|
|
2
|
+
|
|
3
|
+
from common.errors import SkyflowError
|
|
4
|
+
from common.utils import SkyflowMessages as _CommonSkyflowMessages
|
|
5
|
+
from common.vault.data import BaseInsertRequest, BaseInsertResponse
|
|
6
|
+
|
|
7
|
+
_INVALID_INPUT_ERROR_CODE = _CommonSkyflowMessages.ErrorCodes.INVALID_INPUT.value
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class IVaultController(ABC):
|
|
11
|
+
|
|
12
|
+
@abstractmethod
|
|
13
|
+
def insert(self, request: BaseInsertRequest) -> BaseInsertResponse:
|
|
14
|
+
raise NotImplementedError
|
|
15
|
+
|
|
16
|
+
@abstractmethod
|
|
17
|
+
def get(self, request):
|
|
18
|
+
raise NotImplementedError
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def update(self, request):
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
|
|
24
|
+
@abstractmethod
|
|
25
|
+
def delete(self, request):
|
|
26
|
+
raise NotImplementedError
|
|
27
|
+
|
|
28
|
+
@abstractmethod
|
|
29
|
+
def detokenize(self, request):
|
|
30
|
+
raise NotImplementedError
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class BaseVaultController(IVaultController):
|
|
34
|
+
|
|
35
|
+
_skyflow_messages = None
|
|
36
|
+
|
|
37
|
+
def __init__(self, vault_client):
|
|
38
|
+
self._vault_client = vault_client
|
|
39
|
+
|
|
40
|
+
def _validate_table_name_if_present(self, table):
|
|
41
|
+
if table is not None and (not isinstance(table, str) or not table.strip()):
|
|
42
|
+
raise SkyflowError(
|
|
43
|
+
self._skyflow_messages.Error.INVALID_TABLE_NAME_IN_INSERT.value,
|
|
44
|
+
_INVALID_INPUT_ERROR_CODE,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def _validate_field_values(self, values):
|
|
48
|
+
if not isinstance(values, dict) or not values:
|
|
49
|
+
raise SkyflowError(
|
|
50
|
+
self._skyflow_messages.Error.INVALID_RECORD_DATA_IN_INSERT.value,
|
|
51
|
+
_INVALID_INPUT_ERROR_CODE,
|
|
52
|
+
)
|
|
53
|
+
for key, value in values.items():
|
|
54
|
+
if not isinstance(key, str) or not key.strip():
|
|
55
|
+
raise SkyflowError(
|
|
56
|
+
self._skyflow_messages.Error.EMPTY_KEY_IN_INSERT_DATA.value,
|
|
57
|
+
_INVALID_INPUT_ERROR_CODE,
|
|
58
|
+
)
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
class BaseInsertResponse:
|
|
2
|
+
|
|
3
|
+
def __init__(self, inserted_fields=None, errors=None):
|
|
4
|
+
self.inserted_fields = inserted_fields
|
|
5
|
+
self.errors = errors
|
|
6
|
+
|
|
7
|
+
def __repr__(self):
|
|
8
|
+
return f"{type(self).__name__}(inserted_fields={self.inserted_fields}, errors={self.errors})"
|
|
9
|
+
|
|
10
|
+
def __str__(self):
|
|
11
|
+
return self.__repr__()
|
skyflow/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .skyflow import Skyflow
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
from skyflow.utils._http_config import (
|
|
2
|
+
TIMEOUT_KEY,
|
|
3
|
+
CONNECT_TIMEOUT_KEY,
|
|
4
|
+
READ_TIMEOUT_KEY,
|
|
5
|
+
WRITE_TIMEOUT_KEY,
|
|
6
|
+
MAX_RETRIES_KEY,
|
|
7
|
+
INITIAL_RETRY_DELAY_MILLIS_KEY,
|
|
8
|
+
MAX_RETRY_DELAY_MILLIS_KEY,
|
|
9
|
+
)
|
|
10
|
+
from skyflow.utils.validations._validations import validate_http_config_value
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class HttpConfigBuilderMixin:
|
|
14
|
+
def __init__(self):
|
|
15
|
+
super().__init__()
|
|
16
|
+
self._client_http_config = {}
|
|
17
|
+
|
|
18
|
+
def _set_http_config(self, key, value):
|
|
19
|
+
validate_http_config_value(key, value)
|
|
20
|
+
self._client_http_config[key] = value
|
|
21
|
+
return self
|
|
22
|
+
|
|
23
|
+
def timeout(self, seconds):
|
|
24
|
+
return self._set_http_config(TIMEOUT_KEY, seconds)
|
|
25
|
+
|
|
26
|
+
def connect_timeout(self, seconds):
|
|
27
|
+
return self._set_http_config(CONNECT_TIMEOUT_KEY, seconds)
|
|
28
|
+
|
|
29
|
+
def read_timeout(self, seconds):
|
|
30
|
+
return self._set_http_config(READ_TIMEOUT_KEY, seconds)
|
|
31
|
+
|
|
32
|
+
def write_timeout(self, seconds):
|
|
33
|
+
return self._set_http_config(WRITE_TIMEOUT_KEY, seconds)
|
|
34
|
+
|
|
35
|
+
def max_retries(self, retries):
|
|
36
|
+
return self._set_http_config(MAX_RETRIES_KEY, retries)
|
|
37
|
+
|
|
38
|
+
def initial_retry_delay_millis(self, millis):
|
|
39
|
+
return self._set_http_config(INITIAL_RETRY_DELAY_MILLIS_KEY, millis)
|
|
40
|
+
|
|
41
|
+
def max_retry_delay_millis(self, millis):
|
|
42
|
+
return self._set_http_config(MAX_RETRY_DELAY_MILLIS_KEY, millis)
|
|
43
|
+
|
|
44
|
+
def _on_vault_client_created(self, vault_client):
|
|
45
|
+
vault_client.set_common_http_config(self._client_http_config)
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
from common.client.utils import make_skyflow_class
|
|
2
|
+
from common.utils import SkyflowMessages
|
|
3
|
+
from skyflow.client._http_config_builder import HttpConfigBuilderMixin
|
|
4
|
+
from skyflow.utils.validations import validate_vault_config, validate_update_vault_config
|
|
5
|
+
from skyflow.vault.client.client import VaultClient
|
|
6
|
+
from skyflow.vault.controller import VaultController
|
|
7
|
+
|
|
8
|
+
Skyflow = make_skyflow_class(
|
|
9
|
+
vault_client_cls=VaultClient,
|
|
10
|
+
vault_controller_cls=VaultController,
|
|
11
|
+
skyflow_messages=SkyflowMessages,
|
|
12
|
+
validate_vault_config=validate_vault_config,
|
|
13
|
+
validate_update_vault_config=validate_update_vault_config,
|
|
14
|
+
builder_mixins=(HttpConfigBuilderMixin,),
|
|
15
|
+
)
|
|
File without changes
|