skyflow-flowvault-python 1.0.0__py3-none-any.whl

skyflow/utils/validations/_validations.py

@@ -0,0 +1,195 @@
+from common.errors import SkyflowError
+from common.utils import SkyflowMessages as CommonMessages
+from common.utils.validations import (
+    validate_keys,
+    validate_credentials,
+    validate_non_empty_string_list,
+)
+from common.utils.validations import validate_vault_config as _common_validate_vault_config
+from common.utils.validations import validate_update_vault_config as _common_validate_update_vault_config
+from skyflow.utils import SkyflowMessages
+from skyflow.utils.enums import UpsertType
+from skyflow.utils._http_config import (
+    POSITIVE_SECOND_KEYS,
+    NON_NEGATIVE_INT_KEYS,
+    VAULT_URL_KEY,
+    VAULT_CONFIG_KEYS,
+)
+from skyflow.vault.data import GetRequestRecord, InsertRequestRecord, UpdateRequestRecord, UpsertOptions, TokenGroupRedactions
+
+VALID_UPDATE_RECORD_KEYS = ["skyflow_id", "data", "tokens", "table_name"]
+
+invalid_input_error_code = CommonMessages.ErrorCodes.INVALID_INPUT.value
+
+
+def _is_number(value):
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def validate_http_config_value(key, value):
+    if key in POSITIVE_SECOND_KEYS:
+        if not _is_number(value) or value <= 0:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_TIMEOUT.value.format(key), invalid_input_error_code)
+    elif key in NON_NEGATIVE_INT_KEYS:
+        if isinstance(value, bool) or not isinstance(value, int) or value < 0:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_RETRY_SETTING.value.format(key), invalid_input_error_code)
+
+
+def _validate_http_config(logger, config):
+    for key in (*POSITIVE_SECOND_KEYS, *NON_NEGATIVE_INT_KEYS):
+        if key in config:
+            validate_http_config_value(key, config[key])
+    if VAULT_URL_KEY in config:
+        vault_url = config[VAULT_URL_KEY]
+        if not isinstance(vault_url, str) or not vault_url:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_VAULT_URL.value, invalid_input_error_code)
+
+
+def validate_vault_config(logger, config):
+    _validate_http_config(logger, config)
+    return _common_validate_vault_config(logger, config, messages=CommonMessages, allowed_keys=VAULT_CONFIG_KEYS)
+
+
+def validate_update_vault_config(logger, config):
+    _validate_http_config(logger, config)
+    return _common_validate_update_vault_config(logger, config, messages=CommonMessages, allowed_keys=VAULT_CONFIG_KEYS)
+
+
+def _validate_upsert(logger, upsert):
+    if upsert is None:
+        return
+    if not isinstance(upsert, UpsertOptions):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_TYPE_IN_INSERT.value, invalid_input_error_code)
+    unique_columns = upsert.unique_columns
+    if (not isinstance(unique_columns, list) or not unique_columns
+            or not all(isinstance(c, str) for c in unique_columns)):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_UNIQUE_COLUMNS_IN_INSERT.value, invalid_input_error_code)
+    if upsert.update_type is not None and not isinstance(upsert.update_type, UpsertType):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_UPSERT_UPDATE_TYPE_IN_INSERT.value, invalid_input_error_code)
+
+
+def validate_insert_request(logger, request):
+    if not isinstance(request.records, list) or not all(isinstance(r, InsertRequestRecord) for r in request.records):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_INSERT.value, invalid_input_error_code)
+
+    if not request.records:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_INSERT.value, invalid_input_error_code)
+
+    _validate_upsert(logger, request.upsert)
+    for record in request.records:
+        _validate_upsert(logger, record.upsert)
+
+    table_at_request_level = request.table_name is not None
+
+    if table_at_request_level:
+        for record in request.records:
+            if record.table_name is not None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_INSERT.value, invalid_input_error_code)
+    else:
+        for record in request.records:
+            if record.table_name is None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_INSERT.value, invalid_input_error_code)
+
+    if table_at_request_level:
+        for record in request.records:
+            if record.upsert is not None:
+                raise SkyflowError(SkyflowMessages.Error.RECORD_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code)
+    elif request.upsert is not None:
+        raise SkyflowError(SkyflowMessages.Error.REQUEST_LEVEL_UPSERT_NOT_ALLOWED_IN_INSERT.value, invalid_input_error_code)
+
+
+def validate_get_request(logger, request):
+    if request.records is not None:
+        single_table_fields_set = (
+            request.table_name or request.skyflow_ids or request.unique_values or request.columns
+            or request.column_redactions or request.limit is not None or request.offset is not None
+        )
+        if single_table_fields_set:
+            raise SkyflowError(SkyflowMessages.Error.GET_MODE_CONFLICT.value, invalid_input_error_code)
+        if (not isinstance(request.records, list) or not request.records
+                or not all(isinstance(r, GetRequestRecord) for r in request.records)):
+            raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_GET.value, invalid_input_error_code)
+        for record in request.records:
+            if not record.table_name:
+                raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_GET.value, invalid_input_error_code)
+            if not record.skyflow_ids and not record.unique_values:
+                raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_GET.value, invalid_input_error_code)
+            if record.skyflow_ids is not None:
+                validate_non_empty_string_list(logger, record.skyflow_ids, SkyflowMessages.Error.INVALID_IDS_IN_GET.value)
+        return
+
+    if not request.table_name:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_GET.value, invalid_input_error_code)
+
+    if not request.skyflow_ids and not request.unique_values:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_GET.value, invalid_input_error_code)
+
+    if request.skyflow_ids is not None:
+        validate_non_empty_string_list(logger, request.skyflow_ids, SkyflowMessages.Error.INVALID_IDS_IN_GET.value)
+
+
+def validate_update_request(logger, request):
+    if not isinstance(request.records, list) or not all(isinstance(r, UpdateRequestRecord) for r in request.records):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_RECORDS_TYPE_IN_UPDATE.value, invalid_input_error_code)
+
+    if not request.records:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_RECORDS_IN_UPDATE.value, invalid_input_error_code)
+
+    if request.update_type is not None and not isinstance(request.update_type, UpsertType):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_UPDATE_TYPE_IN_UPDATE.value, invalid_input_error_code)
+
+    for record in request.records:
+        skyflow_id = record.skyflow_id
+        if not isinstance(skyflow_id, str) or not skyflow_id.strip():
+            raise SkyflowError(SkyflowMessages.Error.MISSING_SKYFLOW_ID_IN_UPDATE.value, invalid_input_error_code)
+        data = record.data
+        if data is None:
+            raise SkyflowError(SkyflowMessages.Error.MISSING_DATA_IN_UPDATE.value, invalid_input_error_code)
+        if not isinstance(data, dict):
+            raise SkyflowError(SkyflowMessages.Error.INVALID_DATA_TYPE_IN_UPDATE.value, invalid_input_error_code)
+        if not data:
+            raise SkyflowError(SkyflowMessages.Error.MISSING_DATA_IN_UPDATE.value, invalid_input_error_code)
+
+    table_at_request_level = request.table_name is not None
+
+    if table_at_request_level:
+        for record in request.records:
+            if record.table_name is not None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_IN_BOTH_PLACES_IN_UPDATE.value, invalid_input_error_code)
+    else:
+        for record in request.records:
+            if record.table_name is None:
+                raise SkyflowError(SkyflowMessages.Error.TABLE_NAME_MISSING_IN_UPDATE.value, invalid_input_error_code)
+
+
+def validate_delete_request(logger, request):
+    if not request.table_name:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_TABLE_NAME_IN_DELETE.value, invalid_input_error_code)
+
+    if not request.ids and not request.unique_values:
+        raise SkyflowError(SkyflowMessages.Error.MISSING_IDS_OR_UNIQUE_VALUES_IN_DELETE.value, invalid_input_error_code)
+
+    if request.ids is not None:
+        validate_non_empty_string_list(logger, request.ids, SkyflowMessages.Error.INVALID_IDS_IN_DELETE.value)
+
+
+def validate_detokenize_request(logger, request):
+    if (
+        not isinstance(request.tokens, list) or not all(isinstance(t, str) and t.strip() for t in request.tokens)
+    ):
+        raise SkyflowError(SkyflowMessages.Error.INVALID_TOKENS_TYPE_IN_DETOKENIZE.value, invalid_input_error_code)
+
+    if not request.tokens:
+        raise SkyflowError(SkyflowMessages.Error.EMPTY_TOKENS_IN_DETOKENIZE.value, invalid_input_error_code)
+
+    if request.token_group_redactions is not None:
+        valid = (
+            isinstance(request.token_group_redactions, list)
+            and all(
+                isinstance(entry, TokenGroupRedactions)
+                and isinstance(entry.token_group_name, str) and entry.token_group_name.strip()
+                for entry in request.token_group_redactions
+            )
+        )
+        if not valid:
+            raise SkyflowError(SkyflowMessages.Error.INVALID_TOKEN_GROUP_REDACTIONS_IN_DETOKENIZE.value, invalid_input_error_code)

skyflow/vault/client/client.py

@@ -0,0 +1,102 @@
+import httpx
+
+from common.vault.base_vault_client import BaseVaultClient
+from skyflow.generated.rest.client import SkyflowAuth
+from skyflow.utils import get_vault_url
+from skyflow.utils._http_config import (
+    TIMEOUT_KEY,
+    CONNECT_TIMEOUT_KEY,
+    READ_TIMEOUT_KEY,
+    WRITE_TIMEOUT_KEY,
+    MAX_RETRIES_KEY,
+    INITIAL_RETRY_DELAY_MILLIS_KEY,
+    MAX_RETRY_DELAY_MILLIS_KEY,
+    VAULT_URL_KEY,
+    DEFAULT_TIMEOUT,
+    DEFAULT_CONNECT_TIMEOUT,
+    DEFAULT_READ_TIMEOUT,
+    DEFAULT_WRITE_TIMEOUT,
+    DEFAULT_MAX_RETRIES,
+    DEFAULT_INITIAL_RETRY_DELAY_MILLIS,
+    DEFAULT_MAX_RETRY_DELAY_MILLIS,
+    DEFAULT_MAX_CONNECTIONS,
+    DEFAULT_MAX_KEEPALIVE_CONNECTIONS,
+    DEFAULT_KEEPALIVE_EXPIRY,
+    resolve_setting,
+)
+from skyflow.utils._retry import RetryTransport
+
+
+class VaultClient(BaseVaultClient):
+    def __init__(self, config):
+        super().__init__(config)
+        self._common_http_config = {}
+        self._sync_httpx_client = None
+
+    def set_common_http_config(self, common_http_config):
+        self._common_http_config = common_http_config or {}
+
+    def _resolve(self, key, default):
+        return resolve_setting(self._config, self._common_http_config, key, default)
+
+    def resolve_vault_url(self, cluster_id, env, vault_id, logger=None):
+        override = self._config.get(VAULT_URL_KEY)
+        if override:
+            return override
+        return get_vault_url(cluster_id, env, vault_id, logger=logger)
+
+    def _build_timeout(self):
+        read = self._resolve(READ_TIMEOUT_KEY, DEFAULT_READ_TIMEOUT)
+        return httpx.Timeout(
+            connect=self._resolve(CONNECT_TIMEOUT_KEY, DEFAULT_CONNECT_TIMEOUT),
+            read=read,
+            write=self._resolve(WRITE_TIMEOUT_KEY, DEFAULT_WRITE_TIMEOUT),
+            pool=read,
+        )
+
+    def _retry_params(self):
+        return (
+            self._resolve(MAX_RETRIES_KEY, DEFAULT_MAX_RETRIES),
+            self._resolve(INITIAL_RETRY_DELAY_MILLIS_KEY, DEFAULT_INITIAL_RETRY_DELAY_MILLIS),
+            self._resolve(MAX_RETRY_DELAY_MILLIS_KEY, DEFAULT_MAX_RETRY_DELAY_MILLIS),
+            self._resolve(TIMEOUT_KEY, DEFAULT_TIMEOUT),
+        )
+
+    def _build_limits(self):
+        return httpx.Limits(
+            max_connections=DEFAULT_MAX_CONNECTIONS,
+            max_keepalive_connections=DEFAULT_MAX_KEEPALIVE_CONNECTIONS,
+            keepalive_expiry=DEFAULT_KEEPALIVE_EXPIRY,
+        )
+
+    def initialize_api_client(self, vault_url, bearer_token):
+        self.__close_httpx_clients()
+        timeout = self._build_timeout()
+        limits = self._build_limits()
+        max_retries, initial_millis, max_millis, call_timeout = self._retry_params()
+        sync_client = httpx.Client(
+            timeout=timeout,
+            follow_redirects=True,
+            transport=RetryTransport(httpx.HTTPTransport(limits=limits), max_retries, initial_millis, max_millis, call_timeout),
+        )
+        self._sync_httpx_client = sync_client
+        self._api_client = SkyflowAuth(base_url=vault_url, token=bearer_token or "", httpx_client=sync_client)
+
+    def close(self):
+        self.__close_httpx_clients()
+        self._api_client = None
+
+    def __close_httpx_clients(self):
+        sync_client = self._sync_httpx_client
+        if sync_client is not None:
+            try:
+                sync_client.close()
+            except Exception:
+                pass
+        self._sync_httpx_client = None
+
+    def get_records_api(self):
+        return self._api_client.records
+
+    def get_tokens_api(self):
+        return self._api_client.tokens

skyflow/vault/controller/__init__.py

@@ -0,0 +1 @@
+from ._vault import VaultController