qwak-core 0.4.278__py3-none-any.whl → 0.5.19__py3-none-any.whl

frogml_storage/authentication/utils/_authentication_utils.py

@@ -1,281 +0,0 @@
-import http
-import json
-import os
-from typing import Optional, Tuple, Union, List
-
-import requests
-from requests.auth import AuthBase, HTTPBasicAuth
-
-from frogml_storage.utils import join_url
-from frogml_storage.artifactory import ArtifactoryApi
-from frogml_storage.authentication.models import (
-    BearerAuth,
-    EmptyAuth,
-    AuthConfig,
-    LoginArguments,
-)
-from frogml_storage.constants import (
-    CONFIG_FILE_PATH,
-    FROG_ML_CONFIG_ACCESS_TOKEN,
-    FROG_ML_CONFIG_ARTIFACTORY_URL,
-    FROG_ML_CONFIG_PASSWORD,
-    FROG_ML_CONFIG_USER,
-    JF_ACCESS_TOKEN,
-    JF_URL,
-    JFROG_CLI_CONFIG_ACCESS_TOKEN,
-    JFROG_CLI_CONFIG_ARTIFACTORY_URL,
-    JFROG_CLI_CONFIG_FILE_PATH,
-    JFROG_CLI_CONFIG_PASSWORD,
-    JFROG_CLI_CONFIG_URL,
-    JFROG_CLI_CONFIG_USER,
-    SERVER_ID,
-)
-from frogml_storage.logging import logger
-
-
-def read_jfrog_cli_config() -> Union[dict, None]:
-    try:
-        with open(JFROG_CLI_CONFIG_FILE_PATH, "r") as file:
-            config_data = json.load(file)
-            return config_data
-    except FileNotFoundError:
-        logger.debug("JFrog cli config file was not found.")
-        return None
-    except json.JSONDecodeError as e:
-        print(f"JFrog cli config file is not a valid JSON {e}.")
-        return None
-
-
-def read_frogml_config() -> Union[dict, None]:
-    try:
-        with open(CONFIG_FILE_PATH, "r") as file:
-            config_data = json.load(file)
-            return config_data
-    except FileNotFoundError:
-        logger.debug("FrogMl config file was not found.")
-        return None
-    except json.JSONDecodeError as e:
-        print(f"FrogMl config file is not a valid JSON {e}.")
-        return None
-
-
-def parse_cli_config_server(server_config: dict) -> Union[LoginArguments, None]:
-    login_args = LoginArguments()
-    login_args.server_id = server_config.get(SERVER_ID)
-
-    if JFROG_CLI_CONFIG_ARTIFACTORY_URL in server_config:
-        login_args.artifactory_url = server_config.get(JFROG_CLI_CONFIG_ARTIFACTORY_URL)
-    elif (
-        JFROG_CLI_CONFIG_URL in server_config
-        and server_config.get(JFROG_CLI_CONFIG_URL) is not None
-    ):
-        login_args.artifactory_url = join_url(
-            server_config[JFROG_CLI_CONFIG_URL], "artifactory"
-        )
-    else:
-        logger.debug(
-            "Invalid JFrog CLI file, expected either artifactoryUrl or url in jfrog cli config file"
-        )
-        return None
-
-    if JFROG_CLI_CONFIG_ACCESS_TOKEN in server_config:
-        login_args.access_token = server_config.get(JFROG_CLI_CONFIG_ACCESS_TOKEN)
-    elif (
-        JFROG_CLI_CONFIG_USER in server_config
-        and JFROG_CLI_CONFIG_PASSWORD in server_config
-    ):
-        login_args.username = server_config.get(JFROG_CLI_CONFIG_USER)
-        login_args.password = server_config.get(JFROG_CLI_CONFIG_PASSWORD)
-    else:
-        logger.debug(
-            "Expected either accessToken or user/password in jfrog cli config file"
-        )
-        return None
-
-    return login_args
-
-
-def get_frogml_configuration() -> Union[LoginArguments, None]:
-    frog_ml_config = read_frogml_config()
-    if (
-        frog_ml_config is not None
-        and frog_ml_config.get("servers") is not None
-        and len(frog_ml_config["servers"]) > 0
-    ):
-        server_config = frog_ml_config["servers"][0]
-        login_args = LoginArguments()
-        if FROG_ML_CONFIG_ARTIFACTORY_URL in server_config:
-            login_args.artifactory_url = server_config.get(
-                FROG_ML_CONFIG_ARTIFACTORY_URL
-            )
-        else:
-            logger.debug(
-                "Invalid FrogMl authentication file, expected either artifactory_url in FrogMl authentication file"
-            )
-            return None
-
-        if FROG_ML_CONFIG_ACCESS_TOKEN in server_config:
-            login_args.access_token = server_config.get(FROG_ML_CONFIG_ACCESS_TOKEN)
-        elif (
-            FROG_ML_CONFIG_USER in server_config
-            and FROG_ML_CONFIG_PASSWORD in server_config
-        ):
-            login_args.username = server_config.get(FROG_ML_CONFIG_USER)
-            login_args.password = server_config.get(FROG_ML_CONFIG_PASSWORD)
-        elif (
-            FROG_ML_CONFIG_USER in server_config
-            and FROG_ML_CONFIG_PASSWORD not in server_config
-            or (
-                FROG_ML_CONFIG_USER not in server_config
-                and FROG_ML_CONFIG_PASSWORD in server_config
-            )
-        ):
-            logger.debug(
-                "Invalid FrogMl authentication file, username or password is missing in FrogMl authentication file"
-            )
-            return None
-        elif (
-            login_args.username is None
-            and login_args.password is None
-            and login_args.access_token is None
-        ):
-            login_args.isAnonymous = True
-        return login_args
-    else:
-        return None
-
-
-def get_encrypted_password(
-    auth_config: AuthConfig, auth_token: AuthBase
-) -> Optional[str]:
-    try:
-        response = ArtifactoryApi(
-            auth_config.artifactory_url, auth_token
-        ).encrypt_password()
-        if response.status_code != http.HTTPStatus.OK:
-            logger.debug(
-                f"Expected {http.HTTPStatus.OK} status but got {response.status_code} "
-                f"when using url {auth_config.artifactory_url}"
-            )
-            print("Error while trying to encrypt password.")
-            return None
-        return response.text
-    except requests.exceptions.RequestException as e:
-        print(f"Error while trying to encrypt password: {e}.")
-        return None
-
-
-def save_auth_config(auth_config: AuthConfig) -> None:
-    file_content: dict[str, list] = {}
-    file_content.setdefault("servers", []).append(auth_config.to_json())
-    os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True)
-    with open(CONFIG_FILE_PATH, "w") as file:
-        json.dump(file_content, file, indent=2)
-
-
-def get_credentials(auth_config: Optional[AuthConfig] = None) -> Tuple[str, AuthBase]:
-    if not __should_use_file_auth(auth_config):
-        __validate_credentials(auth_config)
-        return __auth_config_to_auth_tuple(auth_config)
-    logger.debug(
-        "Login configuration not supplied, attempting to find environment variables"
-    )
-
-    if __should_use_environment_variables():
-        return get_environment_variables()
-
-    logger.debug(
-        "Environment variables not supplied, attempting to load configuration from file"
-    )
-
-    if os.path.exists(CONFIG_FILE_PATH):
-        return __read_credentials_from_file(CONFIG_FILE_PATH)
-    raise ValueError(
-        f"Configuration were not provided and configuration file not found in {CONFIG_FILE_PATH},"
-        f" either pass configuration in the constructor, add env variables or create the configuration file by "
-        f"running `frogml login`"
-    )
-
-
-def __should_use_environment_variables() -> bool:
-    return os.getenv("JF_URL") is not None
-
-
-def get_environment_variables() -> Tuple[str, AuthBase]:
-    auth_config: AuthConfig = AuthConfig(
-        artifactory_url=os.getenv(JF_URL),
-        access_token=os.getenv(JF_ACCESS_TOKEN),
-    )
-
-    return __auth_config_to_auth_tuple(auth_config)
-
-
-def __should_use_file_auth(credentials: Optional[AuthConfig] = None) -> bool:
-    return credentials is None or (
-        credentials.artifactory_url is None
-        and credentials.user is None
-        and credentials.password is None
-        and credentials.access_token is None
-    )
-
-
-def __validate_credentials(credentials: Optional[AuthConfig]) -> None:
-    if credentials is None:
-        raise ValueError("Credentials must be provided.")
-    if credentials.artifactory_url is None:
-        raise ValueError("Credentials must contain artifactory url.")
-    return None
-
-
-def __read_credentials_from_file(file_path: str) -> Tuple[str, AuthBase]:
-    try:
-        with open(file_path, "r") as file:
-            config_content: dict = json.load(file)
-            servers = config_content.get("servers")
-            if servers is None or len(servers) == 0:
-                raise ValueError(
-                    "Configuration file was found but it's empty, failing authentication"
-                )
-            server = servers[0]
-            return __auth_config_to_auth_tuple(AuthConfig.from_dict(server))
-    except json.JSONDecodeError:
-        raise ValueError(f"Error when reading {file_path}, please recreate the file.")
-
-
-def __auth_config_to_auth_tuple(
-    auth_config: Optional[AuthConfig],
-) -> Tuple[str, AuthBase]:
-
-    if auth_config is None:
-        raise ValueError("No authentication configuration provided")
-
-    auth: AuthBase = EmptyAuth()
-    if auth_config.access_token is not None:
-        auth = BearerAuth(auth_config.access_token)
-    elif auth_config.user is not None and auth_config.password is not None:
-        auth = HTTPBasicAuth(auth_config.user, auth_config.password)
-    elif auth_config.user is not None or auth_config.password is not None:
-        raise ValueError("User and password must be provided together")
-
-    if auth_config.artifactory_url is None:
-        raise ValueError("No artifactory URL provided")
-
-    return auth_config.artifactory_url, auth
-
-
-def get_list_of_servers_from_config(jfrog_cli_config: dict) -> List[str]:
-    if jfrog_cli_config is not None:
-        servers = jfrog_cli_config.get("servers")
-        if servers is not None:
-            return list(map(__map_server_ids, servers))
-
-    return []
-
-
-def __map_server_ids(server: dict) -> str:
-    server_id = ""
-    if server is not None:
-        server_id = str(server.get("serverId"))
-        if server.get("isDefault") is not None and bool(server.get("isDefault")):
-            server_id = server_id + " (Default)"
-    return server_id

frogml_storage/authentication/utils/_login_checks_utils.py

@@ -1,114 +0,0 @@
-from typing import Optional
-
-from frogml_storage.authentication.models import AuthConfig
-
-
-def login_input_checks(
-    url: Optional[str] = None,
-    username: Optional[str] = None,
-    password: Optional[str] = None,
-    token: Optional[str] = None,
-    anonymous: bool = False,
-) -> bool:
-    return (
-        __is_user_name_password_command(url, username, password, token, anonymous)
-        or __is_access_token_command(url, username, password, token, anonymous)
-        or __is_anonymous_command(url, username, password, token, anonymous)
-    )
-
-
-def is_login_without_params(
-    url: Optional[str] = None,
-    username: Optional[str] = None,
-    password: Optional[str] = None,
-    token: Optional[str] = None,
-    anonymous: bool = False,
-) -> bool:
-    return (
-        all(params is None for params in [url, username, password, token])
-        and not anonymous
-    )
-
-
-def is_username_password_login(
-    auth_config: AuthConfig, anonymous: Optional[bool] = False
-) -> bool:
-    return __is_user_name_password_command(
-        url=auth_config.artifactory_url,
-        username=auth_config.user,
-        password=auth_config.password,
-        token=auth_config.access_token,
-        anonymous=anonymous,
-    )
-
-
-def is_access_token_login(
-    auth_config: AuthConfig, anonymous: Optional[bool] = False
-) -> bool:
-    if anonymous is None:
-        anonymous = False
-    return __is_access_token_command(
-        url=auth_config.artifactory_url,
-        username=auth_config.user,
-        password=auth_config.password,
-        token=auth_config.access_token,
-        anonymous=anonymous,
-    )
-
-
-def is_anonymous_login(
-    auth_config: AuthConfig, anonymous: Optional[bool] = False
-) -> bool:
-    if anonymous is None:
-        anonymous = False
-    return __is_anonymous_command(
-        url=auth_config.artifactory_url,
-        username=auth_config.user,
-        password=auth_config.password,
-        token=auth_config.access_token,
-        anonymous=anonymous,
-    )
-
-
-def __is_anonymous_command(
-    url: Optional[str],
-    username: Optional[str],
-    password: Optional[str],
-    token: Optional[str],
-    anonymous: bool,
-) -> bool:
-    return (
-        anonymous
-        and all(params is None for params in [username, password, token])
-        and url is not None
-    )
-
-
-def __is_access_token_command(
-    url: Optional[str],
-    username: Optional[str],
-    password: Optional[str],
-    token: Optional[str],
-    anonymous: bool,
-) -> bool:
-    if anonymous:
-        return False
-    return all(params is None for params in [username, password]) and all(
-        params is not None for params in [url, token]
-    )
-
-
-def __is_user_name_password_command(
-    url: Optional[str],
-    username: Optional[str],
-    password: Optional[str],
-    token: Optional[str],
-    anonymous: Optional[bool] = False,
-) -> bool:
-    return (
-        not anonymous
-        and url is not None
-        and username is not None
-        and password is not None
-        and token is None
-    )

frogml_storage/base_storage.py

@@ -1,140 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Union
-
-from frogml_storage.models.frogml_dataset_version import FrogMLDatasetVersion
-from frogml_storage.models.frogml_model_version import FrogMLModelVersion
-from frogml_storage.models.serialization_metadata import SerializationMetadata
-
-
-class BaseStorage(ABC):
-    """
-    Repository storage to download or store model | dataset artifacts,
-    metrics and relation between these in an Artifactory repository.
-    """
-
-    @abstractmethod
-    def upload_model_version(
-        self,
-        repository: str,
-        model_name: str,
-        model_path: str,
-        model_type: Union[SerializationMetadata, Dict],
-        namespace: Optional[str] = None,
-        version: Optional[str] = None,
-        properties: Optional[dict[str, str]] = None,
-        dependencies_files_paths: Optional[List[str]] = None,
-        code_archive_file_path: Optional[str] = None,
-    ) -> FrogMLModelVersion:
-        """Upload model to a repository in Artifactory. Uploaded models should be stored with the following layout:
-        ├── REPO
-            ├── models
-                ├── ${NAMESPACE}
-                    ├── ${MODEL_NAME}
-                        ├── ${MODEL_VERSION}
-                            ├── model-manifest.json
-                            ├── model
-                                ├── model.pkl
-                                ├── evidence.json
-                                ├── ...
-                            ├── code
-                                ├── code.zip
-                                ├── requirements.txt
-        :param repository: the repository to upload the model to
-        :param model_name: the name of the model
-        :param model_path: the source path of the model
-        :param model_type: the type of the model (PyTorch, HuggingFace, Catboost)
-        :param namespace: the namespace to upload the model to
-        :param version: the version of the model
-        :param properties: tags to associate with the model
-        :param dependencies_files_paths: the list of dependencies files paths
-        :param code_archive_file_path: the path to the code archive file
-        """
-        pass
-
-    @abstractmethod
-    def download_model_version(
-        self,
-        repository: str,
-        model_name: str,
-        version: str,
-        target_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads a model from an Artifactory repository
-        :param repository: the repository to download the model from
-        :param model_name: the name of the model to download
-        :param version: the version of the model to download
-        :param target_path: the target local path to store the model in
-        :param namespace: the namespace of the model to download
-
-        """
-        pass
-
-    def download_model_single_artifact(
-        self,
-        repository: str,
-        model_name: str,
-        version: str,
-        target_path: str,
-        source_artifact_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads an artifact from an Artifactory repository
-        :param repository: the repository to download the artifact from
-        :param model_name: the model to download artifact from
-        :param version: the model's version to download artifact from
-        :param target_path: the target local path to store the artifact in
-        :param source_artifact_path: the path of the artifact under the model's version directory
-        :param namespace: the namespace of the artifact
-        """
-        pass
-
-    @abstractmethod
-    def upload_dataset_version(
-        self,
-        repository: str,
-        dataset_name: str,
-        source_path: str,
-        namespace: Optional[str] = None,
-        version: Optional[str] = None,
-        tags: Optional[dict[str, str]] = None,
-    ) -> FrogMLDatasetVersion:
-        """Uploads a dataset to a repository in Artifactory. Uploaded datasets should be stored with
-         the following layout:
-        ├── REPO
-            ├── datasets
-                ├── ${NAMESPACE}
-                    ├── ${DATASET_NAME}
-                        ├── ${DATASET_VERSION}
-                            ├── dataset-manifest.json
-                            ├── dataset
-                                ├── cli_docs_1.csv
-                                ├── cli_docs_2.csv
-                                ├── ...
-        :param repository: the repository to upload the dataset to
-        :param dataset_name: the name of the dataset
-        :param source_path: the source path of the dataset
-        :param namespace: the namespace to upload the dataset to
-        :param version: the version of the dataset
-        :param tags: tags to associate with the dataset
-        """
-        pass
-
-    @abstractmethod
-    def download_dataset_version(
-        self,
-        repository: str,
-        dataset_name: str,
-        version: str,
-        target_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads a dataset from an Artifactory repository
-        :param repository: the repository to download the dataset from
-        :param dataset_name: the name of the dataset to download
-        :param version: the version of the dataset to download
-        :param target_path: the target local path to store the dataset in
-        :param namespace: the namespace of the dataset to download
-
-        """
-        pass

frogml_storage/constants.py

@@ -1,56 +0,0 @@
-import os
-
-home_dir = os.path.expanduser("~")
-
-JFROG_CLI_CONFIG_FILE_PATH = os.path.join(home_dir, ".jfrog", "jfrog-cli.conf.v6")
-CONFIG_FILE_PATH = os.path.join(home_dir, ".frogml", "config.json")
-
-FROG_ML_CONFIG_USER = "user"
-FROG_ML_CONFIG_ARTIFACTORY_URL = "artifactory_url"
-FROG_ML_CONFIG_PASSWORD = "password"  # nosec B105
-FROG_ML_CONFIG_ACCESS_TOKEN = "access_token"  # nosec B105
-FROG_ML_DEFAULT_HTTP_THREADS_COUNT = 5
-FROG_ML_MAX_CHARS_FOR_NAME = 60
-
-SERVER_ID = "server_id"
-
-JFROG_CLI_CONFIG_ARTIFACTORY_URL = "artifactoryUrl"
-JFROG_CLI_CONFIG_URL = "url"
-JFROG_CLI_CONFIG_USER = "user"
-JFROG_CLI_CONFIG_PASSWORD = "password"  # nosec B105
-JFROG_CLI_CONFIG_ACCESS_TOKEN = "accessToken"  # nosec B105
-
-MODEL = "model"
-ROOT_FROGML_MODEL_UI_DIRECTORY = "models"
-MODEL_UI_DIRECTORY = "model"
-MODEL_METADATA_FILE_NAME = "model-manifest.json"
-BODY_PART_MODEL_MANIFEST_STREAM = "modelManifest"
-
-DATASET = "dataset"
-ROOT_FROGML_DATASET_UI_DIRECTORY = "datasets"
-DATASET_UI_DIRECTORY = "dataset"
-DATASET_METADATA_FILE_NAME = "dataset-manifest.json"
-BODY_PART_DATASET_MANIFEST_STREAM = "datasetManifest"
-
-CHECKSUM_SHA2_HEADER = "X-Checksum-Sha256"
-
-JFML_THREAD_COUNT = "JFML_THREAD_COUNT"
-JF_URL = "JF_URL"
-JF_ACCESS_TOKEN = "JF_ACCESS_TOKEN"  # nosec B105
-
-FROG_ML_IGNORED_FILES = [
-    ".DS_Store",
-    "CVS",
-    ".cvsignore",
-    "SCCS",
-    "vssver.scc",
-    ".svn",
-    ".git",
-    ".gitignore",
-    ".gitattributes",
-    ".gitmodules",
-    ".gitkeep",
-    ".gitconfig",
-    MODEL_METADATA_FILE_NAME,
-    DATASET_METADATA_FILE_NAME,
-]

frogml_storage/exceptions/checksum_verification_error.py

@@ -1,3 +0,0 @@
-class ChecksumVerificationError(Exception):
-    def __init__(self, filename):
-        super().__init__(f"Checksum verification failed for file: '{filename}'")

frogml_storage/exceptions/validation_error.py

@@ -1,4 +0,0 @@
-class FrogMLValidationError(ValueError):
-    def __init__(self, message, errors=None):
-        super().__init__(message)
-        self.errors = errors or {}