qwak-core 0.4.272__py3-none-any.whl → 0.4.274__py3-none-any.whl

frogml_storage/authentication/models/_auth_config.py

@@ -0,0 +1,70 @@
+from typing import Optional
+
+from frogml_storage.constants import (
+    FROG_ML_CONFIG_ACCESS_TOKEN,
+    FROG_ML_CONFIG_ARTIFACTORY_URL,
+    FROG_ML_CONFIG_PASSWORD,
+    FROG_ML_CONFIG_USER,
+    SERVER_ID,
+)
+
+
+class AuthConfig(object):
+    def __init__(
+        self,
+        artifactory_url: Optional[str] = None,
+        access_token: Optional[str] = None,
+        user: Optional[str] = None,
+        password: Optional[str] = None,
+        server_id: Optional[str] = None,
+    ) -> None:
+        self.artifactory_url = artifactory_url
+        self.user = user
+        self.password = password
+        self.access_token = access_token
+        self.server_id = server_id
+
+    def to_json(self):
+        include_keys = [
+            FROG_ML_CONFIG_ARTIFACTORY_URL,
+            FROG_ML_CONFIG_USER,
+            FROG_ML_CONFIG_PASSWORD,
+            FROG_ML_CONFIG_ACCESS_TOKEN,
+            SERVER_ID,
+        ]
+        return {
+            key: getattr(self, key)
+            for key in include_keys
+            if getattr(self, key) is not None
+        }
+
+    @classmethod
+    def by_access_token(cls, artifactory_url: str, access_token: str) -> "AuthConfig":
+        return cls(
+            artifactory_url=artifactory_url,
+            access_token=access_token,
+        )
+
+    @classmethod
+    def by_basic_auth(
+        cls, artifactory_url: str, user: str, password: str
+    ) -> "AuthConfig":
+        return cls(
+            artifactory_url=artifactory_url,
+            user=user,
+            password=password,
+        )
+
+    @classmethod
+    def by_server_id(cls, server_id: str) -> "AuthConfig":
+        return cls(server_id=server_id)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "AuthConfig":
+        return cls(
+            artifactory_url=data.get(FROG_ML_CONFIG_ARTIFACTORY_URL),
+            access_token=data.get(FROG_ML_CONFIG_ACCESS_TOKEN),
+            server_id=data.get(SERVER_ID),
+            user=data.get(FROG_ML_CONFIG_USER),
+            password=data.get(FROG_ML_CONFIG_PASSWORD),
+        )

frogml_storage/authentication/models/_login.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+
+class LoginArguments(object):
+    isAnonymous: bool = False
+    server_id: Optional[str] = None
+    username: Optional[str] = None
+    password: Optional[str] = None
+    access_token: Optional[str] = None
+    artifactory_url: Optional[str] = None
+
+    def __eq__(self, other):
+        if not isinstance(other, LoginArguments):
+            return False
+        return (
+            self.isAnonymous == other.isAnonymous
+            and self.server_id == other.server_id
+            and self.username == other.username
+            and self.password == other.password
+            and self.access_token == other.access_token
+            and self.artifactory_url == other.artifactory_url
+        )

frogml_storage/authentication/utils/__init__.py

@@ -0,0 +1,17 @@
+from ._authentication_utils import (
+    get_credentials,
+    get_frogml_configuration,
+    parse_cli_config_server,
+    read_jfrog_cli_config,
+    get_encrypted_password,
+    save_auth_config,
+    get_list_of_servers_from_config,
+    read_frogml_config,
+)
+from ._login_checks_utils import (
+    login_input_checks,
+    is_login_without_params,
+    is_username_password_login,
+    is_access_token_login,
+    is_anonymous_login,
+)

frogml_storage/authentication/utils/_authentication_utils.py

@@ -0,0 +1,281 @@
+import http
+import json
+import os
+from typing import Optional, Tuple, Union, List
+
+import requests
+from requests.auth import AuthBase, HTTPBasicAuth
+
+from frogml_storage.utils import join_url
+from frogml_storage.artifactory import ArtifactoryApi
+from frogml_storage.authentication.models import (
+    BearerAuth,
+    EmptyAuth,
+    AuthConfig,
+    LoginArguments,
+)
+from frogml_storage.constants import (
+    CONFIG_FILE_PATH,
+    FROG_ML_CONFIG_ACCESS_TOKEN,
+    FROG_ML_CONFIG_ARTIFACTORY_URL,
+    FROG_ML_CONFIG_PASSWORD,
+    FROG_ML_CONFIG_USER,
+    JF_ACCESS_TOKEN,
+    JF_URL,
+    JFROG_CLI_CONFIG_ACCESS_TOKEN,
+    JFROG_CLI_CONFIG_ARTIFACTORY_URL,
+    JFROG_CLI_CONFIG_FILE_PATH,
+    JFROG_CLI_CONFIG_PASSWORD,
+    JFROG_CLI_CONFIG_URL,
+    JFROG_CLI_CONFIG_USER,
+    SERVER_ID,
+)
+from frogml_storage.logging import logger
+
+
+def read_jfrog_cli_config() -> Union[dict, None]:
+    try:
+        with open(JFROG_CLI_CONFIG_FILE_PATH, "r") as file:
+            config_data = json.load(file)
+            return config_data
+    except FileNotFoundError:
+        logger.debug("JFrog cli config file was not found.")
+        return None
+    except json.JSONDecodeError as e:
+        print(f"JFrog cli config file is not a valid JSON {e}.")
+        return None
+
+
+def read_frogml_config() -> Union[dict, None]:
+    try:
+        with open(CONFIG_FILE_PATH, "r") as file:
+            config_data = json.load(file)
+            return config_data
+    except FileNotFoundError:
+        logger.debug("FrogMl config file was not found.")
+        return None
+    except json.JSONDecodeError as e:
+        print(f"FrogMl config file is not a valid JSON {e}.")
+        return None
+
+
+def parse_cli_config_server(server_config: dict) -> Union[LoginArguments, None]:
+    login_args = LoginArguments()
+    login_args.server_id = server_config.get(SERVER_ID)
+
+    if JFROG_CLI_CONFIG_ARTIFACTORY_URL in server_config:
+        login_args.artifactory_url = server_config.get(JFROG_CLI_CONFIG_ARTIFACTORY_URL)
+    elif (
+        JFROG_CLI_CONFIG_URL in server_config
+        and server_config.get(JFROG_CLI_CONFIG_URL) is not None
+    ):
+        login_args.artifactory_url = join_url(
+            server_config[JFROG_CLI_CONFIG_URL], "artifactory"
+        )
+    else:
+        logger.debug(
+            "Invalid JFrog CLI file, expected either artifactoryUrl or url in jfrog cli config file"
+        )
+        return None
+
+    if JFROG_CLI_CONFIG_ACCESS_TOKEN in server_config:
+        login_args.access_token = server_config.get(JFROG_CLI_CONFIG_ACCESS_TOKEN)
+    elif (
+        JFROG_CLI_CONFIG_USER in server_config
+        and JFROG_CLI_CONFIG_PASSWORD in server_config
+    ):
+        login_args.username = server_config.get(JFROG_CLI_CONFIG_USER)
+        login_args.password = server_config.get(JFROG_CLI_CONFIG_PASSWORD)
+    else:
+        logger.debug(
+            "Expected either accessToken or user/password in jfrog cli config file"
+        )
+        return None
+
+    return login_args
+
+
+def get_frogml_configuration() -> Union[LoginArguments, None]:
+    frog_ml_config = read_frogml_config()
+    if (
+        frog_ml_config is not None
+        and frog_ml_config.get("servers") is not None
+        and len(frog_ml_config["servers"]) > 0
+    ):
+        server_config = frog_ml_config["servers"][0]
+        login_args = LoginArguments()
+        if FROG_ML_CONFIG_ARTIFACTORY_URL in server_config:
+            login_args.artifactory_url = server_config.get(
+                FROG_ML_CONFIG_ARTIFACTORY_URL
+            )
+        else:
+            logger.debug(
+                "Invalid FrogMl authentication file, expected either artifactory_url in FrogMl authentication file"
+            )
+            return None
+
+        if FROG_ML_CONFIG_ACCESS_TOKEN in server_config:
+            login_args.access_token = server_config.get(FROG_ML_CONFIG_ACCESS_TOKEN)
+        elif (
+            FROG_ML_CONFIG_USER in server_config
+            and FROG_ML_CONFIG_PASSWORD in server_config
+        ):
+            login_args.username = server_config.get(FROG_ML_CONFIG_USER)
+            login_args.password = server_config.get(FROG_ML_CONFIG_PASSWORD)
+        elif (
+            FROG_ML_CONFIG_USER in server_config
+            and FROG_ML_CONFIG_PASSWORD not in server_config
+            or (
+                FROG_ML_CONFIG_USER not in server_config
+                and FROG_ML_CONFIG_PASSWORD in server_config
+            )
+        ):
+            logger.debug(
+                "Invalid FrogMl authentication file, username or password is missing in FrogMl authentication file"
+            )
+            return None
+        elif (
+            login_args.username is None
+            and login_args.password is None
+            and login_args.access_token is None
+        ):
+            login_args.isAnonymous = True
+        return login_args
+    else:
+        return None
+
+
+def get_encrypted_password(
+    auth_config: AuthConfig, auth_token: AuthBase
+) -> Optional[str]:
+    try:
+        response = ArtifactoryApi(
+            auth_config.artifactory_url, auth_token
+        ).encrypt_password()
+        if response.status_code != http.HTTPStatus.OK:
+            logger.debug(
+                f"Expected {http.HTTPStatus.OK} status but got {response.status_code} "
+                f"when using url {auth_config.artifactory_url}"
+            )
+            print("Error while trying to encrypt password.")
+            return None
+        return response.text
+    except requests.exceptions.RequestException as e:
+        print(f"Error while trying to encrypt password: {e}.")
+        return None
+
+
+def save_auth_config(auth_config: AuthConfig) -> None:
+    file_content: dict[str, list] = {}
+    file_content.setdefault("servers", []).append(auth_config.to_json())
+    os.makedirs(os.path.dirname(CONFIG_FILE_PATH), exist_ok=True)
+    with open(CONFIG_FILE_PATH, "w") as file:
+        json.dump(file_content, file, indent=2)
+
+
+def get_credentials(auth_config: Optional[AuthConfig] = None) -> Tuple[str, AuthBase]:
+    if not __should_use_file_auth(auth_config):
+        __validate_credentials(auth_config)
+        return __auth_config_to_auth_tuple(auth_config)
+    logger.debug(
+        "Login configuration not supplied, attempting to find environment variables"
+    )
+
+    if __should_use_environment_variables():
+        return get_environment_variables()
+
+    logger.debug(
+        "Environment variables not supplied, attempting to load configuration from file"
+    )
+
+    if os.path.exists(CONFIG_FILE_PATH):
+        return __read_credentials_from_file(CONFIG_FILE_PATH)
+    raise ValueError(
+        f"Configuration were not provided and configuration file not found in {CONFIG_FILE_PATH},"
+        f" either pass configuration in the constructor, add env variables or create the configuration file by "
+        f"running `frogml login`"
+    )
+
+
+def __should_use_environment_variables() -> bool:
+    return os.getenv("JF_URL") is not None
+
+
+def get_environment_variables() -> Tuple[str, AuthBase]:
+    auth_config: AuthConfig = AuthConfig(
+        artifactory_url=os.getenv(JF_URL),
+        access_token=os.getenv(JF_ACCESS_TOKEN),
+    )
+
+    return __auth_config_to_auth_tuple(auth_config)
+
+
+def __should_use_file_auth(credentials: Optional[AuthConfig] = None) -> bool:
+    return credentials is None or (
+        credentials.artifactory_url is None
+        and credentials.user is None
+        and credentials.password is None
+        and credentials.access_token is None
+    )
+
+
+def __validate_credentials(credentials: Optional[AuthConfig]) -> None:
+    if credentials is None:
+        raise ValueError("Credentials must be provided.")
+    if credentials.artifactory_url is None:
+        raise ValueError("Credentials must contain artifactory url.")
+    return None
+
+
+def __read_credentials_from_file(file_path: str) -> Tuple[str, AuthBase]:
+    try:
+        with open(file_path, "r") as file:
+            config_content: dict = json.load(file)
+            servers = config_content.get("servers")
+            if servers is None or len(servers) == 0:
+                raise ValueError(
+                    "Configuration file was found but it's empty, failing authentication"
+                )
+            server = servers[0]
+            return __auth_config_to_auth_tuple(AuthConfig.from_dict(server))
+    except json.JSONDecodeError:
+        raise ValueError(f"Error when reading {file_path}, please recreate the file.")
+
+
+def __auth_config_to_auth_tuple(
+    auth_config: Optional[AuthConfig],
+) -> Tuple[str, AuthBase]:
+
+    if auth_config is None:
+        raise ValueError("No authentication configuration provided")
+
+    auth: AuthBase = EmptyAuth()
+    if auth_config.access_token is not None:
+        auth = BearerAuth(auth_config.access_token)
+    elif auth_config.user is not None and auth_config.password is not None:
+        auth = HTTPBasicAuth(auth_config.user, auth_config.password)
+    elif auth_config.user is not None or auth_config.password is not None:
+        raise ValueError("User and password must be provided together")
+
+    if auth_config.artifactory_url is None:
+        raise ValueError("No artifactory URL provided")
+
+    return auth_config.artifactory_url, auth
+
+
+def get_list_of_servers_from_config(jfrog_cli_config: dict) -> List[str]:
+    if jfrog_cli_config is not None:
+        servers = jfrog_cli_config.get("servers")
+        if servers is not None:
+            return list(map(__map_server_ids, servers))
+
+    return []
+
+
+def __map_server_ids(server: dict) -> str:
+    server_id = ""
+    if server is not None:
+        server_id = str(server.get("serverId"))
+        if server.get("isDefault") is not None and bool(server.get("isDefault")):
+            server_id = server_id + " (Default)"
+    return server_id

frogml_storage/authentication/utils/_login_checks_utils.py

@@ -0,0 +1,114 @@
+from typing import Optional
+
+from frogml_storage.authentication.models import AuthConfig
+
+
+def login_input_checks(
+    url: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    token: Optional[str] = None,
+    anonymous: bool = False,
+) -> bool:
+    return (
+        __is_user_name_password_command(url, username, password, token, anonymous)
+        or __is_access_token_command(url, username, password, token, anonymous)
+        or __is_anonymous_command(url, username, password, token, anonymous)
+    )
+
+
+def is_login_without_params(
+    url: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    token: Optional[str] = None,
+    anonymous: bool = False,
+) -> bool:
+    return (
+        all(params is None for params in [url, username, password, token])
+        and not anonymous
+    )
+
+
+def is_username_password_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    return __is_user_name_password_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def is_access_token_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    if anonymous is None:
+        anonymous = False
+    return __is_access_token_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def is_anonymous_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    if anonymous is None:
+        anonymous = False
+    return __is_anonymous_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def __is_anonymous_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: bool,
+) -> bool:
+    return (
+        anonymous
+        and all(params is None for params in [username, password, token])
+        and url is not None
+    )
+
+
+def __is_access_token_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: bool,
+) -> bool:
+    if anonymous:
+        return False
+    return all(params is None for params in [username, password]) and all(
+        params is not None for params in [url, token]
+    )
+
+
+def __is_user_name_password_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: Optional[bool] = False,
+) -> bool:
+    return (
+        not anonymous
+        and url is not None
+        and username is not None
+        and password is not None
+        and token is None
+    )

frogml_storage/base_storage.py

@@ -0,0 +1,140 @@
+from abc import ABC, abstractmethod
+from typing import Dict, List, Optional, Union
+
+from frogml_storage.models.frogml_dataset_version import FrogMLDatasetVersion
+from frogml_storage.models.frogml_model_version import FrogMLModelVersion
+from frogml_storage.models.serialization_metadata import SerializationMetadata
+
+
+class BaseStorage(ABC):
+    """
+    Repository storage to download or store model | dataset artifacts,
+    metrics and relation between these in an Artifactory repository.
+    """
+
+    @abstractmethod
+    def upload_model_version(
+        self,
+        repository: str,
+        model_name: str,
+        model_path: str,
+        model_type: Union[SerializationMetadata, Dict],
+        namespace: Optional[str] = None,
+        version: Optional[str] = None,
+        properties: Optional[dict[str, str]] = None,
+        dependencies_files_paths: Optional[List[str]] = None,
+        code_archive_file_path: Optional[str] = None,
+    ) -> FrogMLModelVersion:
+        """Upload model to a repository in Artifactory. Uploaded models should be stored with the following layout:
+        ├── REPO
+            ├── models
+                ├── ${NAMESPACE}
+                    ├── ${MODEL_NAME}
+                        ├── ${MODEL_VERSION}
+                            ├── model-manifest.json
+                            ├── model
+                                ├── model.pkl
+                                ├── evidence.json
+                                ├── ...
+                            ├── code
+                                ├── code.zip
+                                ├── requirements.txt
+        :param repository: the repository to upload the model to
+        :param model_name: the name of the model
+        :param model_path: the source path of the model
+        :param model_type: the type of the model (PyTorch, HuggingFace, Catboost)
+        :param namespace: the namespace to upload the model to
+        :param version: the version of the model
+        :param properties: tags to associate with the model
+        :param dependencies_files_paths: the list of dependencies files paths
+        :param code_archive_file_path: the path to the code archive file
+        """
+        pass
+
+    @abstractmethod
+    def download_model_version(
+        self,
+        repository: str,
+        model_name: str,
+        version: str,
+        target_path: str,
+        namespace: Optional[str] = None,
+    ) -> None:
+        """Downloads a model from an Artifactory repository
+        :param repository: the repository to download the model from
+        :param model_name: the name of the model to download
+        :param version: the version of the model to download
+        :param target_path: the target local path to store the model in
+        :param namespace: the namespace of the model to download
+
+        """
+        pass
+
+    def download_model_single_artifact(
+        self,
+        repository: str,
+        model_name: str,
+        version: str,
+        target_path: str,
+        source_artifact_path: str,
+        namespace: Optional[str] = None,
+    ) -> None:
+        """Downloads an artifact from an Artifactory repository
+        :param repository: the repository to download the artifact from
+        :param model_name: the model to download artifact from
+        :param version: the model's version to download artifact from
+        :param target_path: the target local path to store the artifact in
+        :param source_artifact_path: the path of the artifact under the model's version directory
+        :param namespace: the namespace of the artifact
+        """
+        pass
+
+    @abstractmethod
+    def upload_dataset_version(
+        self,
+        repository: str,
+        dataset_name: str,
+        source_path: str,
+        namespace: Optional[str] = None,
+        version: Optional[str] = None,
+        tags: Optional[dict[str, str]] = None,
+    ) -> FrogMLDatasetVersion:
+        """Uploads a dataset to a repository in Artifactory. Uploaded datasets should be stored with
+         the following layout:
+        ├── REPO
+            ├── datasets
+                ├── ${NAMESPACE}
+                    ├── ${DATASET_NAME}
+                        ├── ${DATASET_VERSION}
+                            ├── dataset-manifest.json
+                            ├── dataset
+                                ├── cli_docs_1.csv
+                                ├── cli_docs_2.csv
+                                ├── ...
+        :param repository: the repository to upload the dataset to
+        :param dataset_name: the name of the dataset
+        :param source_path: the source path of the dataset
+        :param namespace: the namespace to upload the dataset to
+        :param version: the version of the dataset
+        :param tags: tags to associate with the dataset
+        """
+        pass
+
+    @abstractmethod
+    def download_dataset_version(
+        self,
+        repository: str,
+        dataset_name: str,
+        version: str,
+        target_path: str,
+        namespace: Optional[str] = None,
+    ) -> None:
+        """Downloads a dataset from an Artifactory repository
+        :param repository: the repository to download the dataset from
+        :param dataset_name: the name of the dataset to download
+        :param version: the version of the dataset to download
+        :param target_path: the target local path to store the dataset in
+        :param namespace: the namespace of the dataset to download
+
+        """
+        pass

frogml_storage/constants.py

@@ -0,0 +1,56 @@
+import os
+
+home_dir = os.path.expanduser("~")
+
+JFROG_CLI_CONFIG_FILE_PATH = os.path.join(home_dir, ".jfrog", "jfrog-cli.conf.v6")
+CONFIG_FILE_PATH = os.path.join(home_dir, ".frogml", "config.json")
+
+FROG_ML_CONFIG_USER = "user"
+FROG_ML_CONFIG_ARTIFACTORY_URL = "artifactory_url"
+FROG_ML_CONFIG_PASSWORD = "password"  # nosec B105
+FROG_ML_CONFIG_ACCESS_TOKEN = "access_token"  # nosec B105
+FROG_ML_DEFAULT_HTTP_THREADS_COUNT = 5
+FROG_ML_MAX_CHARS_FOR_NAME = 60
+
+SERVER_ID = "server_id"
+
+JFROG_CLI_CONFIG_ARTIFACTORY_URL = "artifactoryUrl"
+JFROG_CLI_CONFIG_URL = "url"
+JFROG_CLI_CONFIG_USER = "user"
+JFROG_CLI_CONFIG_PASSWORD = "password"  # nosec B105
+JFROG_CLI_CONFIG_ACCESS_TOKEN = "accessToken"  # nosec B105
+
+MODEL = "model"
+ROOT_FROGML_MODEL_UI_DIRECTORY = "models"
+MODEL_UI_DIRECTORY = "model"
+MODEL_METADATA_FILE_NAME = "model-manifest.json"
+BODY_PART_MODEL_MANIFEST_STREAM = "modelManifest"
+
+DATASET = "dataset"
+ROOT_FROGML_DATASET_UI_DIRECTORY = "datasets"
+DATASET_UI_DIRECTORY = "dataset"
+DATASET_METADATA_FILE_NAME = "dataset-manifest.json"
+BODY_PART_DATASET_MANIFEST_STREAM = "datasetManifest"
+
+CHECKSUM_SHA2_HEADER = "X-Checksum-Sha256"
+
+JFML_THREAD_COUNT = "JFML_THREAD_COUNT"
+JF_URL = "JF_URL"
+JF_ACCESS_TOKEN = "JF_ACCESS_TOKEN"  # nosec B105
+
+FROG_ML_IGNORED_FILES = [
+    ".DS_Store",
+    "CVS",
+    ".cvsignore",
+    "SCCS",
+    "vssver.scc",
+    ".svn",
+    ".git",
+    ".gitignore",
+    ".gitattributes",
+    ".gitmodules",
+    ".gitkeep",
+    ".gitconfig",
+    MODEL_METADATA_FILE_NAME,
+    DATASET_METADATA_FILE_NAME,
+]