qwak-core 0.4.266__py3-none-any.whl → 0.4.267__py3-none-any.whl

frogml_storage/http/http_client.py

@@ -1,83 +0,0 @@
-import os
-from typing import Optional, Tuple
-
-import requests
-from requests.adapters import HTTPAdapter
-from urllib3 import Retry
-
-import qwak
-from frogml_storage._log_config import logger
-
-
-class HTTPClient:
-
-    def __init__(
-        self, auth: Tuple[str, str], session: Optional[requests.Session] = None
-    ):
-        self.auth = auth
-        # add default headers
-        if session is None:
-            self.session = self._create_session()
-        self._add_default_headers()
-        self.timeout = os.getenv("JFML_TIMEOUT", default=30)
-
-    @staticmethod
-    def _create_session():
-        session = requests.Session()
-        adapter = HTTPAdapter(
-            max_retries=RetryWithLog(
-                total=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]
-            )
-        )
-        session.mount("http://", adapter)
-        session.mount("https://", adapter)
-        return session
-
-    def post(self, url, data=None, params=None):
-        return self.session.post(
-            url, auth=self.auth, timeout=self.timeout, data=data, params=params
-        )
-
-    def get(self, url, params=None, stream=False):
-        return self.session.get(url, auth=self.auth, params=params, stream=stream)
-
-    def put(self, url, payload=None, files=None, stream=False, headers=None, json=None):
-        return self.session.request(
-            method="PUT",
-            url=url,
-            data=payload,
-            auth=self.auth,
-            files=files,
-            stream=stream,
-            timeout=self.timeout,
-            headers=headers,
-            json=json,
-        )
-
-    def delete(self, url):
-        return self.session.request(
-            method="DELETE",
-            url=url,
-            auth=self.auth,
-            timeout=self.timeout,
-        )
-
-    def head(self, url, params=None, stream=False):
-        return self.session.head(url, auth=self.auth, params=params, stream=stream)
-
-    def _add_default_headers(self):
-        self.session.headers.update(
-            {"User-Agent": "frogml-sdk-python/{}".format(qwak.__version__)}
-        )
-
-
-class RetryWithLog(Retry):
-    """
-    Adding extra logs before making a retry request
-    """
-
-    def __init__(self, *args, **kwargs):
-        history = kwargs.get("history")
-        if history is not None:
-            logger.debug(f"Error: ${history[-1].error}\nretrying...")
-        super().__init__(*args, **kwargs)

frogml_storage/model_manifest.py

@@ -1,60 +0,0 @@
-import os
-from typing import List, Optional
-
-from pydantic import ConfigDict, Field
-
-from frogml_storage.entity_manifest import Artifact, Checksums, EntityManifest
-from frogml_storage.serialization_metadata import SerializationMetadata
-
-
-class ModelManifest(EntityManifest):
-    """
-    Represent a model manifest file
-
-    Attributes:
-        model_format: If the entity is model, holds model format information
-        dependency_artifacts: If the entity is model, holds a list of files specifying the model dependencies
-        code_artifacts: If the entity is model, specifies the archive file artifact
-    """
-
-    artifacts: List[Artifact] = Field(serialization_alias="model_artifacts")
-
-    model_format: SerializationMetadata
-    dependency_artifacts: Optional[List[Artifact]] = None
-    code_artifacts: Optional[Artifact] = None
-
-    # suppress warning on model_format field name.
-    # if one day it collides with pydantic field, it will throw an error.
-    model_config = ConfigDict(protected_namespaces=())
-
-    def add_dependency_file(
-        self, file_path: str, checksums: Checksums, rel_path: str
-    ) -> None:
-        if self.dependency_artifacts is None:
-            self.dependency_artifacts = []
-        self.dependency_artifacts.append(
-            Artifact(
-                artifact_path=rel_path,
-                size=os.path.getsize(file_path),
-                checksums=checksums,
-            )
-        )
-
-    def __eq__(self, other):
-        if not super.__eq__(self, other):
-            return False
-        if self.model_format != other.model_format:
-            return False
-        if self.dependency_artifacts != other.dependency_artifacts:
-            return False
-        if self.dependency_artifacts is not None:
-            if len(self.dependency_artifacts) != len(other.dependency_artifacts):
-                return False
-            for self_artifact, other_artifact in zip(
-                self.dependency_artifacts, other.dependency_artifacts
-            ):
-                if self_artifact != other_artifact:
-                    return False
-        if self.code_artifacts != other.code_artifacts:
-            return False
-        return True

frogml_storage/models/_download_context.py

@@ -1,54 +0,0 @@
-from typing import Optional
-
-from frogml_storage.entity_manifest import Checksums
-
-
-class DownloadContext(object):
-    """
-    A class to represent the arguments for a download operation.
-
-    Attributes
-    ----------
-    repo_key : str
-        The key of the repository where the artifact is located.
-    source_url : str
-        The source relative URL of the artifact, relative to artifactory url and the repo key.
-    target_path : str
-        The target path where the artifact will be downloaded to.
-    exists_locally : bool
-        A flag indicating whether the artifact already exists locally in the target path.
-    artifact_checksum: Checksums
-        The checksum of the artifact.
-    """
-
-    repo_key: str
-    source_url: str
-    target_path: str
-    exists_locally: bool = False
-    artifact_checksum: Optional[Checksums]
-
-    def __init__(
-        self,
-        repo_key: str,
-        source_url: str,
-        target_path: str,
-        exists_locally: bool = False,
-        artifact_checksum: Optional[Checksums] = None,
-    ):
-        self.repo_key = repo_key
-        self.source_url = source_url
-        self.target_path = target_path
-        self.exists_locally = exists_locally
-        self.artifact_checksum = artifact_checksum
-
-    def __eq__(self, other):
-        if not isinstance(other, DownloadContext):
-            return False
-
-        return (
-            self.repo_key == other.repo_key
-            and self.source_url == other.source_url
-            and self.target_path == other.target_path
-            and self.exists_locally == other.exists_locally
-            and self.artifact_checksum == other.artifact_checksum
-        )

frogml_storage/models/frogml_dataset_version.py

@@ -1,21 +0,0 @@
-from frogml_storage.models.frogml_entity_version import FrogMLEntityVersion
-
-
-class FrogMLDatasetVersion(FrogMLEntityVersion):
-    """
-    Represent metadata of an uploaded dataset version.
-
-    Inherits:
-        FrogMLEntityVersion: Base class for entity versions.
-    """
-
-    @classmethod
-    def from_entity_version(
-        cls, entity_version: FrogMLEntityVersion
-    ) -> "FrogMLDatasetVersion":
-        return cls(
-            entity_name=entity_version.entity_name,
-            version=entity_version.version,
-            namespace=entity_version.namespace,
-            entity_manifest=entity_version.entity_manifest,
-        )

frogml_storage/models/frogml_entity_version.py

@@ -1,34 +0,0 @@
-from typing import Optional
-
-from pydantic import BaseModel
-
-from frogml_storage.entity_manifest import EntityManifest
-
-
-class FrogMLEntityVersion(BaseModel):
-    """
-    Represent a metadata of uploaded entity
-
-    Attributes:
-        entity_name: The entity name (model | dataset)
-        namespace: The namespace of the model | dataset
-        version: The version of the model | dataset
-        entity_manifest: The entity manifest file
-    """
-
-    entity_name: str
-    namespace: Optional[str] = None
-    version: str
-    entity_manifest: Optional[EntityManifest] = None
-
-    def __eq__(self, other):
-        if not isinstance(other, FrogMLEntityVersion):
-            return False
-        if self.version != other.version:
-            return False
-        if self.namespace != other.namespace:
-            return False
-        if self.entity_name != other.entity_name:
-            return False
-
-        return True

frogml_storage/models/frogml_model_version.py

@@ -1,21 +0,0 @@
-from frogml_storage.models.frogml_entity_version import FrogMLEntityVersion
-
-
-class FrogMLModelVersion(FrogMLEntityVersion):
-    """
-    Represent metadata of an uploaded model version.
-
-    Inherits:
-        FrogMLEntityVersion: Base class for entity versions.
-    """
-
-    @classmethod
-    def from_entity_version(
-        cls, entity_version: FrogMLEntityVersion
-    ) -> "FrogMLModelVersion":
-        return cls(
-            entity_name=entity_version.entity_name,
-            version=entity_version.version,
-            namespace=entity_version.namespace,
-            entity_manifest=entity_version.entity_manifest,
-        )

frogml_storage/serialization_metadata.py

@@ -1,15 +0,0 @@
-from typing import Dict
-
-from pydantic import BaseModel
-
-
-class SerializationMetadata(BaseModel):
-    framework: str
-    framework_version: str
-    serialization_format: str
-    runtime: str
-    runtime_version: str
-
-    @classmethod
-    def from_json(cls, json_dict: Dict) -> "SerializationMetadata":
-        return cls.model_validate(json_dict)

frogml_storage/storage.py

@@ -1,140 +0,0 @@
-from abc import ABC, abstractmethod
-from typing import Dict, List, Optional, Union
-
-from frogml_storage.models.frogml_dataset_version import FrogMLDatasetVersion
-from frogml_storage.models.frogml_model_version import FrogMLModelVersion
-from frogml_storage.serialization_metadata import SerializationMetadata
-
-
-class Storage(ABC):
-    """
-    Repository storage to download or store model | dataset artifacts,
-    metrics and relation between these in an Artifactory repository.
-    """
-
-    @abstractmethod
-    def upload_model_version(
-        self,
-        repository: str,
-        model_name: str,
-        model_path: str,
-        model_type: Union[SerializationMetadata, Dict],
-        namespace: Optional[str] = None,
-        version: Optional[str] = None,
-        properties: Optional[dict[str, str]] = None,
-        dependencies_files_paths: Optional[List[str]] = None,
-        code_archive_file_path: Optional[str] = None,
-    ) -> FrogMLModelVersion:
-        """Upload model to a repository in Artifactory. Uploaded models should be stored with the following layout:
-        ├── REPO
-            ├── models
-                ├── ${NAMESPACE}
-                    ├── ${MODEL_NAME}
-                        ├── ${MODEL_VERSION}
-                            ├── model-manifest.json
-                            ├── model
-                                ├── model.pkl
-                                ├── evidence.json
-                                ├── ...
-                            ├── code
-                                ├── code.zip
-                                ├── requirements.txt
-        :param repository: the repository to upload the model to
-        :param model_name: the name of the model
-        :param model_path: the source path of the model
-        :param model_type: the type of the model (PyTorch, HuggingFace, Catboost)
-        :param namespace: the namespace to upload the model to
-        :param version: the version of the model
-        :param properties: tags to associate with the model
-        :param dependencies_files_paths: the list of dependencies files paths
-        :param code_archive_file_path: the path to the code archive file
-        """
-        pass
-
-    @abstractmethod
-    def download_model_version(
-        self,
-        repository: str,
-        model_name: str,
-        version: str,
-        target_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads a model from an Artifactory repository
-        :param repository: the repository to download the model from
-        :param model_name: the name of the model to download
-        :param version: the version of the model to download
-        :param target_path: the target local path to store the model in
-        :param namespace: the namespace of the model to download
-
-        """
-        pass
-
-    def download_model_single_artifact(
-        self,
-        repository: str,
-        model_name: str,
-        version: str,
-        target_path: str,
-        source_artifact_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads an artifact from an Artifactory repository
-        :param repository: the repository to download the artifact from
-        :param model_name: the model to download artifact from
-        :param version: the model's version to download artifact from
-        :param target_path: the target local path to store the artifact in
-        :param source_artifact_path: the path of the artifact under the model's version directory
-        :param namespace: the namespace of the artifact
-        """
-        pass
-
-    @abstractmethod
-    def upload_dataset_version(
-        self,
-        repository: str,
-        dataset_name: str,
-        source_path: str,
-        namespace: Optional[str] = None,
-        version: Optional[str] = None,
-        tags: Optional[dict[str, str]] = None,
-    ) -> FrogMLDatasetVersion:
-        """Uploads a dataset to a repository in Artifactory. Uploaded datasets should be stored with
-         the following layout:
-        ├── REPO
-            ├── datasets
-                ├── ${NAMESPACE}
-                    ├── ${DATASET_NAME}
-                        ├── ${DATASET_VERSION}
-                            ├── dataset-manifest.json
-                            ├── dataset
-                                ├── cli_docs_1.csv
-                                ├── cli_docs_2.csv
-                                ├── ...
-        :param repository: the repository to upload the dataset to
-        :param dataset_name: the name of the dataset
-        :param source_path: the source path of the dataset
-        :param namespace: the namespace to upload the dataset to
-        :param version: the version of the dataset
-        :param tags: tags to associate with the dataset
-        """
-        pass
-
-    @abstractmethod
-    def download_dataset_version(
-        self,
-        repository: str,
-        dataset_name: str,
-        version: str,
-        target_path: str,
-        namespace: Optional[str] = None,
-    ) -> None:
-        """Downloads a dataset from an Artifactory repository
-        :param repository: the repository to download the dataset from
-        :param dataset_name: the name of the dataset to download
-        :param version: the version of the dataset to download
-        :param target_path: the target local path to store the dataset in
-        :param namespace: the namespace of the dataset to download
-
-        """
-        pass

frogml_storage/utils/_input_checks_utility.py

@@ -1,104 +0,0 @@
-import os
-import re
-from typing import List, Optional, Union
-
-from frogml_storage._log_config import logger
-from frogml_storage.constants import FROG_ML_MAX_CHARS_FOR_NAME
-from frogml_storage.exceptions.validation_error import FrogMLValidationError
-
-valid_characters_pattern = re.compile(r"^[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$")
-
-
-def user_input_validation(
-    entity_name: Optional[str],
-    namespace: Optional[str],
-    version: Optional[str],
-    properties: Optional[dict[str, str]] = None,
-) -> bool:
-    for arg_name, arg_value in {
-        "entity_name": entity_name,
-        "namespace": namespace,
-        "version": version,
-    }.items():
-        if arg_value is not None:
-            __input_validation(arg_value, arg_name)
-
-    if properties is not None:
-        __user_input_dict_validation(properties)
-
-    return True
-
-
-def __user_input_dict_validation(properties: dict[str, str]) -> bool:
-    if properties is not None:
-        for key, value in properties.items():
-            __input_validation(key, "properties")
-            __input_validation(value, "properties")
-    return True
-
-
-def is_not_none(arg_name: str, arg_value: Optional[object]) -> bool:
-    if arg_value is None:
-        raise FrogMLValidationError("{} can't be 'None'.".format(arg_name))
-    return True
-
-
-def __input_validation(field_value: str, field_name: str) -> bool:
-    if len(str(field_value)) > FROG_ML_MAX_CHARS_FOR_NAME:
-        raise FrogMLValidationError(
-            "Max length for {} is 60 characters.".format(field_name.capitalize())
-        )
-
-    if not field_value or not re.match(valid_characters_pattern, str(field_value)):
-        raise FrogMLValidationError(
-            "Invalid characters detected at {}: {}".format(
-                field_name.capitalize(), field_value
-            )
-        )
-
-    return True
-
-
-def is_valid_thread_number(thread_count: str) -> bool:
-    try:
-        int_thread_count = int(thread_count)
-        cpu_count = os.cpu_count()
-        if int_thread_count <= 0 or (
-            cpu_count is not None and int_thread_count >= cpu_count
-        ):
-            raise ValueError(
-                "Invalid thread count: {}. The default value will be used.".format(
-                    thread_count
-                )
-            )
-        return True
-    except ValueError as e:
-        logger.warning("Thread count {}: {}".format(thread_count, e))
-        return False
-    except TypeError:
-        logger.debug("Thread count not configured. The default value will be used.")
-        return False
-
-
-def validate_not_folder_paths(paths: Union[Optional[List[str]], Optional[str]]) -> bool:
-    if paths is not None:
-        if isinstance(paths, List):
-            for path in paths:
-                __validate_not_folder_path(path)
-        else:
-            __validate_not_folder_path(paths)
-    return True
-
-
-def __validate_not_folder_path(path: str) -> bool:
-    if os.path.isdir(path):
-        raise FrogMLValidationError(
-            "file '{}' must be a file, but is a directory.".format(path)
-        )
-    return True
-
-
-def validate_path_exists(path: str) -> bool:
-    if not os.path.exists(path):
-        raise ValueError(f"Provided path does not exists : '{path}'")
-    return True