qwak-core 0.4.272__py3-none-any.whl → 0.4.273__py3-none-any.whl

frogml_storage/http/http_client.py

@@ -0,0 +1,83 @@
+import os
+from typing import Optional, Tuple
+
+import requests
+from requests.adapters import HTTPAdapter
+from urllib3 import Retry
+
+import frogml_storage
+from frogml_storage.logging import logger
+
+
+class HTTPClient:
+
+    def __init__(
+        self, auth: Tuple[str, str], session: Optional[requests.Session] = None
+    ):
+        self.auth = auth
+        # add default headers
+        if session is None:
+            self.session = self._create_session()
+        self._add_default_headers()
+        self.timeout = os.getenv("JFML_TIMEOUT", default=30)
+
+    @staticmethod
+    def _create_session():
+        session = requests.Session()
+        adapter = HTTPAdapter(
+            max_retries=RetryWithLog(
+                total=5, backoff_factor=0.5, status_forcelist=[500, 502, 503, 504]
+            )
+        )
+        session.mount("http://", adapter)
+        session.mount("https://", adapter)
+        return session
+
+    def post(self, url, data=None, params=None):
+        return self.session.post(
+            url, auth=self.auth, timeout=self.timeout, data=data, params=params
+        )
+
+    def get(self, url, params=None, stream=False):
+        return self.session.get(url, auth=self.auth, params=params, stream=stream)
+
+    def put(self, url, payload=None, files=None, stream=False, headers=None, json=None):
+        return self.session.request(
+            method="PUT",
+            url=url,
+            data=payload,
+            auth=self.auth,
+            files=files,
+            stream=stream,
+            timeout=self.timeout,
+            headers=headers,
+            json=json,
+        )
+
+    def delete(self, url):
+        return self.session.request(
+            method="DELETE",
+            url=url,
+            auth=self.auth,
+            timeout=self.timeout,
+        )
+
+    def head(self, url, params=None, stream=False):
+        return self.session.head(url, auth=self.auth, params=params, stream=stream)
+
+    def _add_default_headers(self):
+        self.session.headers.update(
+            {"User-Agent": "frogml-sdk-python/{}".format(frogml_storage.__version__)}
+        )
+
+
+class RetryWithLog(Retry):
+    """
+    Adding extra logs before making a retry request
+    """
+
+    def __init__(self, *args, **kwargs):
+        history = kwargs.get("history")
+        if history is not None:
+            logger.debug(f"Error: ${history[-1].error}\nretrying...")
+        super().__init__(*args, **kwargs)

frogml_storage/logging/__init__.py

@@ -0,0 +1 @@
+from ._log_config import logger

frogml_storage/logging/_log_config.py

@@ -0,0 +1,45 @@
+import logging.config
+import os
+import sys
+
+log_level = (
+    "DEBUG"
+    if os.getenv("JFML_DEBUG", "false").casefold() == "true".casefold()
+    else "INFO"
+)
+log_file = f'{os.path.expanduser("~")}/.frogml/frogml-log-history.log'
+os.makedirs(os.path.dirname(log_file), exist_ok=True)
+
+DEFAULT_LOGGING = {
+    "version": 1,
+    "formatters": {
+        "standard": {
+            "format": "%(asctime)s - %(levelname)s - %(name)s.%(module)s.%(funcName)s:%(lineno)d - %(message)s"
+        },
+    },
+    "handlers": {
+        "console": {
+            "class": "logging.StreamHandler",
+            "formatter": "standard",
+            "stream": sys.stdout,
+        },
+        "file": {
+            "class": "logging.FileHandler",
+            "formatter": "standard",
+            "filename": log_file,
+        },
+    },
+    "loggers": {
+        __name__: {
+            "level": log_level,
+            "handlers": ["console", "file"],
+            "propagate": False,
+        },
+    },
+}
+
+if os.getenv("IS_LOGGER_SHADED") is not None:
+    logger = logging.getLogger(__name__)
+else:
+    logging.config.dictConfig(DEFAULT_LOGGING)
+    logger = logging.getLogger(__name__)

frogml_storage/logging/log_utils.py

@@ -0,0 +1,21 @@
+from frogml_storage.models.frogml_entity_type_info import FrogMLEntityTypeInfo
+
+
+# The following method affect e2e tests.
+def build_download_success_log(
+    entity_type_info: FrogMLEntityTypeInfo, entity_name: str, version: str
+) -> str:
+    return (
+        f'{entity_type_info.entity_type.capitalize()}: "{entity_name}", version: "{version}"'
+        f" has been downloaded successfully"
+    )
+
+
+# The following method affect e2e tests.
+def build_upload_success_log(
+    entity_type_info: FrogMLEntityTypeInfo, entity_name: str, version: str
+) -> str:
+    return (
+        f'{entity_type_info.entity_type.capitalize()}: "{entity_name}", version: "{version}"'
+        f" has been uploaded successfully"
+    )

frogml_storage/models/__init__.py

@@ -0,0 +1 @@
+from ._download_context import DownloadContext

frogml_storage/models/_download_context.py

@@ -0,0 +1,54 @@
+from typing import Optional
+
+from frogml_storage.models.entity_manifest import Checksums
+
+
+class DownloadContext(object):
+    """
+    A class to represent the arguments for a download operation.
+
+    Attributes
+    ----------
+    repo_key : str
+        The key of the repository where the artifact is located.
+    source_url : str
+        The source relative URL of the artifact, relative to artifactory url and the repo key.
+    target_path : str
+        The target path where the artifact will be downloaded to.
+    exists_locally : bool
+        A flag indicating whether the artifact already exists locally in the target path.
+    artifact_checksum: Checksums
+        The checksum of the artifact.
+    """
+
+    repo_key: str
+    source_url: str
+    target_path: str
+    exists_locally: bool = False
+    artifact_checksum: Optional[Checksums]
+
+    def __init__(
+        self,
+        repo_key: str,
+        source_url: str,
+        target_path: str,
+        exists_locally: bool = False,
+        artifact_checksum: Optional[Checksums] = None,
+    ):
+        self.repo_key = repo_key
+        self.source_url = source_url
+        self.target_path = target_path
+        self.exists_locally = exists_locally
+        self.artifact_checksum = artifact_checksum
+
+    def __eq__(self, other):
+        if not isinstance(other, DownloadContext):
+            return False
+
+        return (
+            self.repo_key == other.repo_key
+            and self.source_url == other.source_url
+            and self.target_path == other.target_path
+            and self.exists_locally == other.exists_locally
+            and self.artifact_checksum == other.artifact_checksum
+        )

frogml_storage/models/dataset_manifest.py

@@ -0,0 +1,13 @@
+from typing import List
+
+from pydantic import Field
+
+from frogml_storage.models.entity_manifest import Artifact, EntityManifest
+
+
+class DatasetManifest(EntityManifest):
+    """
+    Represent a dataset manifest file
+    """
+
+    artifacts: List[Artifact] = Field(serialization_alias="dataset_artifacts")

frogml_storage/models/entity_manifest.py

@@ -0,0 +1,93 @@
+import os
+from abc import ABC
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from frogml_storage.utils import calc_content_sha2, calculate_sha2
+
+
+class Checksums(BaseModel):
+    sha2: str
+
+    @classmethod
+    def calc_checksums(cls, file_path: str) -> "Checksums":
+        return cls(sha2=calculate_sha2(file_path))
+
+    @classmethod
+    def calc_content_checksums(cls, content: str) -> "Checksums":
+        return cls(sha2=calc_content_sha2(content))
+
+
+class Artifact(BaseModel):
+    artifact_path: str
+    size: int
+    checksums: Checksums
+
+    def __eq__(self, other):
+        if not isinstance(other, Artifact):
+            return False
+        return (
+            self.artifact_path == other.artifact_path
+            and self.size == other.size
+            and self.checksums == other.checksums
+        )
+
+
+class EntityManifest(BaseModel, ABC):
+    """
+    Represent an entity manifest file
+
+    Attributes:
+        created_date: The date the model | dataset were uploaded to Artifactory
+        artifacts: A list of artifacts that belong to the model | dataset
+        id: <organization>/<entity_name> - exists only for downloaded EntityInfo
+        version: The entity version - exists only for downloaded EntityInfo
+    """
+
+    created_date: str
+    artifacts: List[Artifact]
+    id: Optional[str] = None
+    version: Optional[str] = None
+
+    def add_file(self, file_path: str, checksums: Checksums, rel_path: str) -> None:
+        self.artifacts.append(
+            Artifact(
+                artifact_path=rel_path,
+                size=os.path.getsize(file_path),
+                checksums=checksums,
+            )
+        )
+
+    def add_content_file(self, rel_path: str, content: str) -> None:
+        checksums = Checksums.calc_content_checksums(content)
+        self.artifacts.append(
+            Artifact(
+                artifact_path=rel_path,
+                size=len((content, "utf-8")),
+                checksums=checksums,
+            )
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "EntityManifest":
+        return cls.model_validate_json(json_str)
+
+    def to_json(self) -> str:
+        return self.model_dump_json(by_alias=True, exclude_none=True)
+
+    def __eq__(self, other):
+        if not isinstance(other, EntityManifest):
+            return False
+        if self.id != other.id:
+            return False
+        if self.version != other.version:
+            return False
+        if self.created_date != other.created_date:
+            return False
+        if len(self.artifacts) != len(other.artifacts):
+            return False
+        for self_artifact, other_artifact in zip(self.artifacts, other.artifacts):
+            if self_artifact != other_artifact:
+                return False
+        return True

frogml_storage/models/frogml_dataset_version.py

@@ -0,0 +1,21 @@
+from frogml_storage.models.frogml_entity_version import FrogMLEntityVersion
+
+
+class FrogMLDatasetVersion(FrogMLEntityVersion):
+    """
+    Represent metadata of an uploaded dataset version.
+
+    Inherits:
+        FrogMLEntityVersion: Base class for entity versions.
+    """
+
+    @classmethod
+    def from_entity_version(
+        cls, entity_version: FrogMLEntityVersion
+    ) -> "FrogMLDatasetVersion":
+        return cls(
+            entity_name=entity_version.entity_name,
+            version=entity_version.version,
+            namespace=entity_version.namespace,
+            entity_manifest=entity_version.entity_manifest,
+        )

frogml_storage/models/frogml_entity_type_info.py

@@ -0,0 +1,50 @@
+from enum import Enum
+
+from typing_extensions import Self
+
+from frogml_storage.constants import (
+    BODY_PART_DATASET_MANIFEST_STREAM,
+    BODY_PART_MODEL_MANIFEST_STREAM,
+    DATASET_METADATA_FILE_NAME,
+    DATASET_UI_DIRECTORY,
+    MODEL_METADATA_FILE_NAME,
+    MODEL_UI_DIRECTORY,
+    ROOT_FROGML_DATASET_UI_DIRECTORY,
+    ROOT_FROGML_MODEL_UI_DIRECTORY,
+)
+
+
+# noinspection PyEnum
+class FrogMLEntityTypeInfo(Enum):
+    MODEL = (
+        MODEL_UI_DIRECTORY,
+        ROOT_FROGML_MODEL_UI_DIRECTORY,
+        MODEL_METADATA_FILE_NAME,
+        BODY_PART_MODEL_MANIFEST_STREAM,
+    )
+    DATASET = (
+        DATASET_UI_DIRECTORY,
+        ROOT_FROGML_DATASET_UI_DIRECTORY,
+        DATASET_METADATA_FILE_NAME,
+        BODY_PART_DATASET_MANIFEST_STREAM,
+    )
+
+    def __init__(
+        self: Self,
+        entity_type: str,
+        folder_name: str,
+        metadata_file_name: str,
+        body_part_stream: str,
+    ):
+        self.entity_type: str = entity_type
+        self.folder_name: str = folder_name
+        self.metadata_file_name: str = metadata_file_name
+        self.body_part_stream: str = body_part_stream
+
+    @classmethod
+    def from_string(cls, entity_type_string: str) -> "FrogMLEntityTypeInfo":
+        for entity_type in cls:
+            if entity_type.entity_type.lower() == entity_type_string.lower():
+                return entity_type
+
+        raise ValueError(f"No enum constant found for entityType: {entity_type_string}")

frogml_storage/models/frogml_entity_version.py

@@ -0,0 +1,34 @@
+from typing import Optional
+
+from pydantic import BaseModel
+
+from frogml_storage.models.entity_manifest import EntityManifest
+
+
+class FrogMLEntityVersion(BaseModel):
+    """
+    Represent a metadata of uploaded entity
+
+    Attributes:
+        entity_name: The entity name (model | dataset)
+        namespace: The namespace of the model | dataset
+        version: The version of the model | dataset
+        entity_manifest: The entity manifest file
+    """
+
+    entity_name: str
+    namespace: Optional[str] = None
+    version: str
+    entity_manifest: Optional[EntityManifest] = None
+
+    def __eq__(self, other):
+        if not isinstance(other, FrogMLEntityVersion):
+            return False
+        if self.version != other.version:
+            return False
+        if self.namespace != other.namespace:
+            return False
+        if self.entity_name != other.entity_name:
+            return False
+
+        return True

frogml_storage/models/frogml_model_version.py

@@ -0,0 +1,21 @@
+from frogml_storage.models.frogml_entity_version import FrogMLEntityVersion
+
+
+class FrogMLModelVersion(FrogMLEntityVersion):
+    """
+    Represent metadata of an uploaded model version.
+
+    Inherits:
+        FrogMLEntityVersion: Base class for entity versions.
+    """
+
+    @classmethod
+    def from_entity_version(
+        cls, entity_version: FrogMLEntityVersion
+    ) -> "FrogMLModelVersion":
+        return cls(
+            entity_name=entity_version.entity_name,
+            version=entity_version.version,
+            namespace=entity_version.namespace,
+            entity_manifest=entity_version.entity_manifest,
+        )

frogml_storage/models/model_manifest.py

@@ -0,0 +1,60 @@
+import os
+from typing import List, Optional
+
+from pydantic import ConfigDict, Field
+
+from frogml_storage.models.entity_manifest import Artifact, Checksums, EntityManifest
+from frogml_storage.models.serialization_metadata import SerializationMetadata
+
+
+class ModelManifest(EntityManifest):
+    """
+    Represent a model manifest file
+
+    Attributes:
+        model_format: If the entity is model, holds model format information
+        dependency_artifacts: If the entity is model, holds a list of files specifying the model dependencies
+        code_artifacts: If the entity is model, specifies the archive file artifact
+    """
+
+    artifacts: List[Artifact] = Field(serialization_alias="model_artifacts")
+
+    model_format: SerializationMetadata
+    dependency_artifacts: Optional[List[Artifact]] = None
+    code_artifacts: Optional[Artifact] = None
+
+    # suppress warning on model_format field name.
+    # if one day it collides with pydantic field, it will throw an error.
+    model_config = ConfigDict(protected_namespaces=())
+
+    def add_dependency_file(
+        self, file_path: str, checksums: Checksums, rel_path: str
+    ) -> None:
+        if self.dependency_artifacts is None:
+            self.dependency_artifacts = []
+        self.dependency_artifacts.append(
+            Artifact(
+                artifact_path=rel_path,
+                size=os.path.getsize(file_path),
+                checksums=checksums,
+            )
+        )
+
+    def __eq__(self, other):
+        if not super.__eq__(self, other):
+            return False
+        if self.model_format != other.model_format:
+            return False
+        if self.dependency_artifacts != other.dependency_artifacts:
+            return False
+        if self.dependency_artifacts is not None:
+            if len(self.dependency_artifacts) != len(other.dependency_artifacts):
+                return False
+            for self_artifact, other_artifact in zip(
+                self.dependency_artifacts, other.dependency_artifacts
+            ):
+                if self_artifact != other_artifact:
+                    return False
+        if self.code_artifacts != other.code_artifacts:
+            return False
+        return True

frogml_storage/models/serialization_metadata.py

@@ -0,0 +1,15 @@
+from typing import Dict
+
+from pydantic import BaseModel
+
+
+class SerializationMetadata(BaseModel):
+    framework: str
+    framework_version: str
+    serialization_format: str
+    runtime: str
+    runtime_version: str
+
+    @classmethod
+    def from_json(cls, json_dict: Dict) -> "SerializationMetadata":
+        return cls.model_validate(json_dict)

frogml_storage/utils/__init__.py

@@ -0,0 +1,12 @@
+from ._input_checks_utility import (
+    is_not_none,
+    is_valid_thread_number,
+    user_input_validation,
+    validate_not_folder_paths,
+    validate_path_exists,
+)
+from ._storage_utils import calculate_sha2, calc_content_sha2
+from ._url_utils import (
+    assemble_artifact_url,
+    join_url,
+)

frogml_storage/utils/_environment.py

@@ -0,0 +1,21 @@
+import importlib.metadata
+import platform
+from typing import List
+
+
+def get_environment_dependencies() -> List[str]:
+    distributions = importlib.metadata.distributions()
+    return sorted(
+        [f"{dist.metadata['Name']}=={dist.version}" for dist in distributions]
+    )
+
+
+def get_environment_details() -> List[str]:
+    return [
+        f"arch={platform.architecture()[0]}",
+        f"cpu={platform.processor()}",
+        f"platform={platform.platform()}",
+        f"python_version={platform.python_version()}",
+        f"python_implementation={platform.python_implementation()}",
+        f"python_compiler={platform.python_compiler()}",
+    ]

frogml_storage/utils/_input_checks_utility.py

@@ -0,0 +1,104 @@
+import os
+import re
+from typing import List, Optional, Union
+
+from frogml_storage.logging import logger
+from frogml_storage.constants import FROG_ML_MAX_CHARS_FOR_NAME
+from frogml_storage.exceptions.validation_error import FrogMLValidationError
+
+valid_characters_pattern = re.compile(r"^[a-zA-Z0-9_-]+(\.[a-zA-Z0-9_-]+)*$")
+
+
+def user_input_validation(
+    entity_name: Optional[str],
+    namespace: Optional[str],
+    version: Optional[str],
+    properties: Optional[dict[str, str]] = None,
+) -> bool:
+    for arg_name, arg_value in {
+        "entity_name": entity_name,
+        "namespace": namespace,
+        "version": version,
+    }.items():
+        if arg_value is not None:
+            __input_validation(arg_value, arg_name)
+
+    if properties is not None:
+        __user_input_dict_validation(properties)
+
+    return True
+
+
+def __user_input_dict_validation(properties: dict[str, str]) -> bool:
+    if properties is not None:
+        for key, value in properties.items():
+            __input_validation(key, "properties")
+            __input_validation(value, "properties")
+    return True
+
+
+def is_not_none(arg_name: str, arg_value: Optional[object]) -> bool:
+    if arg_value is None:
+        raise FrogMLValidationError("{} can't be 'None'.".format(arg_name))
+    return True
+
+
+def __input_validation(field_value: str, field_name: str) -> bool:
+    if len(str(field_value)) > FROG_ML_MAX_CHARS_FOR_NAME:
+        raise FrogMLValidationError(
+            "Max length for {} is 60 characters.".format(field_name.capitalize())
+        )
+
+    if not field_value or not re.match(valid_characters_pattern, str(field_value)):
+        raise FrogMLValidationError(
+            "Invalid characters detected at {}: {}".format(
+                field_name.capitalize(), field_value
+            )
+        )
+
+    return True
+
+
+def is_valid_thread_number(thread_count: str) -> bool:
+    try:
+        int_thread_count = int(thread_count)
+        cpu_count = os.cpu_count()
+        if int_thread_count <= 0 or (
+            cpu_count is not None and int_thread_count >= cpu_count
+        ):
+            raise ValueError(
+                "Invalid thread count: {}. The default value will be used.".format(
+                    thread_count
+                )
+            )
+        return True
+    except ValueError as e:
+        logger.warning("Thread count {}: {}".format(thread_count, e))
+        return False
+    except TypeError:
+        logger.debug("Thread count not configured. The default value will be used.")
+        return False
+
+
+def validate_not_folder_paths(paths: Union[Optional[List[str]], Optional[str]]) -> bool:
+    if paths is not None:
+        if isinstance(paths, List):
+            for path in paths:
+                __validate_not_folder_path(path)
+        else:
+            __validate_not_folder_path(paths)
+    return True
+
+
+def __validate_not_folder_path(path: str) -> bool:
+    if os.path.isdir(path):
+        raise FrogMLValidationError(
+            "file '{}' must be a file, but is a directory.".format(path)
+        )
+    return True
+
+
+def validate_path_exists(path: str) -> bool:
+    if not os.path.exists(path):
+        raise ValueError(f"Provided path does not exists : '{path}'")
+    return True

frogml_storage/utils/_storage_utils.py

@@ -0,0 +1,15 @@
+import hashlib
+
+
+def calculate_sha2(path: str) -> str:
+    sha256 = hashlib.sha256()
+    with open(path, "rb") as f:
+        while chunk := f.read(8192):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+
+def calc_content_sha2(content: str) -> str:
+    sha256 = hashlib.sha256()
+    sha256.update(content.encode("utf-8"))
+    return sha256.hexdigest()