qwak-core 0.4.265__py3-none-any.whl → 0.4.266__py3-none-any.whl

frogml_storage/cli/models/_cli_login_arguments.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+
+class LoginArguments(object):
+    isAnonymous: bool = False
+    server_id: Optional[str] = None
+    username: Optional[str] = None
+    password: Optional[str] = None
+    access_token: Optional[str] = None
+    artifactory_url: Optional[str] = None
+
+    def __eq__(self, other):
+        if not isinstance(other, LoginArguments):
+            return False
+        return (
+            self.isAnonymous == other.isAnonymous
+            and self.server_id == other.server_id
+            and self.username == other.username
+            and self.password == other.password
+            and self.access_token == other.access_token
+            and self.artifactory_url == other.artifactory_url
+        )

frogml_storage/cli/utils/_cli_utils.py

@@ -0,0 +1,19 @@
+from typing import List
+
+
+def get_list_of_servers_from_config(jfrog_cli_config: dict) -> List[str]:
+    if jfrog_cli_config is not None:
+        servers = jfrog_cli_config.get("servers")
+        if servers is not None:
+            return list(map(__map_server_ids, servers))
+
+    return []
+
+
+def __map_server_ids(server: dict) -> str:
+    server_id = ""
+    if server is not None:
+        server_id = str(server.get("serverId"))
+        if server.get("isDefault") is not None and bool(server.get("isDefault")):
+            server_id = server_id + " (Default)"
+    return server_id

frogml_storage/cli/utils/_login_checks_utility.py

@@ -0,0 +1,114 @@
+from typing import Optional
+
+from frogml_storage.authentication.models._auth_config import AuthConfig
+
+
+def login_input_checks(
+    url: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    token: Optional[str] = None,
+    anonymous: bool = False,
+) -> bool:
+    return (
+        __is_user_name_password_command(url, username, password, token, anonymous)
+        or __is_access_token_command(url, username, password, token, anonymous)
+        or __is_anonymous_command(url, username, password, token, anonymous)
+    )
+
+
+def is_login_command_without_params(
+    url: Optional[str] = None,
+    username: Optional[str] = None,
+    password: Optional[str] = None,
+    token: Optional[str] = None,
+    anonymous: bool = False,
+) -> bool:
+    return (
+        all(params is None for params in [url, username, password, token])
+        and not anonymous
+    )
+
+
+def is_username_password_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    return __is_user_name_password_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def is_access_token_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    if anonymous is None:
+        anonymous = False
+    return __is_access_token_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def is_anonymous_login(
+    auth_config: AuthConfig, anonymous: Optional[bool] = False
+) -> bool:
+    if anonymous is None:
+        anonymous = False
+    return __is_anonymous_command(
+        url=auth_config.artifactory_url,
+        username=auth_config.user,
+        password=auth_config.password,
+        token=auth_config.access_token,
+        anonymous=anonymous,
+    )
+
+
+def __is_anonymous_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: bool,
+) -> bool:
+    return (
+        anonymous
+        and all(params is None for params in [username, password, token])
+        and url is not None
+    )
+
+
+def __is_access_token_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: bool,
+) -> bool:
+    if anonymous:
+        return False
+    return all(params is None for params in [username, password]) and all(
+        params is not None for params in [url, token]
+    )
+
+
+def __is_user_name_password_command(
+    url: Optional[str],
+    username: Optional[str],
+    password: Optional[str],
+    token: Optional[str],
+    anonymous: Optional[bool] = False,
+) -> bool:
+    return (
+        not anonymous
+        and url is not None
+        and username is not None
+        and password is not None
+        and token is None
+    )

frogml_storage/constants.py

@@ -0,0 +1,56 @@
+import os
+
+home_dir = os.path.expanduser("~")
+
+JFROG_CLI_CONFIG_FILE_PATH = os.path.join(home_dir, ".jfrog", "jfrog-cli.conf.v6")
+CONFIG_FILE_PATH = os.path.join(home_dir, ".frogml", "config.json")
+
+FROG_ML_CONFIG_USER = "user"
+FROG_ML_CONFIG_ARTIFACTORY_URL = "artifactory_url"
+FROG_ML_CONFIG_PASSWORD = "password"  # nosec B105
+FROG_ML_CONFIG_ACCESS_TOKEN = "access_token"  # nosec B105
+FROG_ML_DEFAULT_HTTP_THREADS_COUNT = 5
+FROG_ML_MAX_CHARS_FOR_NAME = 60
+
+SERVER_ID = "server_id"
+
+JFROG_CLI_CONFIG_ARTIFACTORY_URL = "artifactoryUrl"
+JFROG_CLI_CONFIG_URL = "url"
+JFROG_CLI_CONFIG_USER = "user"
+JFROG_CLI_CONFIG_PASSWORD = "password"  # nosec B105
+JFROG_CLI_CONFIG_ACCESS_TOKEN = "accessToken"  # nosec B105
+
+MODEL = "model"
+ROOT_FROGML_MODEL_UI_DIRECTORY = "models"
+MODEL_UI_DIRECTORY = "model"
+MODEL_METADATA_FILE_NAME = "model-manifest.json"
+BODY_PART_MODEL_MANIFEST_STREAM = "modelManifest"
+
+DATASET = "dataset"
+ROOT_FROGML_DATASET_UI_DIRECTORY = "datasets"
+DATASET_UI_DIRECTORY = "dataset"
+DATASET_METADATA_FILE_NAME = "dataset-manifest.json"
+BODY_PART_DATASET_MANIFEST_STREAM = "datasetManifest"
+
+CHECKSUM_SHA2_HEADER = "X-Checksum-Sha256"
+
+JFML_THREAD_COUNT = "JFML_THREAD_COUNT"
+JF_URL = "JF_URL"
+JF_ACCESS_TOKEN = "JF_ACCESS_TOKEN"  # nosec B105
+
+FROG_ML_IGNORED_FILES = [
+    ".DS_Store",
+    "CVS",
+    ".cvsignore",
+    "SCCS",
+    "vssver.scc",
+    ".svn",
+    ".git",
+    ".gitignore",
+    ".gitattributes",
+    ".gitmodules",
+    ".gitkeep",
+    ".gitconfig",
+    MODEL_METADATA_FILE_NAME,
+    DATASET_METADATA_FILE_NAME,
+]

frogml_storage/dataset_manifest.py

@@ -0,0 +1,13 @@
+from typing import List
+
+from pydantic import Field
+
+from frogml_storage.entity_manifest import Artifact, EntityManifest
+
+
+class DatasetManifest(EntityManifest):
+    """
+    Represent a dataset manifest file
+    """
+
+    artifacts: List[Artifact] = Field(serialization_alias="dataset_artifacts")

frogml_storage/entity_manifest.py

@@ -0,0 +1,93 @@
+import os
+from abc import ABC
+from typing import List, Optional
+
+from pydantic import BaseModel
+
+from frogml_storage._storage_utils import calc_content_sha2, calculate_sha2
+
+
+class Checksums(BaseModel):
+    sha2: str
+
+    @classmethod
+    def calc_checksums(cls, file_path: str) -> "Checksums":
+        return cls(sha2=calculate_sha2(file_path))
+
+    @classmethod
+    def calc_content_checksums(cls, content: str) -> "Checksums":
+        return cls(sha2=calc_content_sha2(content))
+
+
+class Artifact(BaseModel):
+    artifact_path: str
+    size: int
+    checksums: Checksums
+
+    def __eq__(self, other):
+        if not isinstance(other, Artifact):
+            return False
+        return (
+            self.artifact_path == other.artifact_path
+            and self.size == other.size
+            and self.checksums == other.checksums
+        )
+
+
+class EntityManifest(BaseModel, ABC):
+    """
+    Represent an entity manifest file
+
+    Attributes:
+        created_date: The date the model | dataset were uploaded to Artifactory
+        artifacts: A list of artifacts that belong to the model | dataset
+        id: <organization>/<entity_name> - exists only for downloaded EntityInfo
+        version: The entity version - exists only for downloaded EntityInfo
+    """
+
+    created_date: str
+    artifacts: List[Artifact]
+    id: Optional[str] = None
+    version: Optional[str] = None
+
+    def add_file(self, file_path: str, checksums: Checksums, rel_path: str) -> None:
+        self.artifacts.append(
+            Artifact(
+                artifact_path=rel_path,
+                size=os.path.getsize(file_path),
+                checksums=checksums,
+            )
+        )
+
+    def add_content_file(self, rel_path: str, content: str) -> None:
+        checksums = Checksums.calc_content_checksums(content)
+        self.artifacts.append(
+            Artifact(
+                artifact_path=rel_path,
+                size=len((content, "utf-8")),
+                checksums=checksums,
+            )
+        )
+
+    @classmethod
+    def from_json(cls, json_str: str) -> "EntityManifest":
+        return cls.model_validate_json(json_str)
+
+    def to_json(self) -> str:
+        return self.model_dump_json(by_alias=True, exclude_none=True)
+
+    def __eq__(self, other):
+        if not isinstance(other, EntityManifest):
+            return False
+        if self.id != other.id:
+            return False
+        if self.version != other.version:
+            return False
+        if self.created_date != other.created_date:
+            return False
+        if len(self.artifacts) != len(other.artifacts):
+            return False
+        for self_artifact, other_artifact in zip(self.artifacts, other.artifacts):
+            if self_artifact != other_artifact:
+                return False
+        return True

frogml_storage/exceptions/checksum_verification_error.py

@@ -0,0 +1,3 @@
+class ChecksumVerificationError(Exception):
+    def __init__(self, filename):
+        super().__init__(f"Checksum verification failed for file: '{filename}'")

frogml_storage/exceptions/validation_error.py

@@ -0,0 +1,4 @@
+class FrogMLValidationError(ValueError):
+    def __init__(self, message, errors=None):
+        super().__init__(message)
+        self.errors = errors or {}