digitalhub 0.8.0b0__py3-none-any.whl → 0.8.0b1__py3-none-any.whl

digitalhub/utils/file_utils.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+from datetime import datetime
+from hashlib import sha256
+from mimetypes import guess_type
+from pathlib import Path
+
+from pydantic import BaseModel
+
+
+class FileInfo(BaseModel):
+    """
+    File info class.
+    """
+
+    path: str = None
+    name: str = None
+    content_type: str = None
+    size: int = None
+    hash: str = None
+    last_modified: str = None
+
+
+def calculate_blob_hash(data_path: str) -> str:
+    """
+    Calculate the hash of a file.
+
+    Parameters
+    ----------
+    data_path : str
+        Path to the file.
+
+    Returns
+    -------
+    str
+        The hash of the file.
+    """
+    with open(data_path, "rb") as f:
+        data = f.read()
+        return f"sha256:{sha256(data).hexdigest()}"
+
+
+def get_file_size(data_path: str) -> int:
+    """
+    Get the size of a file.
+
+    Parameters
+    ----------
+    data_path : str
+        Path to the file.
+
+    Returns
+    -------
+    int
+        The size of the file.
+    """
+    return Path(data_path).stat().st_size
+
+
+def get_file_mime_type(data_path: str) -> str:
+    """
+    Get the mime type of a file.
+
+    Parameters
+    ----------
+    data_path : str
+        Path to the file.
+
+    Returns
+    -------
+    str
+        The mime type of the file.
+    """
+    return guess_type(data_path)[0]
+
+
+def get_path_name(data_path: str) -> str:
+    """
+    Get the name of a file.
+
+    Parameters
+    ----------
+    data_path : str
+        Path to the file.
+
+    Returns
+    -------
+    str
+        The name of the file.
+    """
+    return Path(data_path).name
+
+
+def get_last_modified(data_path: str) -> str:
+    """
+    Get the last modified date of a file.
+
+    Parameters
+    ----------
+    data_path : str
+        Path to the file.
+
+    Returns
+    -------
+    str
+        The last modified date of the file.
+    """
+    path = Path(data_path)
+    timestamp = path.stat().st_mtime
+    return datetime.fromtimestamp(timestamp).astimezone().isoformat()
+
+
+def get_s3_path(src_path: str) -> str:
+    """
+    Get the S3 path of a file.
+
+    Parameters
+    ----------
+    src_path : str
+        Path to the file.
+
+    Returns
+    -------
+    str
+        The S3 path of the file.
+    """
+    return Path(src_path).as_uri()
+
+
+def get_file_info_from_local(path: str, src_path: str) -> None | dict:
+    """
+    Get file info from path.
+
+    Parameters
+    ----------
+    path : str
+        Target path of the object.
+    src_path : str
+        Local path of some source.
+
+    Returns
+    -------
+    dict
+        File info.
+    """
+    try:
+        name = get_path_name(path)
+        content_type = get_file_mime_type(path)
+        size = get_file_size(path)
+        hash = calculate_blob_hash(path)
+        last_modified = get_last_modified(path)
+
+        return FileInfo(
+            path=src_path,
+            name=name,
+            content_type=content_type,
+            size=size,
+            hash=hash,
+            last_modified=last_modified,
+        ).dict()
+    except Exception:
+        return None
+
+
+def get_file_info_from_s3(path: str, metadata: dict) -> None | dict:
+    """
+    Get file info from path.
+
+    Parameters
+    ----------
+    path : str
+        Object source path.
+    metadata : dict
+        Metadata of the object from S3.
+
+    Returns
+    -------
+    dict
+        File info.
+    """
+    try:
+        size = metadata["ContentLength"]
+        file_hash = metadata["ETag"][1:-1]
+
+        file_size_limit_multipart = 20 * 1024 * 1024
+        if size < file_size_limit_multipart:
+            file_hash = "md5:" + file_hash
+        else:
+            file_hash = "LiteralETag:" + file_hash
+
+        name = get_path_name(path)
+        content_type = metadata["ContentType"]
+        last_modified = metadata["LastModified"].isoformat()
+
+        return FileInfo(
+            path=path,
+            name=name,
+            content_type=content_type,
+            size=size,
+            hash=file_hash,
+            last_modified=last_modified,
+        ).dict()
+    except Exception:
+        return None

digitalhub/utils/generic_utils.py

@@ -0,0 +1,207 @@
+from __future__ import annotations
+
+import base64
+import json
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+from zipfile import ZipFile
+
+import numpy as np
+from boto3 import client as boto3_client
+from requests import get as requests_get
+
+from digitalhub.utils.io_utils import read_text
+
+
+def get_timestamp() -> str:
+    """
+    Get the current timestamp timezoned.
+
+    Returns
+    -------
+    str
+        The current timestamp.
+    """
+    return datetime.now().astimezone().isoformat()
+
+
+def decode_string(string: str) -> str:
+    """
+    Decode a string from base64.
+
+    Parameters
+    ----------
+    string : str
+        The string to decode.
+
+    Returns
+    -------
+    str
+        The string decoded from base64.
+    """
+    return base64.b64decode(string).decode()
+
+
+def encode_string(string: str) -> str:
+    """
+    Encode a string in base64.
+
+    Parameters
+    ----------
+    string : str
+        The string to encode.
+
+    Returns
+    -------
+    str
+        The string encoded in base64.
+    """
+    return base64.b64encode(string.encode()).decode()
+
+
+def encode_source(path: str) -> str:
+    """
+    Read a file and encode in base64 the content.
+
+    Parameters
+    ----------
+    path : str
+        The file path to read.
+
+    Returns
+    -------
+    str
+        The file content encoded in base64.
+    """
+    return encode_string(read_text(path))
+
+
+def requests_chunk_download(source: str, filename: Path) -> None:
+    """
+    Download a file in chunks.
+
+    Parameters
+    ----------
+    source : str
+        URL to download the file.
+    filename : Path
+        Path where to save the file.
+
+    Returns
+    -------
+    None
+    """
+    with requests_get(source, stream=True) as r:
+        r.raise_for_status()
+        with filename.open("wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+
+def extract_archive(path: Path, filename: Path) -> None:
+    """
+    Extract a zip archive.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to extract the archive.
+    filename : Path
+        Path to the archive.
+
+    Returns
+    -------
+    None
+    """
+    with ZipFile(filename, "r") as zip_file:
+        zip_file.extractall(path)
+
+
+def get_s3_source(bucket: str, key: str, filename: Path) -> None:
+    """
+    Get S3 source.
+
+    Parameters
+    ----------
+    bucket : str
+        S3 bucket name.
+    key : str
+        S3 object key.
+    filename : Path
+        Path where to save the function source.
+
+    Returns
+    -------
+    None
+    """
+    s3 = boto3_client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL"))
+    s3.download_file(bucket, key, filename)
+
+
+def get_bucket_and_key(path: str) -> tuple[str, str]:
+    """
+    Get bucket and key from path.
+
+    Parameters
+    ----------
+    path : str
+        The source path to get the key from.
+
+    Returns
+    -------
+    tuple[str, str]
+        The bucket and key.
+    """
+    parsed = urlparse(path)
+    return parsed.netloc, parsed.path
+
+
+class MyEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder to handle numpy types.
+    """
+
+    def default(self, obj: Any) -> Any:
+        """
+        Convert numpy types to json.
+
+        Parameters
+        ----------
+        obj : Any
+            The object to convert.
+
+        Returns
+        -------
+        Any
+            The object converted to json.
+        """
+        if isinstance(obj, (int, str, float, list, dict)):
+            return obj
+        elif isinstance(obj, (np.integer, np.int64)):
+            return int(obj)
+        elif isinstance(obj, (np.floating, np.float64)):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return str(obj)
+
+
+def dict_to_json(struct: dict) -> str:
+    """
+    Convert a dict to json.
+
+    Parameters
+    ----------
+    struct : dict
+        The dict to convert.
+
+    Returns
+    -------
+    str
+        The json string.
+    """
+    return json.dumps(struct, cls=MyEncoder)

digitalhub/utils/git_utils.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import os
+import shutil
+import warnings
+from pathlib import Path
+from urllib.parse import urlparse
+
+try:
+    from git import Repo
+except ImportError as e:
+    if "Bad git executable." in e.args[0]:
+        warnings.warn("git is not installed. Please install git and try again.", RuntimeWarning)
+
+
+def clone_repository(url: str, path: Path) -> None:
+    """
+    Clone git repository.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+    path : Path
+        Path where to save the repository.
+
+    Returns
+    -------
+    None
+    """
+    clean_path(path)
+    checkout_object = get_checkout_object(url)
+    url = add_credentials_git_remote_url(url)
+    repo = clone_from_url(url, path)
+    if checkout_object != "":
+        repo.git.checkout(checkout_object)
+
+
+def get_checkout_object(url: str) -> str:
+    """
+    Get checkout object from url fragment.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+
+    Returns
+    -------
+    str
+        Checkout object (branch, tag, commit).
+    """
+    return urlparse(url).fragment
+
+
+def clean_path(path: Path) -> None:
+    """
+    Clean path from any files.
+
+    Parameters
+    ----------
+    path : Path
+
+    Returns
+    -------
+    None
+    """
+
+    shutil.rmtree(path, ignore_errors=True)
+
+
+def get_git_username_password_from_token(token: str) -> tuple[str, str]:
+    """
+    Parse token to get username and password. The token
+    can be one of the following:
+
+    - GitHub/GitLab personal access token (github_pat_.../glpat...)
+    - GitHub/GitLab access token
+    - Other generic token
+
+    Parameters
+    ----------
+    token : str
+        Token to parse.
+
+    Returns
+    -------
+    tuple[str, str]
+        Username and password.
+    """
+    # Mutued from mlrun
+    if token.startswith("github_pat_") or token.startswith("glpat"):
+        username = "oauth2"
+        password = token
+    else:
+        username = token
+        password = "x-oauth-basic"
+    return username, password
+
+
+def add_credentials_git_remote_url(url: str) -> str:
+    """
+    Add credentials to git remote url.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+
+    Returns
+    -------
+    str
+        URL with credentials.
+    """
+    url_obj = urlparse(url)
+
+    # Get credentials from environment variables
+    username = os.getenv("GIT_USERNAME")
+    password = os.getenv("GIT_PASSWORD")
+    token = os.getenv("GIT_TOKEN")
+
+    # Get credentials from token. Override username and password
+    if token is not None:
+        username, password = get_git_username_password_from_token(token)
+
+    # Add credentials to url if needed
+    if username is not None and password is not None:
+        return f"https://{username}:{password}@{url_obj.hostname}{url_obj.path}"
+    return f"https://{url_obj.hostname}{url_obj.path}"
+
+
+def clone_from_url(url: str, path: Path) -> Repo:
+    """
+    Clone repository from url. Wraps git.Repo.clone_from.
+
+    Parameters
+    ----------
+    url : str
+        HTTP(S) URL of the repository.
+    path : Path
+        Path where to save the repository.
+
+    Returns
+    -------
+    Repo
+        Cloned repository.
+    """
+    return Repo.clone_from(url=url, to_path=path)

digitalhub/utils/io_utils.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+##############################
+# Writers
+##############################
+
+
+def write_yaml(filepath: str | Path, obj: dict | list[dict]) -> None:
+    """
+    Write a dict or a list of dict to a yaml file.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The yaml file path to write.
+    obj : dict
+        The dict to write.
+
+    Returns
+    -------
+    None
+    """
+    if isinstance(obj, list):
+        with open(filepath, "w", encoding="utf-8") as out_file:
+            yaml.dump_all(obj, out_file, sort_keys=False, default_flow_style=False)
+    else:
+        with open(filepath, "w", encoding="utf-8") as out_file:
+            yaml.dump(obj, out_file, sort_keys=False)
+
+
+##############################
+# Readers
+##############################
+
+
+def read_yaml(filepath: str | Path) -> dict | list[dict]:
+    """
+    Read a yaml file and return a dict or a list of dict.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The yaml file path to read.
+
+    Returns
+    -------
+    dict | list[dict]
+        The yaml file content.
+    """
+    try:
+        with open(filepath, "r", encoding="utf-8") as in_file:
+            data = yaml.load(in_file, Loader=yaml.SafeLoader)
+
+    # If yaml contains multiple documents
+    except yaml.composer.ComposerError:
+        with open(filepath, "r", encoding="utf-8") as in_file:
+            data = list(yaml.load_all(in_file, Loader=yaml.SafeLoader))
+    return data
+
+
+def read_text(filepath: str | Path) -> str:
+    """
+    Read a file and return the text.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The file path to read.
+
+    Returns
+    -------
+    str
+        The file content.
+    """
+    return Path(filepath).read_text(encoding="utf-8")

digitalhub/utils/logger.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import logging
+
+# Create logger
+LOGGER = logging.getLogger("digitalhub-core")
+LOGGER.setLevel(logging.INFO)
+
+# Create formatter
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+
+# Create console handler and set formatter
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(formatter)
+
+# Set console handler to the logger
+LOGGER.addHandler(console_handler)

digitalhub/utils/uri_utils.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+
+def map_uri_scheme(uri: str) -> str:
+    """
+    Map an URI scheme to a common scheme.
+
+    Parameters
+    ----------
+    uri : str
+        URI.
+
+    Returns
+    -------
+    str
+        Mapped scheme type.
+
+    Raises
+    ------
+    ValueError
+        If the scheme is unknown.
+    """
+    scheme = urlparse(uri).scheme
+    if scheme in [""]:
+        return "local"
+    if scheme in ["file", "local"]:
+        raise ValueError("For local path, do not use any scheme")
+    if scheme in ["http", "https"]:
+        return "remote"
+    if scheme in ["s3", "s3a", "s3n", "zip+s3"]:
+        return "s3"
+    if scheme in ["sql", "postgresql"]:
+        return "sql"
+    if scheme in ["git", "git+http", "git+https"]:
+        return "git"
+    raise ValueError(f"Unknown scheme '{scheme}'!")
+
+
+def check_local_path(path: str) -> bool:
+    """
+    Check if path is local.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is local.
+    """
+    scheme = map_uri_scheme(path)
+    return scheme == "local"