digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl

digitalhub/utils/generic_utils.py

@@ -0,0 +1,183 @@
+from __future__ import annotations
+
+import base64
+import json
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+from zipfile import ZipFile
+
+import numpy as np
+from requests import get as requests_get
+from slugify import slugify
+
+from digitalhub.utils.io_utils import read_text
+
+
+def get_timestamp() -> str:
+    """
+    Get the current timestamp timezoned.
+
+    Returns
+    -------
+    str
+        The current timestamp.
+    """
+    return datetime.now().astimezone().isoformat()
+
+
+def decode_string(string: str) -> str:
+    """
+    Decode a string from base64.
+
+    Parameters
+    ----------
+    string : str
+        The string to decode.
+
+    Returns
+    -------
+    str
+        The string decoded from base64.
+    """
+    return base64.b64decode(string).decode()
+
+
+def encode_string(string: str) -> str:
+    """
+    Encode a string in base64.
+
+    Parameters
+    ----------
+    string : str
+        The string to encode.
+
+    Returns
+    -------
+    str
+        The string encoded in base64.
+    """
+    return base64.b64encode(string.encode()).decode()
+
+
+def encode_source(path: str) -> str:
+    """
+    Read a file and encode in base64 the content.
+
+    Parameters
+    ----------
+    path : str
+        The file path to read.
+
+    Returns
+    -------
+    str
+        The file content encoded in base64.
+    """
+    return encode_string(read_text(path))
+
+
+def requests_chunk_download(source: str, filename: Path) -> None:
+    """
+    Download a file in chunks.
+
+    Parameters
+    ----------
+    source : str
+        URL to download the file.
+    filename : Path
+        Path where to save the file.
+
+    Returns
+    -------
+    None
+    """
+    with requests_get(source, stream=True) as r:
+        r.raise_for_status()
+        with filename.open("wb") as f:
+            for chunk in r.iter_content(chunk_size=8192):
+                f.write(chunk)
+
+
+def extract_archive(path: Path, filename: Path) -> None:
+    """
+    Extract a zip archive.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to extract the archive.
+    filename : Path
+        Path to the archive.
+
+    Returns
+    -------
+    None
+    """
+    with ZipFile(filename, "r") as zip_file:
+        zip_file.extractall(path)
+
+
+class MyEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder to handle numpy types.
+    """
+
+    def default(self, obj: Any) -> Any:
+        """
+        Convert numpy types to json.
+
+        Parameters
+        ----------
+        obj : Any
+            The object to convert.
+
+        Returns
+        -------
+        Any
+            The object converted to json.
+        """
+        if isinstance(obj, (int, str, float, list, dict)):
+            return obj
+        elif isinstance(obj, (np.integer, np.int64)):
+            return int(obj)
+        elif isinstance(obj, (np.floating, np.float64)):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return str(obj)
+
+
+def dict_to_json(struct: dict) -> str:
+    """
+    Convert a dict to json.
+
+    Parameters
+    ----------
+    struct : dict
+        The dict to convert.
+
+    Returns
+    -------
+    str
+        The json string.
+    """
+    return json.dumps(struct, cls=MyEncoder)
+
+
+def slugify_string(filename: str) -> str:
+    """
+    Sanitize a filename.
+
+    Parameters
+    ----------
+    filename : str
+        The filename to sanitize.
+
+    Returns
+    -------
+    str
+        The sanitized filename.
+    """
+    return slugify(filename, max_length=255)

digitalhub/utils/git_utils.py

@@ -0,0 +1,148 @@
+from __future__ import annotations
+
+import os
+import shutil
+import warnings
+from pathlib import Path
+from urllib.parse import urlparse
+
+try:
+    from git import Repo
+except ImportError as e:
+    if "Bad git executable." in e.args[0]:
+        warnings.warn("git is not installed. Please install git and try again.", RuntimeWarning)
+
+
+def clone_repository(url: str, path: Path) -> None:
+    """
+    Clone git repository.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+    path : Path
+        Path where to save the repository.
+
+    Returns
+    -------
+    None
+    """
+    clean_path(path)
+    checkout_object = get_checkout_object(url)
+    url = add_credentials_git_remote_url(url)
+    repo = clone_from_url(url, path)
+    if checkout_object != "":
+        repo.git.checkout(checkout_object)
+
+
+def get_checkout_object(url: str) -> str:
+    """
+    Get checkout object from url fragment.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+
+    Returns
+    -------
+    str
+        Checkout object (branch, tag, commit).
+    """
+    return urlparse(url).fragment
+
+
+def clean_path(path: Path) -> None:
+    """
+    Clean path from any files.
+
+    Parameters
+    ----------
+    path : Path
+
+    Returns
+    -------
+    None
+    """
+
+    shutil.rmtree(path, ignore_errors=True)
+
+
+def get_git_username_password_from_token(token: str) -> tuple[str, str]:
+    """
+    Parse token to get username and password. The token
+    can be one of the following:
+
+    - GitHub/GitLab personal access token (github_pat_.../glpat...)
+    - GitHub/GitLab access token
+    - Other generic token
+
+    Parameters
+    ----------
+    token : str
+        Token to parse.
+
+    Returns
+    -------
+    tuple[str, str]
+        Username and password.
+    """
+    # Mutued from mlrun
+    if token.startswith("github_pat_") or token.startswith("glpat"):
+        username = "oauth2"
+        password = token
+    else:
+        username = token
+        password = "x-oauth-basic"
+    return username, password
+
+
+def add_credentials_git_remote_url(url: str) -> str:
+    """
+    Add credentials to git remote url.
+
+    Parameters
+    ----------
+    url : str
+        URL of the repository.
+
+    Returns
+    -------
+    str
+        URL with credentials.
+    """
+    url_obj = urlparse(url)
+
+    # Get credentials from environment variables
+    username = os.getenv("GIT_USERNAME")
+    password = os.getenv("GIT_PASSWORD")
+    token = os.getenv("GIT_TOKEN")
+
+    # Get credentials from token. Override username and password
+    if token is not None:
+        username, password = get_git_username_password_from_token(token)
+
+    # Add credentials to url if needed
+    if username is not None and password is not None:
+        return f"https://{username}:{password}@{url_obj.hostname}{url_obj.path}"
+    return f"https://{url_obj.hostname}{url_obj.path}"
+
+
+def clone_from_url(url: str, path: Path) -> Repo:
+    """
+    Clone repository from url. Wraps git.Repo.clone_from.
+
+    Parameters
+    ----------
+    url : str
+        HTTP(S) URL of the repository.
+    path : Path
+        Path where to save the repository.
+
+    Returns
+    -------
+    Repo
+        Cloned repository.
+    """
+    return Repo.clone_from(url=url, to_path=path)

digitalhub/utils/io_utils.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import yaml
+
+##############################
+# Writers
+##############################
+
+
+def write_yaml(filepath: str | Path, obj: dict | list[dict]) -> None:
+    """
+    Write a dict or a list of dict to a yaml file.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The yaml file path to write.
+    obj : dict
+        The dict to write.
+
+    Returns
+    -------
+    None
+    """
+    if isinstance(obj, list):
+        with open(filepath, "w", encoding="utf-8") as out_file:
+            yaml.dump_all(obj, out_file, sort_keys=False, default_flow_style=False)
+    else:
+        with open(filepath, "w", encoding="utf-8") as out_file:
+            yaml.dump(obj, out_file, sort_keys=False)
+
+
+##############################
+# Readers
+##############################
+
+
+class NoDatesSafeLoader(yaml.SafeLoader):
+    """
+    Loader implementation to exclude implicit resolvers.
+
+    Taken from https://stackoverflow.com/a/37958106
+    """
+
+    @classmethod
+    def remove_implicit_resolver(cls, tag_to_remove: str) -> None:
+        """
+        Remove implicit resolvers for a particular tag
+        Takes care not to modify resolvers in super classes.
+        We want to load datetimes as strings, not dates, because we
+        go on to serialise as json which doesn't have the advanced types
+        of yaml, and leads to incompatibilities down the track.
+
+        Parameters
+        ----------
+        tag_to_remove : str
+            The tag to remove
+
+        Returns
+        -------
+        None
+        """
+        if "yaml_implicit_resolvers" not in cls.__dict__:
+            cls.yaml_implicit_resolvers = cls.yaml_implicit_resolvers.copy()
+
+        for first_letter, mappings in cls.yaml_implicit_resolvers.items():
+            cls.yaml_implicit_resolvers[first_letter] = [
+                (tag, regexp) for tag, regexp in mappings if tag != tag_to_remove
+            ]
+
+
+NoDatesSafeLoader.remove_implicit_resolver("tag:yaml.org,2002:timestamp")
+
+
+def read_yaml(filepath: str | Path) -> dict | list[dict]:
+    """
+    Read a yaml file and return a dict or a list of dict.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The yaml file path to read.
+
+    Returns
+    -------
+    dict | list[dict]
+        The yaml file content.
+    """
+    try:
+        with open(filepath, "r", encoding="utf-8") as in_file:
+            data = yaml.load(in_file, Loader=NoDatesSafeLoader)
+
+    # If yaml contains multiple documents
+    except yaml.composer.ComposerError:
+        with open(filepath, "r", encoding="utf-8") as in_file:
+            data = list(yaml.load_all(in_file, Loader=NoDatesSafeLoader))
+    return data
+
+
+def read_text(filepath: str | Path) -> str:
+    """
+    Read a file and return the text.
+
+    Parameters
+    ----------
+    filepath : str | Path
+        The file path to read.
+
+    Returns
+    -------
+    str
+        The file content.
+    """
+    return Path(filepath).read_text(encoding="utf-8")

digitalhub/utils/logger.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+import logging
+
+# Create logger
+LOGGER = logging.getLogger("digitalhub-core")
+LOGGER.setLevel(logging.INFO)
+
+# Create formatter
+formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+
+# Create console handler and set formatter
+console_handler = logging.StreamHandler()
+console_handler.setFormatter(formatter)
+
+# Set console handler to the logger
+LOGGER.addHandler(console_handler)

digitalhub/utils/s3_utils.py

@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from urllib.parse import urlparse
+
+from boto3 import client as boto3_client
+
+
+def get_bucket_and_key(path: str) -> tuple[str, str]:
+    """
+    Get bucket and key from path.
+
+    Parameters
+    ----------
+    path : str
+        The source path to get the key from.
+
+    Returns
+    -------
+    tuple[str, str]
+        The bucket and key.
+    """
+    parsed = urlparse(path)
+    return parsed.netloc, parsed.path
+
+
+def get_s3_source(bucket: str, key: str, filename: Path) -> None:
+    """
+    Get S3 source.
+
+    Parameters
+    ----------
+    bucket : str
+        S3 bucket name.
+    key : str
+        S3 object key.
+    filename : Path
+        Path where to save the function source.
+
+    Returns
+    -------
+    None
+    """
+    s3 = boto3_client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL"))
+    s3.download_file(bucket, key, filename)
+
+
+def get_s3_bucket() -> str | None:
+    """
+    Function to get S3 bucket name.
+
+    Returns
+    -------
+    str
+        The S3 bucket name.
+    """
+    return os.getenv("S3_BUCKET_NAME", "datalake")

digitalhub/utils/uri_utils.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from urllib.parse import urlparse
+
+
+def map_uri_scheme(uri: str) -> str:
+    """
+    Map an URI scheme to a common scheme.
+
+    Parameters
+    ----------
+    uri : str
+        URI.
+
+    Returns
+    -------
+    str
+        Mapped scheme type.
+
+    Raises
+    ------
+    ValueError
+        If the scheme is unknown.
+    """
+    scheme = urlparse(uri).scheme
+    if scheme in [""]:
+        return "local"
+    if scheme in ["file", "local"]:
+        raise ValueError("For local path, do not use any scheme")
+    if scheme in ["http", "https"]:
+        return "remote"
+    if scheme in ["s3", "s3a", "s3n", "zip+s3"]:
+        return "s3"
+    if scheme in ["sql", "postgresql"]:
+        return "sql"
+    if scheme in ["git", "git+http", "git+https"]:
+        return "git"
+    raise ValueError(f"Unknown scheme '{scheme}'!")
+
+
+def check_local_path(path: str) -> bool:
+    """
+    Check if path is local.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is local.
+    """
+    scheme = map_uri_scheme(path)
+    return scheme == "local"

{digitalhub-0.7.0b2.dist-info → digitalhub-0.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: digitalhub
-Version: 0.7.0b2
+Version: 0.8.0
 Summary: Python SDK for Digitalhub
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License: Apache License
@@ -225,25 +225,42 @@ Keywords: data,dataops,kubernetes
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: digitalhub-core<0.8,>=0.7.0b
-Provides-Extra: all
-Requires-Dist: digitalhub-data<0.8,>=0.7.0b; extra == "all"
-Requires-Dist: digitalhub-ml<0.8,>=0.7.0b; extra == "all"
-Provides-Extra: data
-Requires-Dist: digitalhub-data<0.8,>=0.7.0b; extra == "data"
+Requires-Dist: boto3
+Requires-Dist: pydantic<2
+Requires-Dist: sqlalchemy<2
+Requires-Dist: pyarrow
+Requires-Dist: numpy<2
+Requires-Dist: requests
+Requires-Dist: PyYAML
+Requires-Dist: python-dotenv
+Requires-Dist: GitPython>=3
+Requires-Dist: psycopg2-binary
+Requires-Dist: python-slugify
+Provides-Extra: dev
+Requires-Dist: black; extra == "dev"
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: bumpver; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: moto; extra == "dev"
+Provides-Extra: docs
+Requires-Dist: Sphinx>=7; extra == "docs"
+Requires-Dist: pydata-sphinx-theme>=0.15; extra == "docs"
+Requires-Dist: numpydoc>=1.6; extra == "docs"
 Provides-Extra: full
-Requires-Dist: digitalhub-core[full]<0.8,>=0.7.0b; extra == "full"
-Requires-Dist: digitalhub-data[pandas]<0.8,>=0.7.0b; extra == "full"
-Requires-Dist: digitalhub-ml[mlflow]<0.8,>=0.7.0b; extra == "full"
-Provides-Extra: ml
-Requires-Dist: digitalhub-ml<0.8,>=0.7.0b; extra == "ml"
+Requires-Dist: pandas<2.2,>=1.2; extra == "full"
+Requires-Dist: mlflow; extra == "full"
+Provides-Extra: mlflow
+Requires-Dist: mlflow; extra == "mlflow"
+Provides-Extra: pandas
+Requires-Dist: pandas<2.2,>=1.2; extra == "pandas"
 
 # Digitalhub Library
 
 The Digitalhub SDK library is used to manage entities and executions in Digitalhub from Python.
 It comes with a suite of tools to help you manage your projects and executions. It exposes CRUD methods to create, read, update and delete entities, and objects methods to excute functions or workflows, collect or store execution results and data.
 
-A more detailed description of the library can be found in the [official documentation](https://scc-digitalhub.github.io/docs/) of Digitalhub.
+A more detailed description of the library can be found in the [official documentation](https://scc-digitalhub.github.io/sdk-docs/).