digitalhub-runtime-python 0.5.0__py3-none-any.whl

digitalhub_runtime_python/utils/configuration.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import importlib.util as imputil
+from pathlib import Path
+from typing import Callable
+
+from digitalhub_core.utils.generic_utils import (
+    decode_string,
+    extract_archive,
+    get_bucket_and_key,
+    get_s3_source,
+    requests_chunk_download,
+)
+from digitalhub_core.utils.git_utils import clone_repository
+from digitalhub_core.utils.logger import LOGGER
+from digitalhub_core.utils.uri_utils import map_uri_scheme
+
+
+def get_function_from_source(path: Path, source_spec: dict) -> Callable:
+    """
+    Get function from source.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to save the function source.
+    source_spec : dict
+        Funcrion source spec.
+
+    Returns
+    -------
+    Callable
+        Function.
+    """
+    try:
+        function_code = save_function_source(path, source_spec)
+        handler_path, function_name = parse_handler(source_spec["handler"])
+        function_path = (function_code / handler_path).with_suffix(".py")
+        return import_function(function_path, function_name)
+    except Exception as e:
+        msg = f"Some error occurred while getting function. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def parse_handler(handler: str) -> tuple:
+    """
+    Parse handler.
+
+    Parameters
+    ----------
+    handler : str
+        Function handler
+
+    Returns
+    -------
+    str
+        Function handler.
+    """
+    parsed = handler.split(":")
+    if len(parsed) == 1:
+        return Path(""), parsed[0]
+    return Path(*parsed[0].split(".")), parsed[1]
+
+
+def save_function_source(path: Path, source_spec: dict) -> Path:
+    """
+    Save function source.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to save the function source.
+    source_spec : dict
+        Function source spec.
+
+    Returns
+    -------
+    Path
+        Path to the function source.
+    """
+    # Prepare path
+    path.mkdir(parents=True, exist_ok=True)
+
+    # Get relevant information
+    base64 = source_spec.get("base64")
+    source = source_spec.get("source")
+
+    scheme = None
+    if source is not None:
+        scheme = map_uri_scheme(source)
+
+    # Base64
+    if base64 is not None:
+        filename = "main.py"
+        if scheme == "local":
+            filename = Path(source).name
+
+        base64_path = path / filename
+        base64_path.write_text(decode_base64(base64))
+
+        if scheme is None or scheme == "local":
+            return base64_path
+
+    # Git repo
+    if scheme == "git":
+        get_repository(path, source)
+
+    # Http(s) or s3 presigned urls
+    elif scheme == "remote":
+        filename = path / "archive.zip"
+        get_remote_source(source, filename)
+        unzip(path, filename)
+
+    # S3 path
+    elif scheme == "s3":
+        filename = path / "archive.zip"
+        bucket, key = get_bucket_and_key(source)
+        get_s3_source(bucket, key, filename)
+        unzip(path, filename)
+
+    # Unsupported scheme
+    else:
+        raise RuntimeError(f"Unsupported scheme: {scheme}")
+
+    return path
+
+
+def get_remote_source(source: str, filename: Path) -> None:
+    """
+    Get remote source.
+
+    Parameters
+    ----------
+    source : str
+        HTTP(S) or S3 presigned URL.
+    filename : Path
+        Path where to save the function source.
+
+    Returns
+    -------
+    str
+        Function code.
+    """
+    try:
+        requests_chunk_download(source, filename)
+    except Exception as e:
+        msg = f"Some error occurred while downloading function source. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def unzip(path: Path, filename: Path) -> None:
+    """
+    Extract an archive.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to extract the archive.
+    filename : Path
+        Path to the archive.
+
+    Returns
+    -------
+    None
+    """
+
+    try:
+        extract_archive(path, filename)
+    except Exception as e:
+        msg = f"Source must be a valid zipfile. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def get_repository(path: Path, source: str) -> str:
+    """
+    Get repository.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to save the function source.
+    source : str
+        Git repository URL in format git://<url>.
+
+    Returns
+    -------
+    None
+    """
+    try:
+        clone_repository(path, source)
+    except Exception as e:
+        msg = f"Some error occurred while downloading function repo source. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def decode_base64(base64: str) -> str:
+    """
+    Decode base64 encoded code.
+
+    Parameters
+    ----------
+    base64 : str
+        The encoded code.
+
+    Returns
+    -------
+    str
+        The decoded code.
+
+    Raises
+    ------
+    RuntimeError
+        Error while decoding code.
+    """
+    try:
+        return decode_string(base64)
+    except Exception as e:
+        msg = f"Some error occurred while decoding function source. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def import_function(path: Path, handler: str) -> Callable:
+    """
+    Import a function from a module.
+
+    Parameters
+    ----------
+    path : Path
+        Path where the function source is located.
+    handler : str
+        Function name.
+
+    Returns
+    -------
+    Callable
+        Function.
+    """
+    try:
+        spec = imputil.spec_from_file_location(path.stem, path)
+        mod = imputil.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+        return getattr(mod, handler)
+    except Exception as e:
+        msg = f"Some error occurred while importing function. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e

digitalhub_runtime_python/utils/inputs.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import inspect
+import typing
+from typing import Any, Callable
+
+from digitalhub_core.context.builder import get_context
+from digitalhub_core.entities.artifacts.crud import artifact_from_dict
+from digitalhub_core.utils.generic_utils import parse_entity_key
+from digitalhub_core.utils.logger import LOGGER
+from digitalhub_data.entities.dataitems.crud import dataitem_from_dict
+from digitalhub_ml.entities.entity_types import EntityTypes
+from digitalhub_ml.entities.models.crud import model_from_dict
+from digitalhub_ml.entities.projects.crud import get_project
+
+if typing.TYPE_CHECKING:
+    from digitalhub_core.entities._base.entity import Entity
+    from digitalhub_core.entities.projects.entity import Project
+
+
+def get_project_(project_name: str) -> Project:
+    """
+    Get project.
+
+    Parameters
+    ----------
+    project_name : str
+        Project name.
+
+    Returns
+    -------
+    Project
+        Project.
+    """
+    try:
+        ctx = get_context(project_name)
+        return get_project(project_name, local=ctx.local)
+    except Exception as e:
+        msg = f"Error during project collection. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg)
+
+
+def get_entity_inputs(inputs: dict) -> dict[str, Entity]:
+    """
+    Set inputs.
+
+    Parameters
+    ----------
+    inputs : dict
+        Run inputs.
+    parameters : dict
+        Run parameters.
+    tmp_dir : Path
+        Temporary directory for storing dataitms and artifacts.
+
+    Returns
+    -------
+    dict
+        Mlrun inputs.
+    """
+    try:
+        inputs_objects = {}
+        for k, v in inputs.items():
+            _, entity_type, _, _, _ = parse_entity_key(v.get("key"))
+            if entity_type == EntityTypes.DATAITEMS.value:
+                inputs_objects[k] = dataitem_from_dict(v)
+            elif entity_type == EntityTypes.ARTIFACTS.value:
+                inputs_objects[k] = artifact_from_dict(v)
+            elif entity_type == EntityTypes.MODELS.value:
+                inputs_objects[k] = model_from_dict(v)
+        return inputs_objects
+    except Exception as e:
+        msg = f"Error during inputs collection. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
+def compose_inputs(
+    inputs: dict,
+    parameters: dict,
+    local_execution: bool,
+    func: Callable,
+    project: str | Project,
+    context: Any | None = None,
+    event: Any | None = None,
+) -> dict:
+    """
+    Compose inputs.
+
+    Parameters
+    ----------
+    inputs : dict
+        Run inputs.
+    parameters : dict
+        Run parameters.
+    local_execution : bool
+        Local execution.
+    func : Callable
+        Function to execute.
+    project : str
+        Project name.
+    context : nuclio_sdk.Context
+        Nuclio context.
+    event : nuclio_sdk.Event
+        Nuclio event.
+
+    Returns
+    -------
+    dict
+        Function inputs.
+    """
+    try:
+        entity_inputs = get_entity_inputs(inputs)
+        fnc_args = {**parameters, **entity_inputs}
+
+        fnc_parameters = inspect.signature(func).parameters
+        LOGGER.info(f"Function parameters: {'project' in fnc_parameters}")
+
+        _has_project = "project" in fnc_parameters
+        _has_context = "context" in fnc_parameters
+        _has_event = "event" in fnc_parameters
+
+        if _has_project:
+            if _has_context:
+                fnc_args["project"] = context.project
+            elif isinstance(project, str):
+                fnc_args["project"] = get_project_(project)
+            else:
+                fnc_args["project"] = project
+
+        if _has_context:
+            if context is not None and not local_execution:
+                fnc_args["context"] = context
+            else:
+                raise RuntimeError("Context is not available on local execution.")
+
+        if _has_event:
+            if event is not None and not local_execution:
+                fnc_args["event"] = event
+            else:
+                raise RuntimeError("Event is not available on local execution.")
+
+        return fnc_args
+
+    except Exception as e:
+        msg = f"Error during function arguments compostion. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e

digitalhub_runtime_python/utils/outputs.py

@@ -0,0 +1,204 @@
+from __future__ import annotations
+
+import pickle
+from typing import Any
+
+from digitalhub_core.entities._base.status import State
+from digitalhub_core.entities.artifacts.crud import new_artifact
+from digitalhub_core.entities.artifacts.entity import Artifact
+from digitalhub_core.utils.logger import LOGGER
+from digitalhub_data.entities.dataitems.crud import new_dataitem
+from digitalhub_data.entities.dataitems.entity.table import DataitemTable
+from digitalhub_data.readers.registry import DATAFRAME_TYPES
+
+
+def collect_outputs(results: Any, outputs: list[str], project_name: str) -> dict:
+    """
+    Collect outputs. Use the produced results directly.
+
+    Parameters
+    ----------
+    results : Any
+        Function outputs.
+    project : Project
+        Project object.
+
+    Returns
+    -------
+    dict
+        Function outputs.
+    """
+    objects = {}
+    results = listify_results(results)
+
+    for idx, item in enumerate(results):
+        try:
+            name = outputs[idx]
+        except IndexError:
+            name = f"output_{idx}"
+
+        if isinstance(item, (str, int, float, bool, bytes)):
+            objects[name] = item
+
+        elif f"{item.__class__.__module__}.{item.__class__.__name__}" in DATAFRAME_TYPES:
+            objects[name] = build_and_load_dataitem(name, project_name, item)
+
+        else:
+            objects[name] = build_and_load_artifact(name, project_name, item)
+
+    return objects
+
+
+def parse_outputs(results: Any, run_outputs: list, project_name: str) -> dict:
+    """
+    Parse outputs.
+
+    Parameters
+    ----------
+    results : Any
+        Function outputs.
+    project : Project
+        Project object.
+
+    Returns
+    -------
+    dict
+        Function outputs.
+    """
+    results_list = listify_results(results)
+    out_list = []
+    for idx, _ in enumerate(results_list):
+        try:
+            out_list.append(run_outputs.pop(0))
+        except IndexError:
+            out_list.append(f"output_{idx}")
+    return collect_outputs(results, out_list, project_name)
+
+
+def listify_results(results: Any) -> list:
+    """
+    Listify results.
+
+    Parameters
+    ----------
+    results : Any
+        Function outputs.
+
+    Returns
+    -------
+    list
+        Function outputs.
+    """
+    if results is None:
+        return []
+
+    if not isinstance(results, (tuple, list)):
+        results = [results]
+
+    return results
+
+
+def build_and_load_dataitem(name: str, project_name: str, data: Any) -> DataitemTable:
+    """
+    Build and load dataitem.
+
+    Parameters
+    ----------
+    name : str
+        Dataitem name.
+    project_name : str
+        Project name.
+    data : Any
+        Data.
+
+    Returns
+    -------
+    str
+        Dataitem key.
+    """
+    try:
+        path = f"s3://datalake/{project_name}/dataitems/table/{name}.parquet"
+        di: DataitemTable = new_dataitem(project=project_name, name=name, kind="table", path=path)
+        di.write_df(df=data)
+        return di
+    except Exception as e:
+        msg = f"Some error occurred while building and loading dataitem. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg)
+
+
+def build_and_load_artifact(name: str, project_name: str, data: Any) -> Artifact:
+    """
+    Build and load artifact.
+
+    Parameters
+    ----------
+    name : str
+        Artifact name.
+    project_name : str
+        Project name.
+    data : Any
+        Data.
+
+    Returns
+    -------
+    str
+        Artifact key.
+    """
+    try:
+        path = f"s3://datalake/{project_name}/artifacts/artifact/{name}.pickle"
+
+        # Dump item to pickle
+        with open(f"{name}.pickle", "wb") as f:
+            f.write(pickle.dumps(data))
+
+        art = new_artifact(project=project_name, name=name, kind="artifact", path=path)
+        art.upload(source=f"{name}.pickle")
+        return art
+
+    except Exception as e:
+        msg = f"Some error occurred while building and loading artifact. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg)
+
+
+def build_status(
+    parsed_execution: dict,
+    mapped_outputs: dict | None = None,
+) -> dict:
+    """
+    Collect outputs.
+
+    Parameters
+    ----------
+    parsed_execution : dict
+        Parsed execution dict.
+    mapped_outputs : dict
+        Mapped outputs.
+
+    Returns
+    -------
+    dict
+        Status dict.
+    """
+    results = {}
+    outputs = {}
+    if mapped_outputs is None:
+        mapped_outputs = {}
+
+    try:
+        for key, _ in mapped_outputs.items():
+            if key in parsed_execution:
+                if isinstance(parsed_execution[key], (DataitemTable, Artifact)):
+                    outputs[key] = parsed_execution[key].key
+                else:
+                    results[key] = parsed_execution[key]
+        return {
+            "state": State.COMPLETED.value,
+            "outputs": outputs,
+            "results": results,
+        }
+    except Exception as e:
+        msg = f"Something got wrong during run status building. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e

digitalhub_runtime_python/utils/utils.py

@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import functools
+from typing import Callable
+
+from digitalhub_runtime_python.utils.outputs import collect_outputs
+
+
+def handler(outputs: list[str] | None = None) -> Callable:
+    """
+    Decorator that handles the outputs of the function.
+
+    Parameters
+    ----------
+    outputs : list[str]
+        List of named outputs to collect.
+
+    Returns
+    -------
+    Callable
+        Decorated function.
+    """
+
+    def decorator(func: Callable) -> Callable:
+        """
+        Decorator that handles the outputs of the function.
+
+        Parameters
+        ----------
+        func : Callable
+            Function to decorate.
+
+        Returns
+        -------
+        Callable
+            Decorated function.
+        """
+
+        def wrapper(*args, **kwargs) -> dict:
+            """
+            Wrapper that handles the outputs of the function.
+
+            Parameters
+            ----------
+            args : tuple
+                Function arguments.
+            kwargs : dict
+                Function keyword arguments.
+
+            Returns
+            -------
+            Any
+                Function outputs.
+            """
+
+            # Initialize outputs
+            nonlocal outputs
+
+            # We pass the first argument as the project name
+            project_name = args[0]
+            args = args[1:]
+
+            # Execute the function
+            results = func(*args, **kwargs)
+
+            # Parse outputs based on the decorator signature
+            return collect_outputs(results, outputs, project_name)
+
+        wrapper = functools.wraps(func)(wrapper)
+
+        return wrapper
+
+    return decorator