digitalhub-runtime-python 0.5.0b10__tar.gz → 0.6.0__tar.gz

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: digitalhub-runtime-python
-Version: 0.5.0b10
+Version: 0.6.0
 Summary: Python runtime for DHCore
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
@@ -228,6 +228,6 @@ Classifier: Programming Language :: Python :: 3.10
 Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
-Requires-Dist: digitalhub[ml]<0.6,~=0.5.0b
+Requires-Dist: digitalhub[ml]>=0.6.0b
 
 # SDK for DHCore

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/__init__.py

@@ -27,7 +27,7 @@ registry.register(func_kind, func_info)
 
 # Tasks
 entity_type = EntityTypes.TASKS.value
-for i in ["job", "build"]:
+for i in ["job", "build", "serve"]:
     task_kind = f"{func_kind}+{i}"
     prefix = entity_type.removesuffix("s").capitalize()
     suffix = i.capitalize()

digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/functions/models.py

@@ -0,0 +1,101 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+from digitalhub_core.entities.functions.models import SourceCodeParams, SourceCodeStruct
+from digitalhub_core.utils.exceptions import EntityError
+from digitalhub_core.utils.generic_utils import encode_source, encode_string
+from digitalhub_core.utils.uri_utils import map_uri_scheme
+
+
+class SourceCodeStructPython(SourceCodeStruct):
+    """
+    Source code struct for python.
+    """
+
+    def __init__(
+        self,
+        source: str | None = None,
+        handler: str | None = None,
+        code: str | None = None,
+        base64: str | None = None,
+        lang: str | None = None,
+        init_function: str | None = None,
+    ) -> None:
+        super().__init__(
+            source=source,
+            handler=handler,
+            code=code,
+            base64=base64,
+            lang=lang,
+        )
+        self.init_function = init_function
+
+    @staticmethod
+    def source_check(source: dict) -> dict:
+        """
+        Check source. Overrides SourceCodeStruct.source_check.
+
+        Parameters
+        ----------
+        source : dict
+            Source.
+
+        Returns
+        -------
+        dict
+            Checked source.
+        """
+        # Source check
+        source_path = source.get("source")
+        code = source.get("code")
+        base64 = source.get("base64")
+        handler = source.get("handler")
+        source["lang"] = "python"
+
+        if handler is None:
+            raise EntityError("Handler must be provided.")
+
+        if source_path is None and code is None and base64 is None:
+            raise EntityError("Source must be provided.")
+
+        if base64 is not None:
+            return source
+
+        if code is not None:
+            source["base64"] = encode_string(code)
+            return source
+
+        if source_path is not None:
+            if map_uri_scheme(source_path) == "local":
+                if not (Path(source_path).suffix == ".py" and Path(source_path).is_file()):
+                    raise EntityError("Source is not a valid python file.")
+                source["base64"] = encode_source(source_path)
+            else:
+                if handler is None:
+                    raise EntityError("Handler must be provided if source is not local.")
+
+        return source
+
+    def to_dict(self) -> dict:
+        """
+        Convert to dictionary.
+
+        Returns
+        -------
+        dict
+            Dictionary representation of the object.
+        """
+        dict_ = super().to_dict()
+        if self.init_function is not None:
+            dict_["init_function"] = self.init_function
+        return dict_
+
+
+class SourceCodeParamsPython(SourceCodeParams):
+    """
+    Source code params for python.
+    """
+
+    init_function: str = None
+    """Handler for init function."""

digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/functions/spec.py

@@ -0,0 +1,108 @@
+from __future__ import annotations
+
+from typing import Literal
+
+from digitalhub_core.entities.functions.spec import FunctionParams, FunctionSpec
+from digitalhub_runtime_python.entities.functions.models import SourceCodeParamsPython, SourceCodeStructPython
+
+
+class FunctionSpecPython(FunctionSpec):
+    """
+    Specification for a Function job.
+    """
+
+    def __init__(
+        self,
+        source: dict | None = None,
+        code_src: str | None = None,
+        handler: str | None = None,
+        code: str | None = None,
+        base64: str | None = None,
+        init_function: str | None = None,
+        lang: str | None = None,
+        image: str | None = None,
+        base_image: str | None = None,
+        python_version: str | None = None,
+        requirements: list | None = None,
+    ) -> None:
+        super().__init__()
+
+        self.image = image
+        self.base_image = base_image
+        self.python_version = python_version
+        self.requirements = requirements
+
+        # Give source precedence
+        if source is not None:
+            source_dict = source
+        else:
+            source_dict = {
+                "source": code_src,
+                "handler": handler,
+                "code": code,
+                "base64": base64,
+                "lang": lang,
+                "init_function": init_function,
+            }
+
+        source_checked = self.source_check(source_dict)
+        self.source = SourceCodeStructPython(**source_checked)
+
+    @staticmethod
+    def source_check(source: dict) -> dict:
+        """
+        Check source.
+
+        Parameters
+        ----------
+        source : dict
+            Source.
+
+        Returns
+        -------
+        dict
+            Checked source.
+        """
+        return SourceCodeStructPython.source_check(source)
+
+    def show_source_code(self) -> str:
+        """
+        Show source code.
+
+        Returns
+        -------
+        str
+            Source code.
+        """
+        return self.source.show_source_code()
+
+    def to_dict(self) -> dict:
+        """
+        Override to_dict to exclude code from source.
+
+        Returns
+        -------
+        dict
+            Dictionary representation of the object.
+        """
+        dict_ = super().to_dict()
+        dict_["source"] = self.source.to_dict()
+        return dict_
+
+
+class FunctionParamsPython(FunctionParams, SourceCodeParamsPython):
+    """
+    Function python parameters model.
+    """
+
+    python_version: Literal["PYTHON3_9", "PYTHON3_10", "PYTHON3_11"]
+    "Python version"
+
+    image: str = None
+    "Image where the function will be executed"
+
+    base_image: str = None
+    "Base image used to build the image where the function will be executed"
+
+    requirements: list = None
+    "Requirements list to be installed in the image where the function will be executed"

digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/runs/spec.py

@@ -0,0 +1,78 @@
+from __future__ import annotations
+
+from digitalhub_ml.entities.runs.spec import RunParamsMl, RunSpecMl
+
+
+class RunSpecPython(RunSpecMl):
+    """Run Python specification."""
+
+    def __init__(
+        self,
+        task: str,
+        local_execution: bool = False,
+        **kwargs,
+    ) -> None:
+        super().__init__(task, local_execution)
+
+        self.source = kwargs.get("source")
+        self.image = kwargs.get("image")
+        self.base_image = kwargs.get("base_image")
+        self.python_version = kwargs.get("python_version")
+        self.requirements = kwargs.get("requirements")
+
+        self.function = kwargs.get("function")
+        self.node_selector = kwargs.get("node_selector")
+        self.volumes = kwargs.get("volumes")
+        self.resources = kwargs.get("resources")
+        self.affinity = kwargs.get("affinity")
+        self.tolerations = kwargs.get("tolerations")
+        self.env = kwargs.get("env")
+        self.secrets = kwargs.get("secrets")
+        self.backoff_limit = kwargs.get("backoff_limit")
+        self.schedule = kwargs.get("schedule")
+        self.replicas = kwargs.get("replicas")
+
+        # Task job
+
+        # Task build
+        self.instructions = kwargs.get("instructions")
+
+        self.inputs = kwargs.get("inputs")
+        self.outputs = kwargs.get("outputs")
+        self.parameters = kwargs.get("parameters")
+
+
+class RunParamsPython(RunParamsMl):
+    """Run Python parameters."""
+
+    # Function parameters
+    source: dict = None
+    image: str = None
+    base_image: str = None
+    python_version: str = None
+    requirements: list = None
+
+    # Task parameters
+    function: str = None
+    node_selector: list[dict] = None
+    volumes: list[dict] = None
+    resources: list[dict] = None
+    affinity: dict = None
+    tolerations: list[dict] = None
+    env: list[dict] = None
+    secrets: list[str] = None
+
+    # Task job
+    backoff_limit: int = None
+
+    # Task serve
+    service_type: str = None
+    replicas: int = None
+
+    # Task build
+    instructions: list[str] = None
+
+    # Run parameters
+    inputs: dict = None
+    outputs: dict = None
+    parameters: dict = None

digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/tasks/models.py

@@ -0,0 +1,12 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class CorePort(BaseModel):
+    """
+    Port mapper model.
+    """
+
+    port: int
+    target_port: int

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/entities/tasks/spec.py

@@ -1,10 +1,8 @@
-"""
-Task Python specification module.
-"""
 from __future__ import annotations
 
+from typing import Literal
+
 from digitalhub_core.entities.tasks.spec import TaskParamsK8s, TaskSpecK8s
-from digitalhub_runtime_python.entities.tasks.models import ContextRef, ContextSource
 
 
 class TaskSpecJob(TaskSpecK8s):
@@ -15,13 +13,9 @@ class TaskSpecJob(TaskSpecK8s):
         function: str,
         **kwargs,
     ) -> None:
-        """
-        Constructor.
-        """
         super().__init__(function, **kwargs)
 
         self.backoff_limit = kwargs.get("backoff_limit")
-        self.schedule = kwargs.get("schedule")
 
 
 class TaskParamsJob(TaskParamsK8s):
@@ -29,6 +23,9 @@ class TaskParamsJob(TaskParamsK8s):
     TaskParamsJob model.
     """
 
+    backoff_limit: int = None
+    """Backoff limit."""
+
 
 class TaskSpecBuild(TaskSpecK8s):
     """Task Build specification."""
@@ -36,18 +33,11 @@ class TaskSpecBuild(TaskSpecK8s):
     def __init__(
         self,
         function: str,
-        context_refs: list | None = None,
-        context_sources: list | None = None,
         instructions: list | None = None,
         **kwargs,
     ) -> None:
-        """
-        Constructor.
-        """
         super().__init__(function, **kwargs)
 
-        self.context_refs = context_refs
-        self.context_sources = context_sources
         self.instructions = instructions
 
 
@@ -56,12 +46,6 @@ class TaskParamsBuild(TaskParamsK8s):
     TaskParamsBuild model.
     """
 
-    context_refs: list[ContextRef] = None
-    """Context references."""
-
-    context_sources: list[ContextSource] = None
-    """Context sources."""
-
     instructions: list[str] = None
     """Build instructions."""
 
@@ -69,8 +53,23 @@ class TaskParamsBuild(TaskParamsK8s):
 class TaskSpecServe(TaskSpecK8s):
     """Task Serve specification."""
 
+    def __init__(
+        self,
+        function: str,
+        replicas: int | None = None,
+        service_type: str | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(function, **kwargs)
+
+        self.replicas = replicas
+        self.service_type = service_type
+
 
 class TaskParamsServe(TaskParamsK8s):
     """
     TaskParamsServe model.
     """
+
+    replicas: int = None
+    service_type: Literal["ClusterIP", "NodePort", "LoadBalancer"] = "NodePort"

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/entities/tasks/status.py

@@ -13,3 +13,9 @@ class TaskStatusBuild(TaskStatus):
     """
     Task Build status.
     """
+
+
+class TaskStatusServe(TaskStatus):
+    """
+    Task Serve status.
+    """

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/runtimes/kind_registry.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from digitalhub_core.runtimes.registry import KindRegistry
+from digitalhub_core.runtimes.kind_registry import KindRegistry
 
 kind_registry = KindRegistry(
     {

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/runtimes/runtime.py

@@ -1,6 +1,3 @@
-"""
-Runtime class for running Python functions.
-"""
 from __future__ import annotations
 
 import typing
@@ -10,11 +7,11 @@ from digitalhub_core.context.builder import get_context
 from digitalhub_core.runtimes.base import Runtime
 from digitalhub_core.utils.logger import LOGGER
 from digitalhub_runtime_python.utils.configuration import get_function_from_source
-from digitalhub_runtime_python.utils.inputs import get_inputs_parameters
+from digitalhub_runtime_python.utils.inputs import compose_inputs
 from digitalhub_runtime_python.utils.outputs import build_status, parse_outputs
 
 if typing.TYPE_CHECKING:
-    from digitalhub_core.runtimes.registry import KindRegistry
+    from digitalhub_core.runtimes.kind_registry import KindRegistry
 
 
 class RuntimePython(Runtime):
@@ -23,16 +20,13 @@ class RuntimePython(Runtime):
     """
 
     def __init__(self, kind_registry: KindRegistry, project: str) -> None:
-        """
-        Constructor.
-        """
         super().__init__(kind_registry, project)
         ctx = get_context(self.project)
         self.root = ctx.runtime_dir
-        self.tmp_path = ctx.tmp_dir
+        self.tmp_dir = ctx.tmp_dir
 
         self.root.mkdir(parents=True, exist_ok=True)
-        self.tmp_path.mkdir(parents=True, exist_ok=True)
+        self.tmp_dir.mkdir(parents=True, exist_ok=True)
 
     def build(self, function: dict, task: dict, run: dict) -> dict:
         """
@@ -70,16 +64,19 @@ class RuntimePython(Runtime):
         LOGGER.info("Validating task.")
         self._validate_task(run)
 
+        LOGGER.info("Validating run.")
+        self._validate_run(run)
+
         LOGGER.info("Starting task.")
         spec = run.get("spec")
         project = run.get("project")
 
-        LOGGER.info("Collecting inputs.")
-        fnc_args = self._collect_inputs(spec)
-
         LOGGER.info("Configuring execution.")
         fnc, wrapped = self._configure_execution(spec)
 
+        LOGGER.info("Composing function arguments.")
+        fnc_args = self._compose_args(fnc, spec, project)
+
         LOGGER.info("Executing run.")
         if wrapped:
             results: dict = self._execute(fnc, project, **fnc_args)
@@ -111,33 +108,26 @@ class RuntimePython(Runtime):
         """
         raise NotImplementedError
 
-    ####################
-    # Inputs
-    ####################
-
-    def _collect_inputs(self, spec: dict) -> dict:
+    @staticmethod
+    def _validate_run(run: dict) -> None:
         """
-        Collect inputs.
+        Check if run is locally allowed.
 
         Parameters
         ----------
-        spec : dict
-            Run specs.
-        project : str
-            Project name.
+        run : dict
+            Run object dictionary.
 
         Returns
         -------
-        dict
-            Parameters.
+        None
         """
-        LOGGER.info("Getting inputs.")
-        self.tmp_path.mkdir(parents=True, exist_ok=True)
-        return get_inputs_parameters(
-            spec.get("inputs", {}),
-            spec.get("parameters", {}),
-            self.tmp_path,
-        )
+        task_kind = run["spec"]["task"].split(":")[0]
+        local_execution = run["spec"]["local_execution"]
+        if task_kind != "python+job" and local_execution:
+            msg = f"Local execution not allowed for task kind {task_kind}."
+            LOGGER.exception(msg)
+            raise RuntimeError(msg)
 
     ####################
     # Configuration
@@ -162,3 +152,26 @@ class RuntimePython(Runtime):
             spec.get("source", {}),
         )
         return fnc, hasattr(fnc, "__wrapped__")
+
+    def _compose_args(self, func: Callable, spec: dict, project: str) -> dict:
+        """
+        Collect inputs.
+
+        Parameters
+        ----------
+        func : Callable
+            Function to execute.
+        spec : dict
+            Run specs.
+        project : str
+            Project name.
+
+        Returns
+        -------
+        dict
+            Parameters.
+        """
+        inputs = spec.get("inputs", {})
+        parameters = spec.get("parameters", {})
+        local_execution = spec.get("local_execution")
+        return compose_inputs(inputs, parameters, local_execution, func, project)

{digitalhub_runtime_python-0.5.0b10 → digitalhub_runtime_python-0.6.0}/digitalhub_runtime_python/utils/configuration.py

@@ -43,6 +43,35 @@ def get_function_from_source(path: Path, source_spec: dict) -> Callable:
         raise RuntimeError(msg) from e
 
 
+def get_init_function(path: Path, source_spec: dict) -> Callable:
+    """
+    Get function from source.
+
+    Parameters
+    ----------
+    path : Path
+        Path where to save the function source.
+    source_spec : dict
+        Funcrion source spec.
+
+    Returns
+    -------
+    Callable
+        Function.
+    """
+    try:
+        if "init_function" not in source_spec:
+            return
+        function_code = save_function_source(path, source_spec)
+        handler_path, _ = parse_handler(source_spec["handler"])
+        function_path = (function_code / handler_path).with_suffix(".py")
+        return import_function(function_path, source_spec["init_function"])
+    except Exception as e:
+        msg = f"Some error occurred while getting init function. Exception: {e.__class__}. Error: {e.args}"
+        LOGGER.exception(msg)
+        raise RuntimeError(msg) from e
+
+
 def parse_handler(handler: str) -> tuple:
     """
     Parse handler.

digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/utils/env.py

@@ -0,0 +1,3 @@
+import os
+
+S3_BUCKET = os.getenv("S3_BUCKET_NAME")