digitalhub-runtime-python 0.15.0b18__tar.gz → 0.15.1__tar.gz

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalhub-runtime-python
-Version: 0.15.0b18
+Version: 0.15.1
 Summary: Python runtime for DHCore
 Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk-runtime-python
 Author-email: Fondazione Bruno Kessler <digitalhub@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
@@ -215,14 +215,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.10
-Requires-Dist: digitalhub[full]<0.16,>=0.15.0b
+Requires-Dist: digitalhub[full]<0.16,>=0.15.1
 Requires-Dist: msgpack
 Requires-Dist: nuclio-sdk
 Requires-Dist: pip
 Requires-Dist: pyyaml
 Requires-Dist: tomli; python_version < '3.11'
 Provides-Extra: dev
-Requires-Dist: digitalhub[dev]<0.16,>=0.15.0b; extra == 'dev'
+Requires-Dist: digitalhub[dev]<0.16,>=0.15.1; extra == 'dev'
 Description-Content-Type: text/markdown
 
 # DigitalHub SDK Runtime Python

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/digitalhub_runtime_python/entities/function/python/utils.py

@@ -147,8 +147,7 @@ def source_post_check(exec: FunctionPython) -> FunctionPython:
 
         # If source is a folder, zip it and upload it
         if not path_src.is_file():
-            archive_path = path_src.parent / f"{path_src.name}.zip"
-            create_archive(path_src, archive_path)
+            archive_path = create_archive(path_src)
             dst = build_zip_path(exec, archive_path.name)
             get_store(dst).upload(str(archive_path), dst)
             exec.spec.source["source"] = dst

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/digitalhub_runtime_python/runtimes/runtime.py

@@ -4,7 +4,7 @@
 
 from __future__ import annotations
 
-from typing import Callable
+from typing import Any, Callable
 
 from digitalhub.context.api import get_context
 from digitalhub.runtimes._base import Runtime
@@ -12,7 +12,7 @@ from digitalhub.utils.logger.logger import get_logger
 
 from digitalhub_runtime_python.utils.configuration import import_function_from_source
 from digitalhub_runtime_python.utils.inputs import compose_inputs
-from digitalhub_runtime_python.utils.outputs import build_new_status, parse_outputs
+from digitalhub_runtime_python.utils.outputs import build_new_status, collect_outputs
 
 logger = get_logger(__file__)
 
@@ -63,7 +63,8 @@ class RuntimePythonJob(Runtime):
         else:
             exec_result = self._execute(fnc, **fnc_args)
             logger.info("Collecting outputs.")
-            results = parse_outputs(exec_result, project, run_key)
+            named_outputs = self._get_named_outputs(exec_result)
+            results = collect_outputs(exec_result, named_outputs, project, run_key)
 
         status = build_new_status(project, results)
 
@@ -118,6 +119,24 @@ class RuntimePythonJob(Runtime):
         local_execution = spec.get("local_execution")
         return compose_inputs(inputs, parameters, local_execution, func, project)
 
+    def _get_named_outputs(self, exec_result: Any) -> list[str]:
+        """
+        Get named outputs.
+
+        Parameters
+        ----------
+        exec_result : Any
+            Function execution result.
+
+        Returns
+        -------
+        list[str]
+            List of named outputs.
+        """
+        if isinstance(exec_result, list):
+            return [f"output_{idx}" for idx in range(len(exec_result))]
+        return []
+
 
 class RuntimeOpeninference(RuntimePython):
     """

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/digitalhub_runtime_python/utils/configuration.py

@@ -4,7 +4,6 @@
 
 from __future__ import annotations
 
-from importlib import import_module
 from pathlib import Path
 from typing import Callable
 
@@ -28,6 +27,7 @@ from digitalhub.utils.uri_utils import (
 
 logger = get_logger(__file__)
 DEFAULT_PY_FILE = "main.py"
+DEFAULT_SERVERLESS_PATH = Path("/shared")
 
 
 def _parse_handler(handler: str) -> tuple[Path, str]:
@@ -278,7 +278,7 @@ def save_function_source(path: Path, source_spec: dict) -> Path:
     raise RuntimeError(f"Unable to collect source from: {source}")
 
 
-def _get_function_to_import(handler: str, source_code: str | None) -> tuple[str, str]:
+def _get_function_to_import(handler: str, source_code: str | None, root_path: Path) -> tuple[Path, str]:
     """
     Get function name from handler string.
 
@@ -291,8 +291,8 @@ def _get_function_to_import(handler: str, source_code: str | None) -> tuple[str,
 
     Returns
     -------
-    tuple
-        Function module and function name.
+    tuple[Path, str]
+        Function path and function name.
     """
     # If handler is not specified, assume function module is in main
     try:
@@ -304,11 +304,16 @@ def _get_function_to_import(handler: str, source_code: str | None) -> tuple[str,
         else:
             function_module = DEFAULT_PY_FILE.removesuffix(".py")
 
-    return function_module, function_name
+    function_path = (root_path / Path(function_module.replace(".", "/") + ".py")).resolve()
+    if not function_path.exists():
+        raise RuntimeError(f"Function module {function_module} not found at path {function_path}.")
+
+    return function_path, function_name
 
 
 def import_function_and_init_from_source(
     source_spec: dict,
+    root_path: Path | None = None,
 ) -> tuple[Callable, Callable | None]:
     """
     Import main function and optional init function from source.
@@ -317,25 +322,29 @@ def import_function_and_init_from_source(
     ----------
     source_spec : dict
         Function source spec.
+    root_path : Path | None
+        Root path to use if function path is not specified.
 
     Returns
     -------
     tuple
         Main function and optional init function.
     """
+    if root_path is None:
+        root_path = DEFAULT_SERVERLESS_PATH
+
     handler = source_spec.get("handler")
     source_code = source_spec.get("source")
 
-    function_module, function_name = _get_function_to_import(handler, source_code)
+    function_path, function_name = _get_function_to_import(handler, source_code, root_path)
 
     # Import module and get main function
-    module = import_module(function_module)
-    main_function = getattr(module, function_name)
+    main_function = _import_function_from_path(function_path, function_name)
 
     # Import init function if specified
     init_function: Callable | None = None
     init_handler: str | None = source_spec.get("init_function")
     if init_handler is not None:
-        init_function = getattr(module, init_handler)
+        init_function = _import_function_from_path(function_path, init_handler)
 
     return main_function, init_function

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/digitalhub_runtime_python/utils/outputs.py

@@ -6,8 +6,18 @@ from __future__ import annotations
 
 import pickle
 import typing
+from dataclasses import dataclass
 from typing import Any
 
+from digitalhub import (
+    log_croissant,
+    log_generic_artifact,
+    log_generic_dataitem,
+    log_generic_model,
+    log_mlflow,
+    log_sklearn,
+    log_table,
+)
 from digitalhub.context.api import get_context
 from digitalhub.entities._commons.enums import EntityKinds, Relationship, State
 from digitalhub.entities.artifact._base.entity import Artifact
@@ -25,7 +35,122 @@ if typing.TYPE_CHECKING:
 logger = get_logger(__file__)
 
 
-def collect_outputs(results: Any, outputs: list[str], project_name: str, run_key: str) -> dict:
+mapped_logger = {
+    EntityKinds.DATAITEM_DATAITEM.value: log_generic_dataitem,
+    EntityKinds.DATAITEM_TABLE.value: log_table,
+    EntityKinds.DATAITEM_CROISSANT.value: log_croissant,
+    EntityKinds.MODEL_MLFLOW.value: log_mlflow,
+    EntityKinds.MODEL_SKLEARN.value: log_sklearn,
+    EntityKinds.MODEL_MODEL.value: log_generic_model,
+    EntityKinds.ARTIFACT_ARTIFACT.value: log_generic_artifact,
+}
+
+
+@dataclass
+class OutputStruct:
+    """
+    OutputStruct is a class that represents the output of a function.
+    """
+
+    name: str | None
+    kind: str | None
+    kwargs: dict | None = None
+    log: bool = False
+
+
+def _build_default_output_names(outputs_len: int, results_len: int) -> list[str]:
+    return [f"output_{idx}" for idx in range(outputs_len, results_len)]
+
+
+def _parse_UDF_outputs(outputs: list[str] | list[dict[str, Any]], results_len: int) -> list[OutputStruct]:
+    """
+    Parse UDF outputs.
+
+    Parameters
+    ----------
+    outputs : list[str] | list[dict[str, str]]
+        List of named outputs to collect.
+    results_len : int
+        Number of results produced by the function.
+
+    Returns
+    -------
+    list[OutputStruct]
+        List of OutputStruct objects.
+    """
+    normalized_outputs = list(outputs[:results_len])
+
+    if len(normalized_outputs) != results_len:
+        # Set default names for outputs if the number of outputs is different from the number of results.
+        # If the number of outputs is less than the number of results, the missing outputs will be named as output_{idx}.
+        # If the number of outputs is greater than the number of results, the extra outputs will be ignored.
+        logger.warning(
+            f"Number of outputs ({len(outputs)}) is different from number of results ({results_len}). "
+            f"Default names will be used for outputs.",
+        )
+        normalized_outputs.extend(_build_default_output_names(len(normalized_outputs), results_len))
+
+    parsed_outputs = []
+    for idx, item in enumerate(normalized_outputs):
+        if isinstance(item, str):
+            parsed_outputs.append(OutputStruct(name=item, kind=None))
+            continue
+
+        if not isinstance(item, dict):
+            raise ValueError(f"Invalid output format: {item}")
+
+        name = item.get("name") or f"output_{idx}"
+        kind = item.get("kind")
+        if kind is None:
+            raise ValueError(f"Missing kind for logged output: {name}")
+
+        if kind not in mapped_logger:
+            raise ValueError(f"Unsupported output kind: {kind}")
+
+        kwargs = dict(item.get("spec_kwargs") or {})
+        parsed_outputs.append(OutputStruct(name=name, kind=kind, kwargs=kwargs, log=True))
+    return parsed_outputs
+
+
+def _log_mapped_output(project_name: str, item: Any, output: OutputStruct) -> None:
+    logger.info(f"Logging output {output.name}.")
+    spec_kwargs = dict(output.kwargs or {})
+    spec_kwargs["source"] = item
+    mapped_logger[output.kind](project_name, output.name, **spec_kwargs)
+
+
+def _save_existing_object(obj: Dataitem | Artifact | Model, run_key: str) -> Dataitem | Artifact | Model:
+    dest = run_key + ":" + run_key.split("/")[-1]
+    obj.add_relationship(relation=Relationship.PRODUCEDBY.value, dest=dest)
+
+    try:
+        obj.save(update=True)
+    except EntityNotExistsError:
+        obj.save()
+
+    return obj
+
+
+def _materialize_output(name: str, item: Any, project_name: str, run_key: str) -> Any:
+    if isinstance(item, (str, int, float, bool, bytes)):
+        return item
+
+    if isinstance(item, (Dataitem, Artifact, Model)):
+        return _save_existing_object(item, run_key)
+
+    for df_class in get_supported_dataframes():
+        if isinstance(item, df_class):
+            return _log_dataitem(name, project_name, item)
+
+    return _log_artifact(name, project_name, item)
+
+
+def collect_outputs(
+    results: Any,
+    outputs: list[str] | list[dict[str, Any]],
+    project_name: str,
+    run_key: str,
+) -> dict:
     """
     Collect outputs. Use the produced results directly.
 
@@ -33,7 +158,7 @@ def collect_outputs(results: Any, outputs: list[str], project_name: str, run_key
     ----------
     results : Any
         Function outputs.
-    outputs : list[str]
+    outputs : list[OutputStruct]
         List of named outputs to collect.
     project_name : str
         Project name.
@@ -48,42 +173,16 @@ def collect_outputs(results: Any, outputs: list[str], project_name: str, run_key
     objects = {}
     results = listify_results(results)
 
+    parsed_outputs = _parse_UDF_outputs(outputs, len(results))
+
     for idx, item in enumerate(results):
-        # Get mapping of outputs, if not found, create a new one
-        try:
-            name = outputs[idx]
-        except IndexError:
-            name = f"output_{idx}"
-
-        if isinstance(item, (str, int, float, bool, bytes)):
-            objects[name] = item
-
-        else:
-            # Recieve a digitalhub object
-            if isinstance(item, (Dataitem, Artifact, Model)):
-                obj = item
-
-                # Add relationship to object, update it
-                dest = run_key + ":" + run_key.split("/")[-1]
-                obj.add_relationship(relation=Relationship.PRODUCEDBY.value, dest=dest)
-
-                # Save or update object
-                try:
-                    obj.save(update=True)
-                except EntityNotExistsError:
-                    obj.save()
+        output = parsed_outputs[idx]
 
-            else:
-                # Recieve a dataframe object
-                for df_class in get_supported_dataframes():
-                    if isinstance(item, df_class):
-                        obj = _log_dataitem(name, project_name, item)
-                        break
-                # Recieve a generic python object
-                else:
-                    obj = _log_artifact(name, project_name, item)
-
-            objects[name] = obj
+        if output.log:
+            _log_mapped_output(project_name, item, output)
+            continue
+
+        objects[output.name] = _materialize_output(output.name, item, project_name, run_key)
 
     return objects
 

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/digitalhub_runtime_python/utils/utils.py

@@ -10,7 +10,7 @@ from typing import Callable
 from digitalhub_runtime_python.utils.outputs import collect_outputs
 
 
-def handler(outputs: list[str] | None = None) -> Callable:
+def handler(outputs: list[str] | list[dict[str, str]] | None = None) -> Callable:
     """
     Decorator that handles the outputs of the function.
 

{digitalhub_runtime_python-0.15.0b18 → digitalhub_runtime_python-0.15.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "digitalhub-runtime-python"
-version = "0.15.0b18"
+version = "0.15.1"
 description = "Python runtime for DHCore"
 readme = "README.md"
 authors = [
@@ -23,7 +23,7 @@ classifiers = [
 keywords = ["data", "dataops", "kubernetes"]
 requires-python = ">=3.10, <3.15"
 dependencies = [
-"digitalhub[full]>=0.15.0b, <0.16",
+"digitalhub[full]>=0.15.1, <0.16",
 "nuclio-sdk",
 "msgpack",
 "pip",
@@ -33,7 +33,7 @@ dependencies = [
 
 [project.optional-dependencies]
 dev = [
-    "digitalhub[dev]>=0.15.0b, <0.16"
+    "digitalhub[dev]>=0.15.1, <0.16"
 ]
 
 [project.urls]
@@ -54,7 +54,7 @@ docstring-code-line-length = 120
 convention = "numpy"
 
 [tool.bumpver]
-current_version = "0.15.0b18"
+current_version = "0.15.1"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
 commit_message  = "Bump version {old_version} -> {new_version}"
 commit          = false