digitalhub-runtime-python 0.6.0__tar.gz → 0.6.0b0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/PKG-INFO +2 -2
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/__init__.py +1 -1
- digitalhub_runtime_python-0.6.0b0/digitalhub_runtime_python/entities/functions/spec.py +144 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/runs/spec.py +3 -0
- digitalhub_runtime_python-0.6.0b0/digitalhub_runtime_python/entities/tasks/models.py +31 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/tasks/spec.py +12 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/tasks/status.py +0 -6
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/runtimes/runtime.py +6 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/utils/configuration.py +0 -29
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/utils/inputs.py +12 -9
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/utils/outputs.py +7 -32
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python.egg-info/PKG-INFO +2 -2
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python.egg-info/SOURCES.txt +0 -3
- digitalhub_runtime_python-0.6.0b0/digitalhub_runtime_python.egg-info/requires.txt +1 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/pyproject.toml +3 -3
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/functions/models.py +0 -101
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/functions/spec.py +0 -108
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/entities/tasks/models.py +0 -12
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/utils/env.py +0 -3
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python/utils/nuclio_configuration.py +0 -98
- digitalhub_runtime_python-0.6.0/digitalhub_runtime_python.egg-info/requires.txt +0 -1
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/LICENSE.txt +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/README.md +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/__init__.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/functions/__init__.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/functions/status.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/runs/__init__.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/runs/status.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/entities/tasks/__init__.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/runtimes/__init__.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/runtimes/kind_registry.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python/utils/utils.py +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python.egg-info/dependency_links.txt +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/digitalhub_runtime_python.egg-info/top_level.txt +0 -0
- {digitalhub_runtime_python-0.6.0 → digitalhub_runtime_python-0.6.0b0}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: digitalhub-runtime-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.0b0
|
|
4
4
|
Summary: Python runtime for DHCore
|
|
5
5
|
Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|
|
6
6
|
License: Apache License
|
|
@@ -228,6 +228,6 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
228
228
|
Requires-Python: >=3.9
|
|
229
229
|
Description-Content-Type: text/markdown
|
|
230
230
|
License-File: LICENSE.txt
|
|
231
|
-
Requires-Dist: digitalhub[ml]
|
|
231
|
+
Requires-Dist: digitalhub[ml]<0.7,~=0.6.0b
|
|
232
232
|
|
|
233
233
|
# SDK for DHCore
|
|
@@ -27,7 +27,7 @@ registry.register(func_kind, func_info)
|
|
|
27
27
|
|
|
28
28
|
# Tasks
|
|
29
29
|
entity_type = EntityTypes.TASKS.value
|
|
30
|
-
for i in ["job", "build"
|
|
30
|
+
for i in ["job", "build"]:
|
|
31
31
|
task_kind = f"{func_kind}+{i}"
|
|
32
32
|
prefix = entity_type.removesuffix("s").capitalize()
|
|
33
33
|
suffix = i.capitalize()
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Function Conatiner specification module.
|
|
3
|
+
"""
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
from digitalhub_core.entities.functions.spec import FunctionParams, FunctionSpec, SourceCodeStruct
|
|
9
|
+
from digitalhub_core.utils.exceptions import EntityError
|
|
10
|
+
from digitalhub_core.utils.generic_utils import decode_string, encode_source, encode_string
|
|
11
|
+
from digitalhub_core.utils.uri_utils import map_uri_scheme
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class FunctionSpecPython(FunctionSpec):
|
|
15
|
+
"""
|
|
16
|
+
Specification for a Function job.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
def __init__(
|
|
20
|
+
self,
|
|
21
|
+
source: dict,
|
|
22
|
+
image: str | None = None,
|
|
23
|
+
base_image: str | None = None,
|
|
24
|
+
python_version: str | None = None,
|
|
25
|
+
requirements: list | None = None,
|
|
26
|
+
) -> None:
|
|
27
|
+
"""
|
|
28
|
+
Constructor.
|
|
29
|
+
"""
|
|
30
|
+
super().__init__()
|
|
31
|
+
|
|
32
|
+
self.image = image
|
|
33
|
+
self.base_image = base_image
|
|
34
|
+
self.python_version = python_version
|
|
35
|
+
self.requirements = requirements
|
|
36
|
+
|
|
37
|
+
source = self._source_check(source)
|
|
38
|
+
self.source = SourceCodeStruct(**source)
|
|
39
|
+
|
|
40
|
+
@staticmethod
|
|
41
|
+
def _source_check(source: dict) -> dict:
|
|
42
|
+
"""
|
|
43
|
+
Check source.
|
|
44
|
+
|
|
45
|
+
Parameters
|
|
46
|
+
----------
|
|
47
|
+
source : dict
|
|
48
|
+
Source.
|
|
49
|
+
|
|
50
|
+
Returns
|
|
51
|
+
-------
|
|
52
|
+
dict
|
|
53
|
+
Checked source.
|
|
54
|
+
"""
|
|
55
|
+
if source is None:
|
|
56
|
+
raise EntityError("Source must be provided.")
|
|
57
|
+
|
|
58
|
+
# Source check
|
|
59
|
+
source_path = source.get("source")
|
|
60
|
+
code = source.get("code")
|
|
61
|
+
base64 = source.get("base64")
|
|
62
|
+
handler = source.get("handler")
|
|
63
|
+
source["lang"] = "python"
|
|
64
|
+
|
|
65
|
+
if handler is None:
|
|
66
|
+
raise EntityError("Handler must be provided.")
|
|
67
|
+
|
|
68
|
+
if source_path is None and code is None and base64 is None:
|
|
69
|
+
raise EntityError("Source must be provided.")
|
|
70
|
+
|
|
71
|
+
if base64 is not None:
|
|
72
|
+
return source
|
|
73
|
+
|
|
74
|
+
if code is not None:
|
|
75
|
+
source["base64"] = encode_string(code)
|
|
76
|
+
return source
|
|
77
|
+
|
|
78
|
+
if source_path is not None:
|
|
79
|
+
if map_uri_scheme(source_path) == "local":
|
|
80
|
+
if not (Path(source_path).suffix == ".py" and Path(source_path).is_file()):
|
|
81
|
+
raise EntityError("Source is not a valid python file.")
|
|
82
|
+
source["base64"] = encode_source(source_path)
|
|
83
|
+
else:
|
|
84
|
+
if handler is None:
|
|
85
|
+
raise EntityError("Handler must be provided if source is not local.")
|
|
86
|
+
|
|
87
|
+
return source
|
|
88
|
+
|
|
89
|
+
def show_source_code(self) -> str:
|
|
90
|
+
"""
|
|
91
|
+
Show source code.
|
|
92
|
+
|
|
93
|
+
Returns
|
|
94
|
+
-------
|
|
95
|
+
str
|
|
96
|
+
Source code.
|
|
97
|
+
"""
|
|
98
|
+
if self.source.code is not None:
|
|
99
|
+
return str(self.source.code)
|
|
100
|
+
if self.source.base64 is not None:
|
|
101
|
+
try:
|
|
102
|
+
return decode_string(self.source.base64)
|
|
103
|
+
except Exception:
|
|
104
|
+
raise EntityError("Something got wrong during source code decoding.")
|
|
105
|
+
if (self.source.source is not None) and (map_uri_scheme(self.source.source) == "local"):
|
|
106
|
+
try:
|
|
107
|
+
return Path(self.source.source).read_text()
|
|
108
|
+
except Exception:
|
|
109
|
+
raise EntityError("Cannot access source code.")
|
|
110
|
+
return ""
|
|
111
|
+
|
|
112
|
+
def to_dict(self) -> dict:
|
|
113
|
+
"""
|
|
114
|
+
Override to_dict to exclude code from source.
|
|
115
|
+
|
|
116
|
+
Returns
|
|
117
|
+
-------
|
|
118
|
+
dict
|
|
119
|
+
Dictionary representation of the object.
|
|
120
|
+
"""
|
|
121
|
+
dict_ = super().to_dict()
|
|
122
|
+
dict_["source"] = self.source.to_dict()
|
|
123
|
+
return dict_
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class FunctionParamsPython(FunctionParams):
|
|
127
|
+
"""
|
|
128
|
+
Function python parameters model.
|
|
129
|
+
"""
|
|
130
|
+
|
|
131
|
+
source: dict
|
|
132
|
+
"Source code"
|
|
133
|
+
|
|
134
|
+
image: str = None
|
|
135
|
+
"Image"
|
|
136
|
+
|
|
137
|
+
base_image: str = None
|
|
138
|
+
"Base image"
|
|
139
|
+
|
|
140
|
+
python_version: str = None
|
|
141
|
+
"Python version"
|
|
142
|
+
|
|
143
|
+
requirements: list = None
|
|
144
|
+
"Requirements list, as used by the runtime"
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class CorePort(BaseModel):
|
|
7
|
+
"""
|
|
8
|
+
Port mapper model.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
port: int
|
|
12
|
+
target_port: int
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class ContextRef(BaseModel):
|
|
16
|
+
"""
|
|
17
|
+
ContextRef model.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
destination: str = None
|
|
21
|
+
protocol: str = None
|
|
22
|
+
source: str = None
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
class ContextSource(BaseModel):
|
|
26
|
+
"""
|
|
27
|
+
ContextSource model.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
base64: str = None
|
|
31
|
+
name: str = None
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Task Python specification module.
|
|
3
|
+
"""
|
|
1
4
|
from __future__ import annotations
|
|
2
5
|
|
|
3
6
|
from typing import Literal
|
|
@@ -13,6 +16,9 @@ class TaskSpecJob(TaskSpecK8s):
|
|
|
13
16
|
function: str,
|
|
14
17
|
**kwargs,
|
|
15
18
|
) -> None:
|
|
19
|
+
"""
|
|
20
|
+
Constructor.
|
|
21
|
+
"""
|
|
16
22
|
super().__init__(function, **kwargs)
|
|
17
23
|
|
|
18
24
|
self.backoff_limit = kwargs.get("backoff_limit")
|
|
@@ -36,6 +42,9 @@ class TaskSpecBuild(TaskSpecK8s):
|
|
|
36
42
|
instructions: list | None = None,
|
|
37
43
|
**kwargs,
|
|
38
44
|
) -> None:
|
|
45
|
+
"""
|
|
46
|
+
Constructor.
|
|
47
|
+
"""
|
|
39
48
|
super().__init__(function, **kwargs)
|
|
40
49
|
|
|
41
50
|
self.instructions = instructions
|
|
@@ -60,6 +69,9 @@ class TaskSpecServe(TaskSpecK8s):
|
|
|
60
69
|
service_type: str | None = None,
|
|
61
70
|
**kwargs,
|
|
62
71
|
) -> None:
|
|
72
|
+
"""
|
|
73
|
+
Constructor.
|
|
74
|
+
"""
|
|
63
75
|
super().__init__(function, **kwargs)
|
|
64
76
|
|
|
65
77
|
self.replicas = replicas
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Runtime class for running Python functions.
|
|
3
|
+
"""
|
|
1
4
|
from __future__ import annotations
|
|
2
5
|
|
|
3
6
|
import typing
|
|
@@ -20,6 +23,9 @@ class RuntimePython(Runtime):
|
|
|
20
23
|
"""
|
|
21
24
|
|
|
22
25
|
def __init__(self, kind_registry: KindRegistry, project: str) -> None:
|
|
26
|
+
"""
|
|
27
|
+
Constructor.
|
|
28
|
+
"""
|
|
23
29
|
super().__init__(kind_registry, project)
|
|
24
30
|
ctx = get_context(self.project)
|
|
25
31
|
self.root = ctx.runtime_dir
|
|
@@ -43,35 +43,6 @@ def get_function_from_source(path: Path, source_spec: dict) -> Callable:
|
|
|
43
43
|
raise RuntimeError(msg) from e
|
|
44
44
|
|
|
45
45
|
|
|
46
|
-
def get_init_function(path: Path, source_spec: dict) -> Callable:
|
|
47
|
-
"""
|
|
48
|
-
Get function from source.
|
|
49
|
-
|
|
50
|
-
Parameters
|
|
51
|
-
----------
|
|
52
|
-
path : Path
|
|
53
|
-
Path where to save the function source.
|
|
54
|
-
source_spec : dict
|
|
55
|
-
Funcrion source spec.
|
|
56
|
-
|
|
57
|
-
Returns
|
|
58
|
-
-------
|
|
59
|
-
Callable
|
|
60
|
-
Function.
|
|
61
|
-
"""
|
|
62
|
-
try:
|
|
63
|
-
if "init_function" not in source_spec:
|
|
64
|
-
return
|
|
65
|
-
function_code = save_function_source(path, source_spec)
|
|
66
|
-
handler_path, _ = parse_handler(source_spec["handler"])
|
|
67
|
-
function_path = (function_code / handler_path).with_suffix(".py")
|
|
68
|
-
return import_function(function_path, source_spec["init_function"])
|
|
69
|
-
except Exception as e:
|
|
70
|
-
msg = f"Some error occurred while getting init function. Exception: {e.__class__}. Error: {e.args}"
|
|
71
|
-
LOGGER.exception(msg)
|
|
72
|
-
raise RuntimeError(msg) from e
|
|
73
|
-
|
|
74
|
-
|
|
75
46
|
def parse_handler(handler: str) -> tuple:
|
|
76
47
|
"""
|
|
77
48
|
Parse handler.
|
|
@@ -6,7 +6,7 @@ from typing import Any, Callable
|
|
|
6
6
|
|
|
7
7
|
from digitalhub_core.context.builder import get_context
|
|
8
8
|
from digitalhub_core.entities.artifacts.crud import artifact_from_dict
|
|
9
|
-
from digitalhub_core.
|
|
9
|
+
from digitalhub_core.utils.generic_utils import parse_entity_key
|
|
10
10
|
from digitalhub_core.utils.logger import LOGGER
|
|
11
11
|
from digitalhub_data.entities.dataitems.crud import dataitem_from_dict
|
|
12
12
|
from digitalhub_ml.entities.entity_types import EntityTypes
|
|
@@ -115,28 +115,31 @@ def compose_inputs(
|
|
|
115
115
|
fnc_args = {**parameters, **entity_inputs}
|
|
116
116
|
|
|
117
117
|
fnc_parameters = inspect.signature(func).parameters
|
|
118
|
+
LOGGER.info(f"Function parameters: {'project' in fnc_parameters}")
|
|
118
119
|
|
|
119
120
|
_has_project = "project" in fnc_parameters
|
|
120
121
|
_has_context = "context" in fnc_parameters
|
|
121
122
|
_has_event = "event" in fnc_parameters
|
|
122
123
|
|
|
123
|
-
# Project is reserved keyword argument
|
|
124
|
-
# both in local and remote executions
|
|
125
124
|
if _has_project:
|
|
126
|
-
if _has_context
|
|
125
|
+
if _has_context:
|
|
127
126
|
fnc_args["project"] = context.project
|
|
128
127
|
elif isinstance(project, str):
|
|
129
128
|
fnc_args["project"] = get_project_(project)
|
|
130
129
|
else:
|
|
131
130
|
fnc_args["project"] = project
|
|
132
131
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if not local_execution:
|
|
136
|
-
if _has_context:
|
|
132
|
+
if _has_context:
|
|
133
|
+
if context is not None and not local_execution:
|
|
137
134
|
fnc_args["context"] = context
|
|
138
|
-
|
|
135
|
+
else:
|
|
136
|
+
raise RuntimeError("Context is not available on local execution.")
|
|
137
|
+
|
|
138
|
+
if _has_event:
|
|
139
|
+
if event is not None and not local_execution:
|
|
139
140
|
fnc_args["event"] = event
|
|
141
|
+
else:
|
|
142
|
+
raise RuntimeError("Event is not available on local execution.")
|
|
140
143
|
|
|
141
144
|
return fnc_args
|
|
142
145
|
|
|
@@ -4,16 +4,12 @@ import pickle
|
|
|
4
4
|
from typing import Any
|
|
5
5
|
|
|
6
6
|
from digitalhub_core.entities._base.status import State
|
|
7
|
-
from digitalhub_core.entities._builders.uuid import build_uuid
|
|
8
7
|
from digitalhub_core.entities.artifacts.crud import new_artifact
|
|
9
8
|
from digitalhub_core.entities.artifacts.entity import Artifact
|
|
10
9
|
from digitalhub_core.utils.logger import LOGGER
|
|
11
|
-
from digitalhub_data.entities.dataitems.crud import
|
|
10
|
+
from digitalhub_data.entities.dataitems.crud import new_dataitem
|
|
12
11
|
from digitalhub_data.entities.dataitems.entity.table import DataitemTable
|
|
13
|
-
from digitalhub_data.readers.builder import get_reader_by_object
|
|
14
12
|
from digitalhub_data.readers.registry import DATAFRAME_TYPES
|
|
15
|
-
from digitalhub_ml.entities.entity_types import EntityTypes
|
|
16
|
-
from digitalhub_runtime_python.utils.env import S3_BUCKET
|
|
17
13
|
|
|
18
14
|
|
|
19
15
|
def collect_outputs(results: Any, outputs: list[str], project_name: str) -> dict:
|
|
@@ -121,22 +117,8 @@ def build_and_load_dataitem(name: str, project_name: str, data: Any) -> Dataitem
|
|
|
121
117
|
Dataitem key.
|
|
122
118
|
"""
|
|
123
119
|
try:
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
kwargs["name"] = name
|
|
127
|
-
kwargs["kind"] = "table"
|
|
128
|
-
new_id = build_uuid()
|
|
129
|
-
kwargs["uuid"] = new_id
|
|
130
|
-
kwargs["path"] = f"s3://{S3_BUCKET}/{project_name}/{EntityTypes.DATAITEMS.value}/{new_id}/data.parquet"
|
|
131
|
-
|
|
132
|
-
di: DataitemTable = create_dataitem(**kwargs)
|
|
133
|
-
|
|
134
|
-
reader = get_reader_by_object(data)
|
|
135
|
-
di.spec.schema = reader.get_schema(data)
|
|
136
|
-
di.status.preview = reader.get_preview(data)
|
|
137
|
-
|
|
138
|
-
di.save()
|
|
139
|
-
|
|
120
|
+
path = f"s3://datalake/{project_name}/dataitems/table/{name}.parquet"
|
|
121
|
+
di: DataitemTable = new_dataitem(project=project_name, name=name, kind="table", path=path)
|
|
140
122
|
di.write_df(df=data)
|
|
141
123
|
return di
|
|
142
124
|
except Exception as e:
|
|
@@ -164,21 +146,14 @@ def build_and_load_artifact(name: str, project_name: str, data: Any) -> Artifact
|
|
|
164
146
|
Artifact key.
|
|
165
147
|
"""
|
|
166
148
|
try:
|
|
167
|
-
|
|
168
|
-
kwargs["project"] = project_name
|
|
169
|
-
kwargs["name"] = name
|
|
170
|
-
kwargs["kind"] = "artifact"
|
|
171
|
-
new_id = build_uuid()
|
|
172
|
-
kwargs["uuid"] = new_id
|
|
173
|
-
pickle_file = f"{name}.pickle"
|
|
174
|
-
kwargs["path"] = f"s3://{S3_BUCKET}/{project_name}/{EntityTypes.ARTIFACTS.value}/{new_id}/{pickle_file}"
|
|
149
|
+
path = f"s3://datalake/{project_name}/artifacts/artifact/{name}.pickle"
|
|
175
150
|
|
|
176
151
|
# Dump item to pickle
|
|
177
|
-
with open(
|
|
152
|
+
with open(f"{name}.pickle", "wb") as f:
|
|
178
153
|
f.write(pickle.dumps(data))
|
|
179
154
|
|
|
180
|
-
art = new_artifact(
|
|
181
|
-
art.upload(source=
|
|
155
|
+
art = new_artifact(project=project_name, name=name, kind="artifact", path=path)
|
|
156
|
+
art.upload(source=f"{name}.pickle")
|
|
182
157
|
return art
|
|
183
158
|
|
|
184
159
|
except Exception as e:
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: digitalhub-runtime-python
|
|
3
|
-
Version: 0.6.
|
|
3
|
+
Version: 0.6.0b0
|
|
4
4
|
Summary: Python runtime for DHCore
|
|
5
5
|
Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|
|
6
6
|
License: Apache License
|
|
@@ -228,6 +228,6 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
228
228
|
Requires-Python: >=3.9
|
|
229
229
|
Description-Content-Type: text/markdown
|
|
230
230
|
License-File: LICENSE.txt
|
|
231
|
-
Requires-Dist: digitalhub[ml]
|
|
231
|
+
Requires-Dist: digitalhub[ml]<0.7,~=0.6.0b
|
|
232
232
|
|
|
233
233
|
# SDK for DHCore
|
|
@@ -9,7 +9,6 @@ digitalhub_runtime_python.egg-info/requires.txt
|
|
|
9
9
|
digitalhub_runtime_python.egg-info/top_level.txt
|
|
10
10
|
digitalhub_runtime_python/entities/__init__.py
|
|
11
11
|
digitalhub_runtime_python/entities/functions/__init__.py
|
|
12
|
-
digitalhub_runtime_python/entities/functions/models.py
|
|
13
12
|
digitalhub_runtime_python/entities/functions/spec.py
|
|
14
13
|
digitalhub_runtime_python/entities/functions/status.py
|
|
15
14
|
digitalhub_runtime_python/entities/runs/__init__.py
|
|
@@ -23,8 +22,6 @@ digitalhub_runtime_python/runtimes/__init__.py
|
|
|
23
22
|
digitalhub_runtime_python/runtimes/kind_registry.py
|
|
24
23
|
digitalhub_runtime_python/runtimes/runtime.py
|
|
25
24
|
digitalhub_runtime_python/utils/configuration.py
|
|
26
|
-
digitalhub_runtime_python/utils/env.py
|
|
27
25
|
digitalhub_runtime_python/utils/inputs.py
|
|
28
|
-
digitalhub_runtime_python/utils/nuclio_configuration.py
|
|
29
26
|
digitalhub_runtime_python/utils/outputs.py
|
|
30
27
|
digitalhub_runtime_python/utils/utils.py
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
digitalhub[ml]<0.7,~=0.6.0b
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "digitalhub-runtime-python"
|
|
7
|
-
version = "0.6.
|
|
7
|
+
version = "0.6.0b0"
|
|
8
8
|
description = "Python runtime for DHCore"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -20,7 +20,7 @@ classifiers = [
|
|
|
20
20
|
keywords = ["data", "dataops", "kubernetes"]
|
|
21
21
|
requires-python = ">=3.9"
|
|
22
22
|
dependencies = [
|
|
23
|
-
"digitalhub[ml]
|
|
23
|
+
"digitalhub[ml]~=0.6.0b, <0.7",
|
|
24
24
|
]
|
|
25
25
|
|
|
26
26
|
[project.urls]
|
|
@@ -39,7 +39,7 @@ line-length = 120
|
|
|
39
39
|
convention = "numpy"
|
|
40
40
|
|
|
41
41
|
[tool.bumpver]
|
|
42
|
-
current_version = "0.6.
|
|
42
|
+
current_version = "0.6.0b0"
|
|
43
43
|
version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
|
|
44
44
|
commit_message = "Bump version {old_version} -> {new_version}"
|
|
45
45
|
commit = false
|
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
|
|
5
|
-
from digitalhub_core.entities.functions.models import SourceCodeParams, SourceCodeStruct
|
|
6
|
-
from digitalhub_core.utils.exceptions import EntityError
|
|
7
|
-
from digitalhub_core.utils.generic_utils import encode_source, encode_string
|
|
8
|
-
from digitalhub_core.utils.uri_utils import map_uri_scheme
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
class SourceCodeStructPython(SourceCodeStruct):
|
|
12
|
-
"""
|
|
13
|
-
Source code struct for python.
|
|
14
|
-
"""
|
|
15
|
-
|
|
16
|
-
def __init__(
|
|
17
|
-
self,
|
|
18
|
-
source: str | None = None,
|
|
19
|
-
handler: str | None = None,
|
|
20
|
-
code: str | None = None,
|
|
21
|
-
base64: str | None = None,
|
|
22
|
-
lang: str | None = None,
|
|
23
|
-
init_function: str | None = None,
|
|
24
|
-
) -> None:
|
|
25
|
-
super().__init__(
|
|
26
|
-
source=source,
|
|
27
|
-
handler=handler,
|
|
28
|
-
code=code,
|
|
29
|
-
base64=base64,
|
|
30
|
-
lang=lang,
|
|
31
|
-
)
|
|
32
|
-
self.init_function = init_function
|
|
33
|
-
|
|
34
|
-
@staticmethod
|
|
35
|
-
def source_check(source: dict) -> dict:
|
|
36
|
-
"""
|
|
37
|
-
Check source. Overrides SourceCodeStruct.source_check.
|
|
38
|
-
|
|
39
|
-
Parameters
|
|
40
|
-
----------
|
|
41
|
-
source : dict
|
|
42
|
-
Source.
|
|
43
|
-
|
|
44
|
-
Returns
|
|
45
|
-
-------
|
|
46
|
-
dict
|
|
47
|
-
Checked source.
|
|
48
|
-
"""
|
|
49
|
-
# Source check
|
|
50
|
-
source_path = source.get("source")
|
|
51
|
-
code = source.get("code")
|
|
52
|
-
base64 = source.get("base64")
|
|
53
|
-
handler = source.get("handler")
|
|
54
|
-
source["lang"] = "python"
|
|
55
|
-
|
|
56
|
-
if handler is None:
|
|
57
|
-
raise EntityError("Handler must be provided.")
|
|
58
|
-
|
|
59
|
-
if source_path is None and code is None and base64 is None:
|
|
60
|
-
raise EntityError("Source must be provided.")
|
|
61
|
-
|
|
62
|
-
if base64 is not None:
|
|
63
|
-
return source
|
|
64
|
-
|
|
65
|
-
if code is not None:
|
|
66
|
-
source["base64"] = encode_string(code)
|
|
67
|
-
return source
|
|
68
|
-
|
|
69
|
-
if source_path is not None:
|
|
70
|
-
if map_uri_scheme(source_path) == "local":
|
|
71
|
-
if not (Path(source_path).suffix == ".py" and Path(source_path).is_file()):
|
|
72
|
-
raise EntityError("Source is not a valid python file.")
|
|
73
|
-
source["base64"] = encode_source(source_path)
|
|
74
|
-
else:
|
|
75
|
-
if handler is None:
|
|
76
|
-
raise EntityError("Handler must be provided if source is not local.")
|
|
77
|
-
|
|
78
|
-
return source
|
|
79
|
-
|
|
80
|
-
def to_dict(self) -> dict:
|
|
81
|
-
"""
|
|
82
|
-
Convert to dictionary.
|
|
83
|
-
|
|
84
|
-
Returns
|
|
85
|
-
-------
|
|
86
|
-
dict
|
|
87
|
-
Dictionary representation of the object.
|
|
88
|
-
"""
|
|
89
|
-
dict_ = super().to_dict()
|
|
90
|
-
if self.init_function is not None:
|
|
91
|
-
dict_["init_function"] = self.init_function
|
|
92
|
-
return dict_
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
class SourceCodeParamsPython(SourceCodeParams):
|
|
96
|
-
"""
|
|
97
|
-
Source code params for python.
|
|
98
|
-
"""
|
|
99
|
-
|
|
100
|
-
init_function: str = None
|
|
101
|
-
"""Handler for init function."""
|
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from typing import Literal
|
|
4
|
-
|
|
5
|
-
from digitalhub_core.entities.functions.spec import FunctionParams, FunctionSpec
|
|
6
|
-
from digitalhub_runtime_python.entities.functions.models import SourceCodeParamsPython, SourceCodeStructPython
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
class FunctionSpecPython(FunctionSpec):
|
|
10
|
-
"""
|
|
11
|
-
Specification for a Function job.
|
|
12
|
-
"""
|
|
13
|
-
|
|
14
|
-
def __init__(
|
|
15
|
-
self,
|
|
16
|
-
source: dict | None = None,
|
|
17
|
-
code_src: str | None = None,
|
|
18
|
-
handler: str | None = None,
|
|
19
|
-
code: str | None = None,
|
|
20
|
-
base64: str | None = None,
|
|
21
|
-
init_function: str | None = None,
|
|
22
|
-
lang: str | None = None,
|
|
23
|
-
image: str | None = None,
|
|
24
|
-
base_image: str | None = None,
|
|
25
|
-
python_version: str | None = None,
|
|
26
|
-
requirements: list | None = None,
|
|
27
|
-
) -> None:
|
|
28
|
-
super().__init__()
|
|
29
|
-
|
|
30
|
-
self.image = image
|
|
31
|
-
self.base_image = base_image
|
|
32
|
-
self.python_version = python_version
|
|
33
|
-
self.requirements = requirements
|
|
34
|
-
|
|
35
|
-
# Give source precedence
|
|
36
|
-
if source is not None:
|
|
37
|
-
source_dict = source
|
|
38
|
-
else:
|
|
39
|
-
source_dict = {
|
|
40
|
-
"source": code_src,
|
|
41
|
-
"handler": handler,
|
|
42
|
-
"code": code,
|
|
43
|
-
"base64": base64,
|
|
44
|
-
"lang": lang,
|
|
45
|
-
"init_function": init_function,
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
source_checked = self.source_check(source_dict)
|
|
49
|
-
self.source = SourceCodeStructPython(**source_checked)
|
|
50
|
-
|
|
51
|
-
@staticmethod
|
|
52
|
-
def source_check(source: dict) -> dict:
|
|
53
|
-
"""
|
|
54
|
-
Check source.
|
|
55
|
-
|
|
56
|
-
Parameters
|
|
57
|
-
----------
|
|
58
|
-
source : dict
|
|
59
|
-
Source.
|
|
60
|
-
|
|
61
|
-
Returns
|
|
62
|
-
-------
|
|
63
|
-
dict
|
|
64
|
-
Checked source.
|
|
65
|
-
"""
|
|
66
|
-
return SourceCodeStructPython.source_check(source)
|
|
67
|
-
|
|
68
|
-
def show_source_code(self) -> str:
|
|
69
|
-
"""
|
|
70
|
-
Show source code.
|
|
71
|
-
|
|
72
|
-
Returns
|
|
73
|
-
-------
|
|
74
|
-
str
|
|
75
|
-
Source code.
|
|
76
|
-
"""
|
|
77
|
-
return self.source.show_source_code()
|
|
78
|
-
|
|
79
|
-
def to_dict(self) -> dict:
|
|
80
|
-
"""
|
|
81
|
-
Override to_dict to exclude code from source.
|
|
82
|
-
|
|
83
|
-
Returns
|
|
84
|
-
-------
|
|
85
|
-
dict
|
|
86
|
-
Dictionary representation of the object.
|
|
87
|
-
"""
|
|
88
|
-
dict_ = super().to_dict()
|
|
89
|
-
dict_["source"] = self.source.to_dict()
|
|
90
|
-
return dict_
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
class FunctionParamsPython(FunctionParams, SourceCodeParamsPython):
|
|
94
|
-
"""
|
|
95
|
-
Function python parameters model.
|
|
96
|
-
"""
|
|
97
|
-
|
|
98
|
-
python_version: Literal["PYTHON3_9", "PYTHON3_10", "PYTHON3_11"]
|
|
99
|
-
"Python version"
|
|
100
|
-
|
|
101
|
-
image: str = None
|
|
102
|
-
"Image where the function will be executed"
|
|
103
|
-
|
|
104
|
-
base_image: str = None
|
|
105
|
-
"Base image used to build the image where the function will be executed"
|
|
106
|
-
|
|
107
|
-
requirements: list = None
|
|
108
|
-
"Requirements list to be installed in the image where the function will be executed"
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
from pathlib import Path
|
|
4
|
-
from typing import Callable, Union
|
|
5
|
-
|
|
6
|
-
from digitalhub_core.utils.uri_utils import map_uri_scheme
|
|
7
|
-
from digitalhub_runtime_python.utils.configuration import import_function
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
def parse_handler(handler: str) -> tuple[Path, str]:
|
|
11
|
-
"""
|
|
12
|
-
Parse handler.
|
|
13
|
-
|
|
14
|
-
Parameters
|
|
15
|
-
----------
|
|
16
|
-
handler : str
|
|
17
|
-
Function handler
|
|
18
|
-
|
|
19
|
-
Returns
|
|
20
|
-
-------
|
|
21
|
-
tuple[Path, str]
|
|
22
|
-
Function handler.
|
|
23
|
-
"""
|
|
24
|
-
parsed = handler.split(":")
|
|
25
|
-
if len(parsed) == 1:
|
|
26
|
-
return None, parsed[0]
|
|
27
|
-
return Path(*parsed[0].split(".")), parsed[1]
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
def get_function_source(source_spec: dict) -> Path:
|
|
31
|
-
"""
|
|
32
|
-
Get function source.
|
|
33
|
-
|
|
34
|
-
Parameters
|
|
35
|
-
----------
|
|
36
|
-
source : dict
|
|
37
|
-
Function source.
|
|
38
|
-
|
|
39
|
-
Returns
|
|
40
|
-
-------
|
|
41
|
-
Path
|
|
42
|
-
Path to function source.
|
|
43
|
-
"""
|
|
44
|
-
path = Path("/shared")
|
|
45
|
-
|
|
46
|
-
# Get relevant information
|
|
47
|
-
base64 = source_spec.get("base64")
|
|
48
|
-
source = source_spec.get("source", "main.py")
|
|
49
|
-
handler = source_spec.get("handler")
|
|
50
|
-
|
|
51
|
-
handler_path, _ = parse_handler(handler)
|
|
52
|
-
|
|
53
|
-
# Get scheme in source
|
|
54
|
-
scheme = map_uri_scheme(source)
|
|
55
|
-
|
|
56
|
-
# Check base64. If it is set, it means
|
|
57
|
-
# that the source comes from a local file
|
|
58
|
-
if base64 is not None:
|
|
59
|
-
if scheme == "local":
|
|
60
|
-
if handler_path is not None:
|
|
61
|
-
return path / handler_path / Path(source)
|
|
62
|
-
return path / Path(source)
|
|
63
|
-
raise RuntimeError("Source is not a local file.")
|
|
64
|
-
|
|
65
|
-
if handler_path is not None:
|
|
66
|
-
return path / handler_path.with_suffix(".py")
|
|
67
|
-
raise RuntimeError("Must provide handler path in handler in form <root>.<dir>.<module>:<function_name>.")
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
def import_function_and_init(source: dict) -> tuple[Callable, Union[Callable, None]]:
|
|
71
|
-
"""
|
|
72
|
-
Import function from source.
|
|
73
|
-
|
|
74
|
-
Parameters
|
|
75
|
-
----------
|
|
76
|
-
source : dict
|
|
77
|
-
Function source.
|
|
78
|
-
|
|
79
|
-
Returns
|
|
80
|
-
-------
|
|
81
|
-
tuple
|
|
82
|
-
Function and init function.
|
|
83
|
-
"""
|
|
84
|
-
|
|
85
|
-
# Get function source
|
|
86
|
-
function_path = get_function_source(source)
|
|
87
|
-
_, handler_name = parse_handler(source.get("handler"))
|
|
88
|
-
|
|
89
|
-
# Import function
|
|
90
|
-
fnc = import_function(function_path, handler_name)
|
|
91
|
-
|
|
92
|
-
# Get init function
|
|
93
|
-
init_handler = source.get("init_function")
|
|
94
|
-
init_fnc = None
|
|
95
|
-
if init_handler is not None:
|
|
96
|
-
init_fnc = import_function(function_path, init_handler)
|
|
97
|
-
|
|
98
|
-
return fnc, init_fnc
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
digitalhub[ml]>=0.6.0b
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|