digitalhub 0.2.30__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of digitalhub might be problematic. Click here for more details.
- {digitalhub-0.2.30 → digitalhub-0.3.1}/PKG-INFO +8 -8
- {digitalhub-0.2.30 → digitalhub-0.3.1}/digitalhub/__init__.py +19 -7
- {digitalhub-0.2.30 → digitalhub-0.3.1}/digitalhub.egg-info/PKG-INFO +8 -8
- digitalhub-0.3.1/digitalhub.egg-info/SOURCES.txt +17 -0
- digitalhub-0.3.1/digitalhub.egg-info/requires.txt +15 -0
- {digitalhub-0.2.30 → digitalhub-0.3.1}/digitalhub.egg-info/top_level.txt +1 -0
- {digitalhub-0.2.30 → digitalhub-0.3.1}/pyproject.toml +9 -9
- digitalhub-0.3.1/test/test_crud_artifacts.py +91 -0
- digitalhub-0.3.1/test/test_crud_dataitems.py +90 -0
- digitalhub-0.3.1/test/test_crud_functions.py +109 -0
- digitalhub-0.3.1/test/test_crud_runs.py +86 -0
- digitalhub-0.3.1/test/test_crud_tasks.py +81 -0
- digitalhub-0.3.1/test/test_imports.py +66 -0
- digitalhub-0.3.1/test/testkfp.py +34 -0
- digitalhub-0.3.1/test/testkfp_pipeline.py +22 -0
- digitalhub-0.2.30/digitalhub.egg-info/SOURCES.txt +0 -9
- digitalhub-0.2.30/digitalhub.egg-info/requires.txt +0 -15
- {digitalhub-0.2.30 → digitalhub-0.3.1}/LICENSE.txt +0 -0
- {digitalhub-0.2.30 → digitalhub-0.3.1}/README.md +0 -0
- {digitalhub-0.2.30 → digitalhub-0.3.1}/digitalhub.egg-info/dependency_links.txt +0 -0
- {digitalhub-0.2.30 → digitalhub-0.3.1}/setup.cfg +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: digitalhub
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Python SDK for DigitalHub
|
|
5
5
|
Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|
|
6
6
|
License: Apache License
|
|
@@ -228,17 +228,17 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
228
228
|
Requires-Python: >=3.9
|
|
229
229
|
Description-Content-Type: text/markdown
|
|
230
230
|
License-File: LICENSE.txt
|
|
231
|
-
Requires-Dist: digitalhub-core~=0.
|
|
231
|
+
Requires-Dist: digitalhub-core~=0.3
|
|
232
232
|
Provides-Extra: all
|
|
233
|
-
Requires-Dist: digitalhub-data~=0.
|
|
234
|
-
Requires-Dist: digitalhub-ml~=0.
|
|
235
|
-
Requires-Dist: digitalhub-ai~=0.
|
|
233
|
+
Requires-Dist: digitalhub-data~=0.3; extra == "all"
|
|
234
|
+
Requires-Dist: digitalhub-ml~=0.3; extra == "all"
|
|
235
|
+
Requires-Dist: digitalhub-ai~=0.3; extra == "all"
|
|
236
236
|
Provides-Extra: ml
|
|
237
|
-
Requires-Dist: digitalhub-ml~=0.
|
|
237
|
+
Requires-Dist: digitalhub-ml~=0.3; extra == "ml"
|
|
238
238
|
Provides-Extra: ai
|
|
239
|
-
Requires-Dist: digitalhub-ai~=0.
|
|
239
|
+
Requires-Dist: digitalhub-ai~=0.3; extra == "ai"
|
|
240
240
|
Provides-Extra: data
|
|
241
|
-
Requires-Dist: digitalhub-data~=0.
|
|
241
|
+
Requires-Dist: digitalhub-data~=0.3; extra == "data"
|
|
242
242
|
|
|
243
243
|
# Digitalhub Library
|
|
244
244
|
|
|
@@ -4,28 +4,30 @@ from digitalhub_core import (
|
|
|
4
4
|
delete_project,
|
|
5
5
|
delete_run,
|
|
6
6
|
delete_secret,
|
|
7
|
-
delete_service,
|
|
8
7
|
delete_task,
|
|
9
8
|
delete_workflow,
|
|
10
9
|
get_artifact,
|
|
11
10
|
get_function,
|
|
12
11
|
get_run,
|
|
13
12
|
get_secret,
|
|
14
|
-
get_service,
|
|
15
13
|
get_task,
|
|
16
14
|
get_workflow,
|
|
17
15
|
import_artifact,
|
|
18
16
|
import_function,
|
|
19
17
|
import_run,
|
|
20
18
|
import_secret,
|
|
21
|
-
import_service,
|
|
22
19
|
import_task,
|
|
23
20
|
import_workflow,
|
|
21
|
+
list_artifacts,
|
|
22
|
+
list_functions,
|
|
23
|
+
list_runs,
|
|
24
|
+
list_secrets,
|
|
25
|
+
list_tasks,
|
|
26
|
+
list_workflows,
|
|
24
27
|
new_artifact,
|
|
25
28
|
new_function,
|
|
26
29
|
new_run,
|
|
27
30
|
new_secret,
|
|
28
|
-
new_service,
|
|
29
31
|
new_task,
|
|
30
32
|
new_workflow,
|
|
31
33
|
set_dhub_env,
|
|
@@ -35,7 +37,6 @@ from digitalhub_core import (
|
|
|
35
37
|
update_project,
|
|
36
38
|
update_run,
|
|
37
39
|
update_secret,
|
|
38
|
-
update_service,
|
|
39
40
|
update_task,
|
|
40
41
|
update_workflow,
|
|
41
42
|
)
|
|
@@ -44,7 +45,14 @@ _PROJECT_IMPORTED = False
|
|
|
44
45
|
|
|
45
46
|
if not _PROJECT_IMPORTED:
|
|
46
47
|
try:
|
|
47
|
-
from digitalhub_data import
|
|
48
|
+
from digitalhub_data import (
|
|
49
|
+
delete_dataitem,
|
|
50
|
+
get_dataitem,
|
|
51
|
+
import_dataitem,
|
|
52
|
+
list_dataitems,
|
|
53
|
+
new_dataitem,
|
|
54
|
+
update_dataitem,
|
|
55
|
+
)
|
|
48
56
|
from digitalhub_ml import (
|
|
49
57
|
delete_model,
|
|
50
58
|
get_model,
|
|
@@ -52,6 +60,8 @@ if not _PROJECT_IMPORTED:
|
|
|
52
60
|
get_project,
|
|
53
61
|
import_model,
|
|
54
62
|
import_project,
|
|
63
|
+
list_models,
|
|
64
|
+
load_project,
|
|
55
65
|
new_model,
|
|
56
66
|
new_project,
|
|
57
67
|
update_model,
|
|
@@ -70,6 +80,8 @@ if not _PROJECT_IMPORTED:
|
|
|
70
80
|
get_project,
|
|
71
81
|
import_dataitem,
|
|
72
82
|
import_project,
|
|
83
|
+
list_dataitems,
|
|
84
|
+
load_project,
|
|
73
85
|
new_dataitem,
|
|
74
86
|
new_project,
|
|
75
87
|
update_dataitem,
|
|
@@ -80,4 +92,4 @@ if not _PROJECT_IMPORTED:
|
|
|
80
92
|
...
|
|
81
93
|
|
|
82
94
|
if not _PROJECT_IMPORTED:
|
|
83
|
-
from digitalhub_core import get_or_create_project, get_project, import_project, new_project
|
|
95
|
+
from digitalhub_core import get_or_create_project, get_project, import_project, load_project, new_project
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: digitalhub
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Python SDK for DigitalHub
|
|
5
5
|
Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
|
|
6
6
|
License: Apache License
|
|
@@ -228,17 +228,17 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
228
228
|
Requires-Python: >=3.9
|
|
229
229
|
Description-Content-Type: text/markdown
|
|
230
230
|
License-File: LICENSE.txt
|
|
231
|
-
Requires-Dist: digitalhub-core~=0.
|
|
231
|
+
Requires-Dist: digitalhub-core~=0.3
|
|
232
232
|
Provides-Extra: all
|
|
233
|
-
Requires-Dist: digitalhub-data~=0.
|
|
234
|
-
Requires-Dist: digitalhub-ml~=0.
|
|
235
|
-
Requires-Dist: digitalhub-ai~=0.
|
|
233
|
+
Requires-Dist: digitalhub-data~=0.3; extra == "all"
|
|
234
|
+
Requires-Dist: digitalhub-ml~=0.3; extra == "all"
|
|
235
|
+
Requires-Dist: digitalhub-ai~=0.3; extra == "all"
|
|
236
236
|
Provides-Extra: ml
|
|
237
|
-
Requires-Dist: digitalhub-ml~=0.
|
|
237
|
+
Requires-Dist: digitalhub-ml~=0.3; extra == "ml"
|
|
238
238
|
Provides-Extra: ai
|
|
239
|
-
Requires-Dist: digitalhub-ai~=0.
|
|
239
|
+
Requires-Dist: digitalhub-ai~=0.3; extra == "ai"
|
|
240
240
|
Provides-Extra: data
|
|
241
|
-
Requires-Dist: digitalhub-data~=0.
|
|
241
|
+
Requires-Dist: digitalhub-data~=0.3; extra == "data"
|
|
242
242
|
|
|
243
243
|
# Digitalhub Library
|
|
244
244
|
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
LICENSE.txt
|
|
2
|
+
README.md
|
|
3
|
+
pyproject.toml
|
|
4
|
+
digitalhub/__init__.py
|
|
5
|
+
digitalhub.egg-info/PKG-INFO
|
|
6
|
+
digitalhub.egg-info/SOURCES.txt
|
|
7
|
+
digitalhub.egg-info/dependency_links.txt
|
|
8
|
+
digitalhub.egg-info/requires.txt
|
|
9
|
+
digitalhub.egg-info/top_level.txt
|
|
10
|
+
test/test_crud_artifacts.py
|
|
11
|
+
test/test_crud_dataitems.py
|
|
12
|
+
test/test_crud_functions.py
|
|
13
|
+
test/test_crud_runs.py
|
|
14
|
+
test/test_crud_tasks.py
|
|
15
|
+
test/test_imports.py
|
|
16
|
+
test/testkfp.py
|
|
17
|
+
test/testkfp_pipeline.py
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "digitalhub"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.3.1"
|
|
8
8
|
description = "Python SDK for DigitalHub"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
authors = [
|
|
@@ -20,18 +20,18 @@ classifiers = [
|
|
|
20
20
|
keywords = ["data", "dataops", "kubernetes"]
|
|
21
21
|
requires-python = ">=3.9"
|
|
22
22
|
dependencies = [
|
|
23
|
-
"digitalhub-core~=0.
|
|
23
|
+
"digitalhub-core~=0.3",
|
|
24
24
|
]
|
|
25
25
|
|
|
26
26
|
[project.optional-dependencies]
|
|
27
27
|
all = [
|
|
28
|
-
"digitalhub-data~=0.
|
|
29
|
-
"digitalhub-ml~=0.
|
|
30
|
-
"digitalhub-ai~=0.
|
|
28
|
+
"digitalhub-data~=0.3",
|
|
29
|
+
"digitalhub-ml~=0.3",
|
|
30
|
+
"digitalhub-ai~=0.3",
|
|
31
31
|
]
|
|
32
|
-
ml = ["digitalhub-ml~=0.
|
|
33
|
-
ai = ["digitalhub-ai~=0.
|
|
34
|
-
data = ["digitalhub-data~=0.
|
|
32
|
+
ml = ["digitalhub-ml~=0.3"]
|
|
33
|
+
ai = ["digitalhub-ai~=0.3"]
|
|
34
|
+
data = ["digitalhub-data~=0.3"]
|
|
35
35
|
|
|
36
36
|
[project.urls]
|
|
37
37
|
Homepage = "https://github.com/scc-digitalhub/digitalhub-sdk"
|
|
@@ -52,7 +52,7 @@ line-length = 120
|
|
|
52
52
|
convention = "numpy"
|
|
53
53
|
|
|
54
54
|
[tool.bumpver]
|
|
55
|
-
current_version = "0.
|
|
55
|
+
current_version = "0.3.1"
|
|
56
56
|
version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"
|
|
57
57
|
commit_message = "Bump version {old_version} -> {new_version}"
|
|
58
58
|
commit = false
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
import dotenv
|
|
4
|
+
from digitalhub_core.entities.artifacts.entity import Artifact
|
|
5
|
+
|
|
6
|
+
import digitalhub
|
|
7
|
+
|
|
8
|
+
dotenv.load_dotenv()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
names = ["test1", "test2", "test3", "test4"]
|
|
12
|
+
uuids = [
|
|
13
|
+
"12a01efa-o44f-4991-b153-9a3c358b7bb0",
|
|
14
|
+
"8e367f52-25bb-4df1-b9c9-a58045b377a0",
|
|
15
|
+
"1678f9ab-a2e0-48ff-870a-2384o3fa1334",
|
|
16
|
+
"adb746dd-4e81-4ff8-82de-4916624o17dc",
|
|
17
|
+
]
|
|
18
|
+
paths = ["./data/test.csv", "s3://bucket/key.csv", "sql://database/schema/table", "https://url.com/file.csv"]
|
|
19
|
+
kind = ["artifact", "artifact", "artifact", "artifact"]
|
|
20
|
+
|
|
21
|
+
dicts = []
|
|
22
|
+
for i in range(len(names)):
|
|
23
|
+
dicts.append({"name": names[i], "uuid": uuids[i], "path": paths[i], "kind": kind[i]})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
digitalhub.delete_project("test")
|
|
27
|
+
|
|
28
|
+
p = digitalhub.get_or_create_project("test")
|
|
29
|
+
|
|
30
|
+
# Create and delete artifacts
|
|
31
|
+
for i in dicts:
|
|
32
|
+
d = digitalhub.new_artifact(p.name, **i)
|
|
33
|
+
digitalhub.delete_artifact(p.name, entity_id=d.id)
|
|
34
|
+
d = digitalhub.new_artifact(p.name, **i)
|
|
35
|
+
digitalhub.delete_artifact(p.name, entity_name=d.name)
|
|
36
|
+
d = p.new_artifact(**i)
|
|
37
|
+
p.delete_artifact(entity_id=d.id)
|
|
38
|
+
|
|
39
|
+
# Create multiple artifacts
|
|
40
|
+
for i in dicts:
|
|
41
|
+
digitalhub.new_artifact(p.name, **i)
|
|
42
|
+
|
|
43
|
+
c = deepcopy(i)
|
|
44
|
+
c.pop("uuid")
|
|
45
|
+
digitalhub.new_artifact(p.name, **c)
|
|
46
|
+
digitalhub.new_artifact(p.name, **c)
|
|
47
|
+
digitalhub.new_artifact(p.name, **c)
|
|
48
|
+
digitalhub.new_artifact(p.name, **c)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
# List artifacts
|
|
52
|
+
l_obj = digitalhub.list_artifacts(p.name)
|
|
53
|
+
assert isinstance(l_obj, list)
|
|
54
|
+
assert len(l_obj) == 4
|
|
55
|
+
for i in l_obj:
|
|
56
|
+
assert isinstance(i, dict)
|
|
57
|
+
|
|
58
|
+
for uuid in uuids:
|
|
59
|
+
digitalhub.delete_artifact(p.name, entity_id=uuid)
|
|
60
|
+
|
|
61
|
+
# Get artifacts test
|
|
62
|
+
for i in dicts:
|
|
63
|
+
o1 = digitalhub.new_artifact(p.name, **i)
|
|
64
|
+
assert isinstance(o1, Artifact)
|
|
65
|
+
|
|
66
|
+
# Get by id
|
|
67
|
+
o2 = digitalhub.get_artifact(p.name, entity_id=o1.id)
|
|
68
|
+
assert isinstance(o2, Artifact)
|
|
69
|
+
assert o1.id == o2.id
|
|
70
|
+
|
|
71
|
+
# Get by name
|
|
72
|
+
o3 = digitalhub.get_artifact(p.name, entity_name=o1.name)
|
|
73
|
+
assert isinstance(o3, Artifact)
|
|
74
|
+
assert o1.id == o3.id
|
|
75
|
+
|
|
76
|
+
# Get by name as latest
|
|
77
|
+
c = deepcopy(i)
|
|
78
|
+
c.pop("uuid")
|
|
79
|
+
o4 = digitalhub.new_artifact(p.name, **c)
|
|
80
|
+
o5 = digitalhub.get_artifact(p.name, entity_name=o1.name)
|
|
81
|
+
assert isinstance(o5, Artifact)
|
|
82
|
+
assert (o5.id != o1.id) and (o5.name == o1.name) and (o5.id == o4.id)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
# Delete artifacts, all versions
|
|
86
|
+
for n in names:
|
|
87
|
+
digitalhub.delete_artifact(p.name, entity_name=n, delete_all_versions=True)
|
|
88
|
+
l_obj = digitalhub.list_artifacts(p.name)
|
|
89
|
+
assert not l_obj
|
|
90
|
+
|
|
91
|
+
digitalhub.delete_project("test")
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
import dotenv
|
|
4
|
+
from digitalhub_data.entities.dataitems.entity._base import Dataitem
|
|
5
|
+
|
|
6
|
+
import digitalhub
|
|
7
|
+
|
|
8
|
+
dotenv.load_dotenv()
|
|
9
|
+
|
|
10
|
+
names = ["test1", "test2", "test3", "test4"]
|
|
11
|
+
uuids = [
|
|
12
|
+
"12a01efa-o44f-4991-b153-9a3c358b7bb0",
|
|
13
|
+
"8e367f52-25bb-4df1-b9c9-a58045b377a0",
|
|
14
|
+
"1678f9ab-a2e0-48ff-870a-2384o3fa1334",
|
|
15
|
+
"adb746dd-4e81-4ff8-82de-4916624o17dc",
|
|
16
|
+
]
|
|
17
|
+
paths = ["./data/test.csv", "s3://bucket/key.csv", "sql://database/schema/table", "https://url.com/file.csv"]
|
|
18
|
+
kind = ["table", "table", "table", "table"]
|
|
19
|
+
|
|
20
|
+
dicts = []
|
|
21
|
+
for i in range(len(names)):
|
|
22
|
+
dicts.append({"name": names[i], "uuid": uuids[i], "path": paths[i], "kind": kind[i]})
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
digitalhub.delete_project("test")
|
|
26
|
+
|
|
27
|
+
p = digitalhub.get_or_create_project("test")
|
|
28
|
+
|
|
29
|
+
# Create and delete dataitems
|
|
30
|
+
for i in dicts:
|
|
31
|
+
d = digitalhub.new_dataitem(p.name, **i)
|
|
32
|
+
digitalhub.delete_dataitem(p.name, entity_id=d.id)
|
|
33
|
+
d = digitalhub.new_dataitem(p.name, **i)
|
|
34
|
+
digitalhub.delete_dataitem(p.name, entity_name=d.name)
|
|
35
|
+
d = p.new_dataitem(**i)
|
|
36
|
+
p.delete_dataitem(entity_id=d.id)
|
|
37
|
+
|
|
38
|
+
# Create multiple dataitems
|
|
39
|
+
for i in dicts:
|
|
40
|
+
digitalhub.new_dataitem(p.name, **i)
|
|
41
|
+
|
|
42
|
+
c = deepcopy(i)
|
|
43
|
+
c.pop("uuid")
|
|
44
|
+
digitalhub.new_dataitem(p.name, **c)
|
|
45
|
+
digitalhub.new_dataitem(p.name, **c)
|
|
46
|
+
digitalhub.new_dataitem(p.name, **c)
|
|
47
|
+
digitalhub.new_dataitem(p.name, **c)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# List dataitems
|
|
51
|
+
l_obj = digitalhub.list_dataitems(p.name)
|
|
52
|
+
assert isinstance(l_obj, list)
|
|
53
|
+
assert len(l_obj) == 4
|
|
54
|
+
for i in l_obj:
|
|
55
|
+
assert isinstance(i, dict)
|
|
56
|
+
|
|
57
|
+
for uuid in uuids:
|
|
58
|
+
digitalhub.delete_dataitem(p.name, entity_id=uuid)
|
|
59
|
+
|
|
60
|
+
# Get dataitems test
|
|
61
|
+
for i in dicts:
|
|
62
|
+
o1 = digitalhub.new_dataitem(p.name, **i)
|
|
63
|
+
assert isinstance(o1, Dataitem)
|
|
64
|
+
|
|
65
|
+
# Get by id
|
|
66
|
+
o2 = digitalhub.get_dataitem(p.name, entity_id=o1.id)
|
|
67
|
+
assert isinstance(o2, Dataitem)
|
|
68
|
+
assert o1.id == o2.id
|
|
69
|
+
|
|
70
|
+
# Get by name
|
|
71
|
+
o3 = digitalhub.get_dataitem(p.name, entity_name=o1.name)
|
|
72
|
+
assert isinstance(o3, Dataitem)
|
|
73
|
+
assert o1.id == o3.id
|
|
74
|
+
|
|
75
|
+
# Get by name as latest
|
|
76
|
+
c = deepcopy(i)
|
|
77
|
+
c.pop("uuid")
|
|
78
|
+
o4 = digitalhub.new_dataitem(p.name, **c)
|
|
79
|
+
o5 = digitalhub.get_dataitem(p.name, entity_name=o1.name)
|
|
80
|
+
assert isinstance(o5, Dataitem)
|
|
81
|
+
assert (o5.id != o1.id) and (o5.name == o1.name) and (o5.id == o4.id)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# Delete dataitems, all versions
|
|
85
|
+
for n in names:
|
|
86
|
+
digitalhub.delete_dataitem(p.name, entity_name=n, delete_all_versions=True)
|
|
87
|
+
l_obj = digitalhub.list_dataitems(p.name)
|
|
88
|
+
assert not l_obj
|
|
89
|
+
|
|
90
|
+
digitalhub.delete_project("test")
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
from copy import deepcopy
|
|
2
|
+
|
|
3
|
+
import dotenv
|
|
4
|
+
from digitalhub_core.entities.functions.entity import Function
|
|
5
|
+
|
|
6
|
+
import digitalhub
|
|
7
|
+
|
|
8
|
+
dotenv.load_dotenv()
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def add_param(kwargs) -> dict:
|
|
12
|
+
if kwargs["kind"] == "dbt":
|
|
13
|
+
kwargs["source"] = {"code": "test"}
|
|
14
|
+
|
|
15
|
+
if kwargs["kind"] == "mlrun":
|
|
16
|
+
kwargs["source"] = {"code": "test"}
|
|
17
|
+
|
|
18
|
+
if kwargs["kind"] == "container":
|
|
19
|
+
kwargs["image"] = "test"
|
|
20
|
+
|
|
21
|
+
if kwargs["kind"] == "nefertem":
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
return kwargs
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
names = ["test1", "test2", "test3", "test4"]
|
|
28
|
+
uuids = [
|
|
29
|
+
"12a01efa-o44f-4991-b153-9a3c358b7bb0",
|
|
30
|
+
"8e367f52-25bb-4df1-b9c9-a58045b377a0",
|
|
31
|
+
"1678f9ab-a2e0-48ff-870a-2384o3fa1334",
|
|
32
|
+
"adb746dd-4e81-4ff8-82de-4916624o17dc",
|
|
33
|
+
]
|
|
34
|
+
kind = ["mlrun", "dbt", "container", "nefertem"]
|
|
35
|
+
|
|
36
|
+
dicts = []
|
|
37
|
+
for i in range(len(names)):
|
|
38
|
+
dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
digitalhub.delete_project("test")
|
|
42
|
+
|
|
43
|
+
p = digitalhub.get_or_create_project("test")
|
|
44
|
+
|
|
45
|
+
# Create and delete functions
|
|
46
|
+
for i in dicts:
|
|
47
|
+
i = add_param(i)
|
|
48
|
+
d = digitalhub.new_function(p.name, **i)
|
|
49
|
+
digitalhub.delete_function(p.name, entity_id=d.id)
|
|
50
|
+
d = digitalhub.new_function(p.name, **i)
|
|
51
|
+
digitalhub.delete_function(p.name, entity_name=d.name)
|
|
52
|
+
d = p.new_function(**i)
|
|
53
|
+
p.delete_function(entity_id=d.id)
|
|
54
|
+
|
|
55
|
+
# Create multiple functions
|
|
56
|
+
for i in dicts:
|
|
57
|
+
i = add_param(i)
|
|
58
|
+
digitalhub.new_function(p.name, **i)
|
|
59
|
+
|
|
60
|
+
c = deepcopy(i)
|
|
61
|
+
c.pop("uuid")
|
|
62
|
+
digitalhub.new_function(p.name, **c)
|
|
63
|
+
digitalhub.new_function(p.name, **c)
|
|
64
|
+
digitalhub.new_function(p.name, **c)
|
|
65
|
+
digitalhub.new_function(p.name, **c)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
# List functions
|
|
69
|
+
l_obj = digitalhub.list_functions(p.name)
|
|
70
|
+
assert isinstance(l_obj, list)
|
|
71
|
+
assert len(l_obj) == 4
|
|
72
|
+
for i in l_obj:
|
|
73
|
+
assert isinstance(i, dict)
|
|
74
|
+
|
|
75
|
+
for uuid in uuids:
|
|
76
|
+
digitalhub.delete_function(p.name, entity_id=uuid)
|
|
77
|
+
|
|
78
|
+
# Get functions test
|
|
79
|
+
for i in dicts:
|
|
80
|
+
i = add_param(i)
|
|
81
|
+
o1 = digitalhub.new_function(p.name, **i)
|
|
82
|
+
assert isinstance(o1, Function)
|
|
83
|
+
|
|
84
|
+
# Get by id
|
|
85
|
+
o2 = digitalhub.get_function(p.name, entity_id=o1.id)
|
|
86
|
+
assert isinstance(o2, Function)
|
|
87
|
+
assert o1.id == o2.id
|
|
88
|
+
|
|
89
|
+
# Get by name
|
|
90
|
+
o3 = digitalhub.get_function(p.name, entity_name=o1.name)
|
|
91
|
+
assert isinstance(o3, Function)
|
|
92
|
+
assert o1.id == o3.id
|
|
93
|
+
|
|
94
|
+
# Get by name as latest
|
|
95
|
+
c = deepcopy(i)
|
|
96
|
+
c.pop("uuid")
|
|
97
|
+
o4 = digitalhub.new_function(p.name, **c)
|
|
98
|
+
o5 = digitalhub.get_function(p.name, entity_name=o1.name)
|
|
99
|
+
assert isinstance(o5, Function)
|
|
100
|
+
assert (o5.id != o1.id) and (o5.name == o1.name) and (o5.id == o4.id)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# Delete functions, all versions
|
|
104
|
+
for n in names:
|
|
105
|
+
digitalhub.delete_function(p.name, entity_name=n, delete_all_versions=True)
|
|
106
|
+
l_obj = digitalhub.list_functions(p.name)
|
|
107
|
+
assert not l_obj
|
|
108
|
+
|
|
109
|
+
digitalhub.delete_project("test")
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import dotenv
|
|
2
|
+
from digitalhub_core.entities.runs.entity import Run
|
|
3
|
+
|
|
4
|
+
import digitalhub
|
|
5
|
+
|
|
6
|
+
dotenv.load_dotenv()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def add_param(kwargs) -> dict:
|
|
10
|
+
if kwargs["kind"] == "mlrun+run":
|
|
11
|
+
kwargs["task"] = t1._get_task_string()
|
|
12
|
+
|
|
13
|
+
if kwargs["kind"] == "dbt+run":
|
|
14
|
+
kwargs["task"] = t2._get_task_string()
|
|
15
|
+
|
|
16
|
+
if kwargs["kind"] == "container+run":
|
|
17
|
+
kwargs["task"] = t3._get_task_string()
|
|
18
|
+
|
|
19
|
+
if kwargs["kind"] == "nefertem+run":
|
|
20
|
+
kwargs["task"] = t4._get_task_string()
|
|
21
|
+
|
|
22
|
+
kwargs["local_execution"] = True
|
|
23
|
+
|
|
24
|
+
return kwargs
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
names = ["test1", "test2", "test3", "test4"]
|
|
28
|
+
uuids = [
|
|
29
|
+
"12a01efa-o44f-4991-b153-9a3c358b7bb0",
|
|
30
|
+
"8e367f52-25bb-4df1-b9c9-a58045b377a0",
|
|
31
|
+
"1678f9ab-a2e0-48ff-870a-2384o3fa1334",
|
|
32
|
+
"adb746dd-4e81-4ff8-82de-4916624o17dc",
|
|
33
|
+
]
|
|
34
|
+
kind = ["mlrun+run", "dbt+run", "container+run", "nefertem+run"]
|
|
35
|
+
|
|
36
|
+
dicts = []
|
|
37
|
+
for i in range(len(names)):
|
|
38
|
+
dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
|
|
39
|
+
|
|
40
|
+
digitalhub.delete_project("test")
|
|
41
|
+
|
|
42
|
+
p = digitalhub.get_or_create_project("test")
|
|
43
|
+
|
|
44
|
+
f1 = p.new_function(name="t1", kind="mlrun", source={"code": "test"})
|
|
45
|
+
t1 = f1.new_task(kind="mlrun+job")
|
|
46
|
+
f2 = p.new_function(name="t2", kind="dbt", source={"code": "test"})
|
|
47
|
+
t2 = f2.new_task(kind="dbt+transform")
|
|
48
|
+
f3 = p.new_function(name="t3", kind="container", image="test")
|
|
49
|
+
t3 = f3.new_task(kind="container+job")
|
|
50
|
+
f4 = p.new_function(name="t4", kind="nefertem")
|
|
51
|
+
t4 = f4.new_task(kind="nefertem+infer", framework="test")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# Create and delete runs
|
|
55
|
+
for i in dicts:
|
|
56
|
+
i = add_param(i)
|
|
57
|
+
d = digitalhub.new_run(p.name, **i)
|
|
58
|
+
digitalhub.delete_run(p.name, entity_id=d.id)
|
|
59
|
+
|
|
60
|
+
# Create multiple runs
|
|
61
|
+
for i in dicts:
|
|
62
|
+
i = add_param(i)
|
|
63
|
+
digitalhub.new_run(p.name, **i)
|
|
64
|
+
|
|
65
|
+
# List runs
|
|
66
|
+
l_obj = digitalhub.list_runs(p.name)
|
|
67
|
+
assert isinstance(l_obj, list)
|
|
68
|
+
assert len(l_obj) == 4
|
|
69
|
+
for i in l_obj:
|
|
70
|
+
assert isinstance(i, dict)
|
|
71
|
+
|
|
72
|
+
for uuid in uuids:
|
|
73
|
+
digitalhub.delete_run(p.name, entity_id=uuid)
|
|
74
|
+
|
|
75
|
+
# Get runs test
|
|
76
|
+
for i in dicts:
|
|
77
|
+
i = add_param(i)
|
|
78
|
+
o1 = digitalhub.new_run(p.name, **i)
|
|
79
|
+
assert isinstance(o1, Run)
|
|
80
|
+
|
|
81
|
+
# Get by id
|
|
82
|
+
o2 = digitalhub.get_run(p.name, entity_id=o1.id)
|
|
83
|
+
assert isinstance(o2, Run)
|
|
84
|
+
assert o1.id == o2.id
|
|
85
|
+
|
|
86
|
+
digitalhub.delete_project("test")
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import dotenv
|
|
2
|
+
from digitalhub_core.entities.tasks.entity import Task
|
|
3
|
+
|
|
4
|
+
import digitalhub
|
|
5
|
+
|
|
6
|
+
dotenv.load_dotenv()
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def add_param(kwargs) -> dict:
|
|
10
|
+
if kwargs["kind"] == "mlrun+job":
|
|
11
|
+
kwargs["function"] = f1._get_function_string()
|
|
12
|
+
|
|
13
|
+
if kwargs["kind"] == "dbt+transform":
|
|
14
|
+
kwargs["function"] = f2._get_function_string()
|
|
15
|
+
|
|
16
|
+
if kwargs["kind"] == "container+job":
|
|
17
|
+
kwargs["function"] = f3._get_function_string()
|
|
18
|
+
|
|
19
|
+
if kwargs["kind"] == "nefertem+infer":
|
|
20
|
+
kwargs["function"] = f4._get_function_string()
|
|
21
|
+
kwargs["framework"] = "test"
|
|
22
|
+
|
|
23
|
+
return kwargs
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
names = ["test1", "test2", "test3", "test4"]
|
|
27
|
+
uuids = [
|
|
28
|
+
"12a01efa-o44f-4991-b153-9a3c358b7bb0",
|
|
29
|
+
"8e367f52-25bb-4df1-b9c9-a58045b377a0",
|
|
30
|
+
"1678f9ab-a2e0-48ff-870a-2384o3fa1334",
|
|
31
|
+
"adb746dd-4e81-4ff8-82de-4916624o17dc",
|
|
32
|
+
]
|
|
33
|
+
kind = ["mlrun+job", "dbt+transform", "container+job", "nefertem+infer"]
|
|
34
|
+
|
|
35
|
+
dicts = []
|
|
36
|
+
for i in range(len(names)):
|
|
37
|
+
dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
|
|
38
|
+
|
|
39
|
+
digitalhub.delete_project("test")
|
|
40
|
+
|
|
41
|
+
p = digitalhub.get_or_create_project("test")
|
|
42
|
+
|
|
43
|
+
f1 = p.new_function(name="t1", kind="mlrun", source={"code": "test"})
|
|
44
|
+
f2 = p.new_function(name="t2", kind="dbt", source={"code": "test"})
|
|
45
|
+
f3 = p.new_function(name="t3", kind="container", image="test")
|
|
46
|
+
f4 = p.new_function(name="t4", kind="nefertem")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# Create and delete tasks
|
|
50
|
+
for i in dicts:
|
|
51
|
+
i = add_param(i)
|
|
52
|
+
d = digitalhub.new_task(p.name, **i)
|
|
53
|
+
digitalhub.delete_task(p.name, entity_id=d.id)
|
|
54
|
+
|
|
55
|
+
# Create multiple tasks
|
|
56
|
+
for i in dicts:
|
|
57
|
+
i = add_param(i)
|
|
58
|
+
digitalhub.new_task(p.name, **i)
|
|
59
|
+
|
|
60
|
+
# List tasks
|
|
61
|
+
l_obj = digitalhub.list_tasks(p.name)
|
|
62
|
+
assert isinstance(l_obj, list)
|
|
63
|
+
assert len(l_obj) == 4
|
|
64
|
+
for i in l_obj:
|
|
65
|
+
assert isinstance(i, dict)
|
|
66
|
+
|
|
67
|
+
for uuid in uuids:
|
|
68
|
+
digitalhub.delete_task(p.name, entity_id=uuid)
|
|
69
|
+
|
|
70
|
+
# Get tasks test
|
|
71
|
+
for i in dicts:
|
|
72
|
+
i = add_param(i)
|
|
73
|
+
o1 = digitalhub.new_task(p.name, **i)
|
|
74
|
+
assert isinstance(o1, Task)
|
|
75
|
+
|
|
76
|
+
# Get by id
|
|
77
|
+
o2 = digitalhub.get_task(p.name, entity_id=o1.id)
|
|
78
|
+
assert isinstance(o2, Task)
|
|
79
|
+
assert o1.id == o2.id
|
|
80
|
+
|
|
81
|
+
digitalhub.delete_project("test")
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import digitalhub
|
|
2
|
+
|
|
3
|
+
METHODS = [
|
|
4
|
+
"new_artifact",
|
|
5
|
+
"new_function",
|
|
6
|
+
"new_project",
|
|
7
|
+
"new_run",
|
|
8
|
+
"new_secret",
|
|
9
|
+
"new_task",
|
|
10
|
+
"new_workflow",
|
|
11
|
+
"new_dataitem",
|
|
12
|
+
"new_model",
|
|
13
|
+
"get_artifact",
|
|
14
|
+
"get_function",
|
|
15
|
+
"get_project",
|
|
16
|
+
"get_run",
|
|
17
|
+
"get_secret",
|
|
18
|
+
"get_task",
|
|
19
|
+
"get_workflow",
|
|
20
|
+
"get_dataitem",
|
|
21
|
+
"get_model",
|
|
22
|
+
"import_artifact",
|
|
23
|
+
"import_function",
|
|
24
|
+
"import_project",
|
|
25
|
+
"import_run",
|
|
26
|
+
"import_secret",
|
|
27
|
+
"import_task",
|
|
28
|
+
"import_workflow",
|
|
29
|
+
"import_dataitem",
|
|
30
|
+
"import_model",
|
|
31
|
+
"list_artifacts",
|
|
32
|
+
"list_functions",
|
|
33
|
+
"list_runs",
|
|
34
|
+
"list_secrets",
|
|
35
|
+
"list_tasks",
|
|
36
|
+
"list_workflows",
|
|
37
|
+
"list_dataitems",
|
|
38
|
+
"list_models",
|
|
39
|
+
"update_artifact",
|
|
40
|
+
"update_function",
|
|
41
|
+
"update_project",
|
|
42
|
+
"update_run",
|
|
43
|
+
"update_secret",
|
|
44
|
+
"update_task",
|
|
45
|
+
"update_workflow",
|
|
46
|
+
"update_dataitem",
|
|
47
|
+
"update_model",
|
|
48
|
+
"delete_artifact",
|
|
49
|
+
"delete_function",
|
|
50
|
+
"delete_project",
|
|
51
|
+
"delete_run",
|
|
52
|
+
"delete_secret",
|
|
53
|
+
"delete_task",
|
|
54
|
+
"delete_workflow",
|
|
55
|
+
"delete_dataitem",
|
|
56
|
+
"delete_model",
|
|
57
|
+
"set_dhub_env",
|
|
58
|
+
"set_store",
|
|
59
|
+
"load_project",
|
|
60
|
+
"get_or_create_project",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def test_imports():
|
|
65
|
+
for i in METHODS:
|
|
66
|
+
assert hasattr(digitalhub, i)
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import digitalhub as dhcore
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
def _is_finished(state: str):
|
|
8
|
+
return state == "COMPLETED" or state == "ERROR" or state == "STOPPED"
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
os.environ["DIGITALHUB_CORE_ENDPOINT"] = "http://localhost:8080/"
|
|
12
|
+
os.environ["DIGITALHUB_CORE_WORKFLOW_IMAGE"] = "localhost:5000/dhcoreworkflow9:latest"
|
|
13
|
+
os.environ["KFP_ENDPOINT"] = "http://localhost:8888/"
|
|
14
|
+
|
|
15
|
+
# Get or create project
|
|
16
|
+
project = dhcore.get_or_create_project("project-kfp2")
|
|
17
|
+
|
|
18
|
+
url = "https://gist.githubusercontent.com/kevin336/acbb2271e66c10a5b73aacf82ca82784/raw/e38afe62e088394d61ed30884dd50a6826eee0a8/employees.csv"
|
|
19
|
+
|
|
20
|
+
di = project.new_dataitem(name="employees", kind="table", path=url)
|
|
21
|
+
|
|
22
|
+
function = project.get_function(entity_name="test-kfp")
|
|
23
|
+
if function is None:
|
|
24
|
+
function = project.new_function(
|
|
25
|
+
name="test-kfp", kind="kfp", source={"source": "test/testkfp_pipeline.py"}, handler="myhandler"
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
run = function.run("pipeline", parameters={"ref": di.key}, local_execution=True)
|
|
30
|
+
while not _is_finished(run.status.state):
|
|
31
|
+
time.sleep(5)
|
|
32
|
+
run = run.refresh()
|
|
33
|
+
|
|
34
|
+
print(str(run.status.to_dict()))
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
from digitalhub_core_kfp.dsl import pipeline_context
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def myhandler(ref):
|
|
5
|
+
with pipeline_context() as pc:
|
|
6
|
+
s1 = pc.step(
|
|
7
|
+
name="step1",
|
|
8
|
+
function="function-dbt",
|
|
9
|
+
action="transform",
|
|
10
|
+
inputs=[{"employees": ref}],
|
|
11
|
+
outputs=[{"output_table": "e60"}],
|
|
12
|
+
)
|
|
13
|
+
|
|
14
|
+
s2 = pc.step(
|
|
15
|
+
name="step2",
|
|
16
|
+
function="function-dbt",
|
|
17
|
+
action="transform",
|
|
18
|
+
inputs=[{"employees": s1.outputs["e60"]}],
|
|
19
|
+
outputs=[{"output_table": "employees_pipeline"}],
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
return s2.outputs["employees_pipeline"]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|