digitalhub 0.10.0b2__py3-none-any.whl → 0.10.0b3__py3-none-any.whl

digitalhub/entities/dataitem/crud.py

@@ -5,7 +5,7 @@ from typing import Any
 
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._operations.processor import processor
-from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_source, post_process, process_kwargs
+from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_data, eval_source, post_process, process_kwargs
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.dataitem._base.entity import Dataitem
@@ -81,6 +81,8 @@ def log_dataitem(
     source: list[str] | str | None = None,
     data: Any | None = None,
     path: str | None = None,
+    file_format: str | None = None,
+    engine: str | None = "pandas",
     **kwargs,
 ) -> Dataitem:
     """
@@ -100,6 +102,10 @@ def log_dataitem(
         Dataframe to log. Alternative to source.
     path : str
         Destination path of the dataitem. If not provided, it's generated.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Dataframe engine (pandas, polars, etc.).
     **kwargs : dict
         New dataitem spec parameters.
 
@@ -115,7 +121,12 @@ def log_dataitem(
     >>>                    kind="table",
     >>>                    data=df)
     """
+    cleanup = False
+    if data is not None:
+        cleanup = True
+
     source = eval_source(source, data, kind, name, project)
+    data = eval_data(project, kind, source, data, file_format, engine)
     kwargs = process_kwargs(project, name, kind, source=source, data=data, path=path, **kwargs)
     obj = processor.log_material_entity(
         source=source,
@@ -124,8 +135,8 @@ def log_dataitem(
         kind=kind,
         **kwargs,
     )
-    if data is not None:
-        obj = post_process(obj, data)
+    obj = post_process(obj, data)
+    if cleanup:
         clean_tmp_path(source)
     return obj
 

digitalhub/entities/dataitem/utils.py

@@ -9,6 +9,7 @@ from digitalhub.entities._base.entity._constructors.uuid import build_uuid
 from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
 from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
 from digitalhub.readers.data.api import get_reader_by_object
+from digitalhub.stores.api import get_store
 from digitalhub.utils.generic_utils import slugify_string
 
 if typing.TYPE_CHECKING:
@@ -54,6 +55,44 @@ def eval_source(
     raise NotImplementedError
 
 
+def eval_data(
+    project: str,
+    kind: str,
+    source: str,
+    data: Any | None = None,
+    file_format: str | None = None,
+    engine: str | None = None,
+) -> Any:
+    """
+    Evaluate data is loaded.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    source : str
+        Source(s).
+    data : Any
+        Dataframe to log. Alternative to source.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Engine to use.
+
+    Returns
+    -------
+    None
+    """
+    if kind == EntityKinds.DATAITEM_TABLE.value:
+        if data is None:
+            return get_store(project, source).read_df(
+                source,
+                file_format=file_format,
+                engine=engine,
+            )
+    return data
+
+
 def process_kwargs(
     project: str,
     name: str,

digitalhub/stores/_base/store.py

@@ -59,7 +59,7 @@ class Store:
     def read_df(
         self,
         path: str | list[str],
-        extension: str,
+        file_format: str | None = None,
         engine: str | None = None,
         **kwargs,
     ) -> Any:

{digitalhub-0.10.0b2.dist-info → digitalhub-0.10.0b3.dist-info}/METADATA

@@ -1,7 +1,8 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.10.0b2
+Version: 0.10.0b3
 Summary: Python SDK for Digitalhub
+Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
                                 Version 2.0, January 2004
@@ -219,48 +220,41 @@ License:                               Apache License
         
         (SIL OPEN FONT LICENSE Version 1.1) The Inter font family (https://github.com/rsms/inter)
         (SIL OPEN FONT LICENSE Version 1.1) The Fira Code font family (https://github.com/tonsky/FiraCode)
-        
-Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
+License-File: LICENSE.txt
 Keywords: data,dataops,kubernetes
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
+Requires-Python: <3.13,>=3.9
 Requires-Dist: boto3
-Requires-Dist: pydantic
-Requires-Dist: sqlalchemy
-Requires-Dist: pyarrow
+Requires-Dist: gitpython>=3
 Requires-Dist: numpy<2
-Requires-Dist: requests
-Requires-Dist: PyYAML
-Requires-Dist: python-dotenv
-Requires-Dist: GitPython>=3
 Requires-Dist: psycopg2-binary
+Requires-Dist: pyarrow
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv
 Requires-Dist: python-slugify
+Requires-Dist: pyyaml
+Requires-Dist: requests
+Requires-Dist: sqlalchemy
+Provides-Extra: dev
+Requires-Dist: bumpver; extra == 'dev'
+Requires-Dist: jsonschema; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
 Provides-Extra: full
-Requires-Dist: pandas; extra == "full"
-Requires-Dist: mlflow; extra == "full"
-Requires-Dist: fsspec; extra == "full"
-Requires-Dist: s3fs; extra == "full"
-Provides-Extra: pandas
-Requires-Dist: pandas; extra == "pandas"
-Requires-Dist: fsspec; extra == "pandas"
-Requires-Dist: s3fs; extra == "pandas"
+Requires-Dist: fsspec; extra == 'full'
+Requires-Dist: mlflow; extra == 'full'
+Requires-Dist: pandas; extra == 'full'
+Requires-Dist: s3fs; extra == 'full'
 Provides-Extra: mlflow
-Requires-Dist: mlflow; extra == "mlflow"
-Provides-Extra: dev
-Requires-Dist: black; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: bumpver; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
-Requires-Dist: moto; extra == "dev"
-Provides-Extra: docs
-Requires-Dist: Sphinx>=7; extra == "docs"
-Requires-Dist: pydata-sphinx-theme>=0.15; extra == "docs"
-Requires-Dist: numpydoc>=1.6; extra == "docs"
+Requires-Dist: mlflow; extra == 'mlflow'
+Provides-Extra: pandas
+Requires-Dist: fsspec; extra == 'pandas'
+Requires-Dist: pandas; extra == 'pandas'
+Requires-Dist: s3fs; extra == 'pandas'
+Description-Content-Type: text/markdown
 
 # Digitalhub Library
 

{digitalhub-0.10.0b2.dist-info → digitalhub-0.10.0b3.dist-info}/RECORD

@@ -84,8 +84,8 @@ digitalhub/entities/artifact/artifact/entity.py,sha256=hj6UlJMWnWimqizhKGcKpvOpc
 digitalhub/entities/artifact/artifact/spec.py,sha256=iX6czlG6W_ZoW86PD2GPc_-xLk6Zb1Vt3fk8Lzjbp3c,593
 digitalhub/entities/artifact/artifact/status.py,sha256=x-lTgO2KkjwzlJnEhIfUtF9rzJ1DTIAd3-Hn6ZeLRqo,305
 digitalhub/entities/dataitem/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-digitalhub/entities/dataitem/crud.py,sha256=-y3lTjEtISje6X_lo-rx9Aynf_WkvzYi9G6WY7QJvRA,8090
-digitalhub/entities/dataitem/utils.py,sha256=zCxUMyhEdL7jP4nK42tT1obYMkJTT2txo1lGHPwG1H4,3232
+digitalhub/entities/dataitem/crud.py,sha256=Vel2zyJl8DbdJJoNBx1nEoFYy5e2Gi1AmJUsCE0nhZE,8418
+digitalhub/entities/dataitem/utils.py,sha256=frrJrNnk4dJbhmBgqGWHpXqW4tpmD6JgGDhEo_LgGr4,4047
 digitalhub/entities/dataitem/_base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 digitalhub/entities/dataitem/_base/builder.py,sha256=XZ5Ul-aBkx6ygpN9rhjNQOD-6jzJhulyM0WCD-f3big,2256
 digitalhub/entities/dataitem/_base/entity.py,sha256=J8G7Xm_AKETg02RtNHlUyM-bmvT__HKu1Npv4Od037A,945
@@ -207,7 +207,7 @@ digitalhub/stores/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU
 digitalhub/stores/api.py,sha256=7bspqJaexE8PKnVDotpAX3mgLsgcCNhrImD1kNI4heQ,560
 digitalhub/stores/builder.py,sha256=6_19vEYnh3WYkgOY0h5xAjpFujMYR0oHF5PBDupsgnc,2314
 digitalhub/stores/_base/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-digitalhub/stores/_base/store.py,sha256=cMZVHJKmEaMfeMQ9mUU_d4WhtscBhipUCi0-QXY6Wvw,5132
+digitalhub/stores/_base/store.py,sha256=8YzbQ7x1p1gx5H0WgVpqZsCmNTpezPT67tNIU3fAp_M,5148
 digitalhub/stores/local/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 digitalhub/stores/local/store.py,sha256=cpQ1hfw5TZ_akDVS_4gT_rqWdDF1qNMDwHwuXt-iafs,7333
 digitalhub/stores/remote/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -231,18 +231,7 @@ digitalhub/utils/git_utils.py,sha256=air8jn73FxzSWRxpvObcdOJBWcFOqb5A7D4ISwPEs7A
 digitalhub/utils/io_utils.py,sha256=8jD4Rp_b7LZEpY5JSMxVUowZsnifKnbGpHT5Hijx9-g,3299
 digitalhub/utils/logger.py,sha256=ml3ne6D8wuRdNZ4F6ywmvWotSxjmZWnmKgNiuHb4R5M,437
 digitalhub/utils/uri_utils.py,sha256=AKfvMtdt_0OY_cNtJY1t0YXrQ12OYGf-KYBF_2m0ULA,3821
-test/test_crud_functions.py,sha256=tQs_QBaPCuYVSBpbl-he5_6jr_tteCXVmohj1ZluNsA,2769
-test/test_crud_runs.py,sha256=lkssy15UPJKymgazmi5gG6RLxyTsG-tM_CpNCowD2gQ,2220
-test/test_crud_tasks.py,sha256=sIbY-Hq6C2N20hWHfbCMw9c-zpYS0m_UJGnPINR4Q6s,2111
-test/testkfp.py,sha256=fy__EywOCtucL5HUEKD23gwit1qu9SrCWFSjyeKxe68,1085
-test/testkfp_pipeline.py,sha256=WceFrCp-avHI7PcwIvnv7Kgs2xK3oQqU6sjaonGamg8,622
-test/local/CRUD/test_artifacts.py,sha256=Y3J_C7SDRSsQd2SGIZjPIOvyTL92B1sTFrUONG3YG0Y,2968
-test/local/CRUD/test_dataitems.py,sha256=LQqTzI59uwTGy4zoq8jL0yWVe2W9vXlatkgDU9aB6xg,2968
-test/local/CRUD/test_models.py,sha256=msosbZuRwIMbZtmi3ZaOva4TjQ4lrzkNu9AguIFhrSo,2929
-test/local/imports/test_imports.py,sha256=Z_RomI5EN_5KJUU1p7S1lLl7krTuEwB0L1cBtbXXoOA,1254
-test/local/instances/test_validate.py,sha256=bGPKRFR_Tb5nlzzmI_ty_6UVUvYGseE2-pkNVoGWeO0,1842
-digitalhub-0.10.0b2.dist-info/LICENSE.txt,sha256=qmrTTXPlgU0kSRlRVbjhlyGs1IXs2QPxo_Y-Mn06J0k,11589
-digitalhub-0.10.0b2.dist-info/METADATA,sha256=t4q9tWFbz_YUduJulntoZJoaXVpFjj4SoH0cSnDRgJk,15368
-digitalhub-0.10.0b2.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-digitalhub-0.10.0b2.dist-info/top_level.txt,sha256=ae9pDfCF27ZoaVAxuBKONMP0lm5P-N_I-e-no1WlvD8,16
-digitalhub-0.10.0b2.dist-info/RECORD,,
+digitalhub-0.10.0b3.dist-info/METADATA,sha256=Xmttpbzf2euECdRnht5Q3sDjmJ7gAtHx_UHfBBQ7sqk,15173
+digitalhub-0.10.0b3.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+digitalhub-0.10.0b3.dist-info/licenses/LICENSE.txt,sha256=qmrTTXPlgU0kSRlRVbjhlyGs1IXs2QPxo_Y-Mn06J0k,11589
+digitalhub-0.10.0b3.dist-info/RECORD,,

{digitalhub-0.10.0b2.dist-info → digitalhub-0.10.0b3.dist-info}/WHEEL

@@ -1,5 +1,4 @@
 Wheel-Version: 1.0
-Generator: setuptools (75.8.0)
+Generator: hatchling 1.27.0
 Root-Is-Purelib: true
 Tag: py3-none-any
-

digitalhub-0.10.0b2.dist-info/top_level.txt

@@ -1,2 +0,0 @@
-digitalhub
-test

test/local/CRUD/test_artifacts.py

@@ -1,96 +0,0 @@
-"""
-Unit tests for the entity Artifact
-"""
-
-import digitalhub as dh
-from digitalhub.entities.artifact._base.entity import Artifact
-
-
-class TestArtifactCRUD:
-    def create_test_dicts(self):
-        names = ["test1", "test2", "test3", "test4"]
-        uuids = [
-            "d150bcca-bb64-451d-8455-dff862254b95",
-            "31acdd2d-0c41-428c-b68b-1b133da9e97b",
-            "b4a3dfdc-b917-44c4-9a29-613dcf734244",
-            "2618d9c4-cd61-440f-aebb-7e5761709f3b",
-        ]
-        paths = [
-            "./data/test.csv",
-            "s3://bucket/key.csv",
-            "sql://database/schema/table",
-            "https://url.com/file.csv",
-        ]
-        kind = ["artifact", "artifact", "artifact", "artifact"]
-
-        dicts = []
-        for i in range(len(names)):
-            dicts.append({"name": names[i], "uuid": uuids[i], "path": paths[i], "kind": kind[i]})
-
-        return dicts
-
-    def test_create_delete(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-        # Create and delete artifacts
-        for i in dicts:
-            d = dh.new_artifact(p.name, **i)
-            dh.delete_artifact(d.key)
-            d = dh.new_artifact(p.name, **i)
-            dh.delete_artifact(d.name, project=p.name, entity_id=d.id)
-            d = p.new_artifact(**i)
-            p.delete_artifact(d.key)
-
-        assert dh.list_artifacts(p.name) == []
-        dh.delete_project("test", local=True, clean_context=True)
-
-    def test_list(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        assert dh.list_artifacts(p.name) == []
-
-        for i in dicts:
-            dh.new_artifact(p.name, **i)
-
-        # List artifacts
-        l_obj = dh.list_artifacts(p.name)
-        assert isinstance(l_obj, list)
-        assert len(l_obj) == 4
-        for i in l_obj:
-            assert isinstance(i, Artifact)
-
-        # delete listed objects
-        for obj in l_obj:
-            dh.delete_artifact(obj.key)
-
-        assert len(dh.list_artifacts(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)
-
-    def test_get(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        for i in dicts:
-            o1 = dh.new_artifact(p.name, **i)
-            assert isinstance(o1, Artifact)
-
-            # Get by name and id
-            o2 = dh.get_artifact(o1.name, project=p.name, entity_id=o1.id)
-            assert isinstance(o2, Artifact)
-            assert o1.id == o2.id
-
-            # Get by key
-            o3 = dh.get_artifact(o1.key)
-            assert isinstance(o3, Artifact)
-            assert o1.id == o3.id
-
-        # delete listed objects
-        l_obj = dh.list_artifacts(p.name)
-        for obj in l_obj:
-            dh.delete_artifact(obj.key)
-
-        assert len(dh.list_artifacts(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)

test/local/CRUD/test_dataitems.py

@@ -1,96 +0,0 @@
-"""
-Unit tests for the entity Dataitem
-"""
-
-import digitalhub as dh
-from digitalhub.entities.dataitem._base.entity import Dataitem
-
-
-class TestDataitemCRUD:
-    def create_test_dicts(self):
-        names = ["test1", "test2", "test3", "test4"]
-        uuids = [
-            "d150bcca-bb64-451d-8455-dff862254b95",
-            "31acdd2d-0c41-428c-b68b-1b133da9e97b",
-            "b4a3dfdc-b917-44c4-9a29-613dcf734244",
-            "2618d9c4-cd61-440f-aebb-7e5761709f3b",
-        ]
-        paths = [
-            "./data/test.csv",
-            "s3://bucket/key.csv",
-            "sql://database/schema/table",
-            "https://url.com/file.csv",
-        ]
-        kind = ["dataitem", "dataitem", "dataitem", "dataitem"]
-
-        dicts = []
-        for i in range(len(names)):
-            dicts.append({"name": names[i], "uuid": uuids[i], "path": paths[i], "kind": kind[i]})
-
-        return dicts
-
-    def test_create_delete(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-        # Create and delete dataitems
-        for i in dicts:
-            d = dh.new_dataitem(p.name, **i)
-            dh.delete_dataitem(d.key)
-            d = dh.new_dataitem(p.name, **i)
-            dh.delete_dataitem(d.name, project=p.name, entity_id=d.id)
-            d = p.new_dataitem(**i)
-            p.delete_dataitem(d.key)
-
-        assert dh.list_dataitems(p.name) == []
-        dh.delete_project("test", local=True, clean_context=True)
-
-    def test_list(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        assert dh.list_dataitems(p.name) == []
-
-        for i in dicts:
-            dh.new_dataitem(p.name, **i)
-
-        # List dataitems
-        l_obj = dh.list_dataitems(p.name)
-        assert isinstance(l_obj, list)
-        assert len(l_obj) == 4
-        for i in l_obj:
-            assert isinstance(i, Dataitem)
-
-        # delete listed objects
-        for obj in l_obj:
-            dh.delete_dataitem(obj.key)
-
-        assert len(dh.list_dataitems(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)
-
-    def test_get(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        for i in dicts:
-            o1 = dh.new_dataitem(p.name, **i)
-            assert isinstance(o1, Dataitem)
-
-            # Get by name and id
-            o2 = dh.get_dataitem(o1.name, project=p.name, entity_id=o1.id)
-            assert isinstance(o2, Dataitem)
-            assert o1.id == o2.id
-
-            # Get by key
-            o3 = dh.get_dataitem(o1.key)
-            assert isinstance(o3, Dataitem)
-            assert o1.id == o3.id
-
-        # delete listed objects
-        l_obj = dh.list_dataitems(p.name)
-        for obj in l_obj:
-            dh.delete_dataitem(obj.key)
-
-        assert len(dh.list_dataitems(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)

test/local/CRUD/test_models.py

@@ -1,95 +0,0 @@
-"""
-Unit tests for the entity Models
-"""
-
-import digitalhub as dh
-from digitalhub.entities.model._base.entity import Model
-
-
-class TestModelCRUD:
-    def create_test_dicts(self):
-        names = ["test1", "test2", "test3", "test4"]
-        uuids = [
-            "d150bcca-bb64-451d-8455-dff862254b95",
-            "31acdd2d-0c41-428c-b68b-1b133da9e97b",
-            "b4a3dfdc-b917-44c4-9a29-613dcf734244",
-            "2618d9c4-cd61-440f-aebb-7e5761709f3b",
-        ]
-        paths = [
-            "./data/my_random_forest_model.pkl",
-            "s3://bucket/model.pkl",
-            "sql://database/schema/linear_regression_model.joblib",
-            "https://url.com/bert_base_uncased.pt",
-        ]
-        kind = ["model", "model", "model", "model"]
-
-        dicts = []
-        for i in range(len(names)):
-            dicts.append({"name": names[i], "uuid": uuids[i], "path": paths[i], "kind": kind[i]})
-
-        return dicts
-
-    def test_create_delete(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-        # Create and delete models
-        for i in dicts:
-            d = dh.new_model(p.name, **i)
-            dh.delete_model(d.key)
-            d = dh.new_model(p.name, **i)
-            dh.delete_model(d.name, project=p.name, entity_id=d.id)
-            d = p.new_model(**i)
-            p.delete_model(d.key)
-        assert dh.list_models(p.name) == []
-        dh.delete_project("test", local=True, clean_context=True)
-
-    def test_list(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        assert dh.list_models(p.name) == []
-
-        for i in dicts:
-            dh.new_model(p.name, **i)
-
-        # List models
-        l_obj = dh.list_models(p.name)
-        assert isinstance(l_obj, list)
-        assert len(l_obj) == 4
-        for i in l_obj:
-            assert isinstance(i, Model)
-
-        # delete listed objects
-        for obj in l_obj:
-            dh.delete_model(obj.key)
-
-        assert len(dh.list_models(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)
-
-    def test_get(self):
-        dicts = self.create_test_dicts()
-        p = dh.get_or_create_project("test", local=True)
-
-        for i in dicts:
-            o1 = dh.new_model(p.name, **i)
-            assert isinstance(o1, Model)
-
-            # Get by name and id
-            o2 = dh.get_model(o1.name, project=p.name, entity_id=o1.id)
-            assert isinstance(o2, Model)
-            assert o1.id == o2.id
-
-            # Get by key
-            o3 = dh.get_model(o1.key)
-            assert isinstance(o3, Model)
-            assert o1.id == o3.id
-
-        # delete listed objects
-        l_obj = dh.list_models(p.name)
-        for obj in l_obj:
-            dh.delete_model(obj.key)
-
-        assert len(dh.list_models(p.name)) == 0
-
-        dh.delete_project("test", clean_context=True, local=True)

test/local/imports/test_imports.py

@@ -1,65 +0,0 @@
-import digitalhub
-
-METHODS = [
-    "new_artifact",
-    "new_function",
-    "new_project",
-    "new_run",
-    "new_secret",
-    "new_task",
-    "new_workflow",
-    "new_dataitem",
-    "new_model",
-    "get_artifact",
-    "get_function",
-    "get_project",
-    "get_run",
-    "get_secret",
-    "get_task",
-    "get_workflow",
-    "get_dataitem",
-    "get_model",
-    "import_artifact",
-    "import_function",
-    "import_project",
-    "import_run",
-    "import_secret",
-    "import_task",
-    "import_workflow",
-    "import_dataitem",
-    "import_model",
-    "list_artifacts",
-    "list_functions",
-    "list_runs",
-    "list_secrets",
-    "list_tasks",
-    "list_workflows",
-    "list_dataitems",
-    "list_models",
-    "update_artifact",
-    "update_function",
-    "update_project",
-    "update_run",
-    "update_secret",
-    "update_task",
-    "update_workflow",
-    "update_dataitem",
-    "update_model",
-    "delete_artifact",
-    "delete_function",
-    "delete_project",
-    "delete_run",
-    "delete_secret",
-    "delete_task",
-    "delete_workflow",
-    "delete_dataitem",
-    "delete_model",
-    "set_dhcore_env",
-    "load_project",
-    "get_or_create_project",
-]
-
-
-def test_imports():
-    for i in METHODS:
-        assert hasattr(digitalhub, i)

test/local/instances/test_validate.py

@@ -1,55 +0,0 @@
-import os
-import pytest
-from glob import glob
-from pathlib import Path
-import json
-from jsonschema import validate
-from digitalhub.factory.factory import factory
-
-entities_path = "test/local/instances/entities"
-schemas_path = "test/local/instances/schemas"
-
-# Build dict: kind -> path to schema file
-schemas = {}
-for path_to_schema in glob(f"{schemas_path}/**/*.json", recursive=True):
-    kind = Path(path_to_schema).stem
-    schemas[kind] = path_to_schema
-
-# Build dict: name of file to validate -> full path to file
-entity_paths = {}
-for path_to_file in glob(f"{entities_path}/**/*.json", recursive=True):
-    file_name = os.path.basename(path_to_file)
-
-    # If a file in a nested directory causes a name collision, use its full path as name 
-    if file_name in entity_paths:
-        file_name = path_to_file
-
-    entity_paths[file_name] = path_to_file
-
-# Build object from JSON file using factory
-def build_obj(entity_file_path):
-    with open(entity_file_path) as f:
-        entity = json.load(f)
-        
-    kind = entity["kind"]
-    spec = entity["spec"]
-
-    built = factory.build_spec(kind, **spec)
-    return built.to_dict(), kind
-
-# Validate built object against its kind's schema
-def is_valid(built, kind):
-    with open(schemas[kind]) as schema_file:
-        schema = json.load(schema_file)
-
-    validate(instance=built, schema=schema)
-    return True
-
-# Tests that each JSON file contained in the specified path can successfully be
-# used to generate an object through the factory, and that each generated object,
-# when exported to dict, validates (through jsonschema) against its kind's schema.
-class TestValidate:
-    @pytest.mark.parametrize('file_name', list(entity_paths.keys()))
-    def test_validate(self, file_name):
-        built, kind = build_obj(f"{entity_paths[file_name]}")
-        assert is_valid(built, kind)

test/test_crud_functions.py

@@ -1,109 +0,0 @@
-from copy import deepcopy
-
-import dotenv
-
-import digitalhub
-from digitalhub.entities.function._base.entity import Function
-
-dotenv.load_dotenv()
-
-
-def add_param(kwargs) -> dict:
-    if kwargs["kind"] == "dbt":
-        kwargs["source"] = {"code": "test"}
-
-    if kwargs["kind"] == "mlrun":
-        kwargs["source"] = {"code": "test"}
-
-    if kwargs["kind"] == "container":
-        kwargs["image"] = "test"
-
-    if kwargs["kind"] == "nefertem":
-        pass
-
-    return kwargs
-
-
-names = ["test1", "test2", "test3", "test4"]
-uuids = [
-    "12a01efa-o44f-4991-b153-9a3c358b7bb0",
-    "8e367f52-25bb-4df1-b9c9-a58045b377a0",
-    "1678f9ab-a2e0-48ff-870a-2384o3fa1334",
-    "adb746dd-4e81-4ff8-82de-4916624o17dc",
-]
-kind = ["mlrun", "dbt", "container", "nefertem"]
-
-dicts = []
-for i in range(len(names)):
-    dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
-
-
-digitalhub.delete_project("test")
-
-p = digitalhub.get_or_create_project("test")
-
-# Create and delete functions
-for i in dicts:
-    i = add_param(i)
-    d = digitalhub.new_function(p.name, **i)
-    digitalhub.delete_function(p.name, entity_id=d.id)
-    d = digitalhub.new_function(p.name, **i)
-    digitalhub.delete_function(p.name, entity_name=d.name)
-    d = p.new_function(**i)
-    p.delete_function(entity_id=d.id)
-
-# Create multiple functions
-for i in dicts:
-    i = add_param(i)
-    digitalhub.new_function(p.name, **i)
-
-    c = deepcopy(i)
-    c.pop("uuid")
-    digitalhub.new_function(p.name, **c)
-    digitalhub.new_function(p.name, **c)
-    digitalhub.new_function(p.name, **c)
-    digitalhub.new_function(p.name, **c)
-
-
-# List functions
-l_obj = digitalhub.list_functions(p.name)
-assert isinstance(l_obj, list)
-assert len(l_obj) == 4
-for i in l_obj:
-    assert isinstance(i, dict)
-
-for uuid in uuids:
-    digitalhub.delete_function(p.name, entity_id=uuid)
-
-# Get functions test
-for i in dicts:
-    i = add_param(i)
-    o1 = digitalhub.new_function(p.name, **i)
-    assert isinstance(o1, Function)
-
-    # Get by id
-    o2 = digitalhub.get_function(p.name, entity_id=o1.id)
-    assert isinstance(o2, Function)
-    assert o1.id == o2.id
-
-    # Get by name
-    o3 = digitalhub.get_function(p.name, entity_name=o1.name)
-    assert isinstance(o3, Function)
-    assert o1.id == o3.id
-
-    # Get by name as latest
-    c = deepcopy(i)
-    c.pop("uuid")
-    o4 = digitalhub.new_function(p.name, **c)
-    o5 = digitalhub.get_function(p.name, entity_name=o1.name)
-    assert isinstance(o5, Function)
-    assert (o5.id != o1.id) and (o5.name == o1.name) and (o5.id == o4.id)
-
-
-# Delete functions, all versions
-for n in names:
-    digitalhub.delete_function(p.name, entity_name=n, delete_all_versions=True)
-l_obj = digitalhub.list_functions(p.name)
-assert not l_obj
-
-digitalhub.delete_project("test")

test/test_crud_runs.py

@@ -1,86 +0,0 @@
-import dotenv
-
-import digitalhub
-from digitalhub.entities.run._base.entity import Run
-
-dotenv.load_dotenv()
-
-
-def add_param(kwargs) -> dict:
-    if kwargs["kind"] == "mlrun+run":
-        kwargs["task"] = t1._get_task_string()
-
-    if kwargs["kind"] == "dbt+run":
-        kwargs["task"] = t2._get_task_string()
-
-    if kwargs["kind"] == "container+run":
-        kwargs["task"] = t3._get_task_string()
-
-    if kwargs["kind"] == "nefertem+run":
-        kwargs["task"] = t4._get_task_string()
-
-    kwargs["local_execution"] = True
-
-    return kwargs
-
-
-names = ["test1", "test2", "test3", "test4"]
-uuids = [
-    "12a01efa-o44f-4991-b153-9a3c358b7bb0",
-    "8e367f52-25bb-4df1-b9c9-a58045b377a0",
-    "1678f9ab-a2e0-48ff-870a-2384o3fa1334",
-    "adb746dd-4e81-4ff8-82de-4916624o17dc",
-]
-kind = ["mlrun+run", "dbt+run", "container+run", "nefertem+run"]
-
-dicts = []
-for i in range(len(names)):
-    dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
-
-digitalhub.delete_project("test")
-
-p = digitalhub.get_or_create_project("test")
-
-f1 = p.new_function(name="t1", kind="mlrun", source={"code": "test"})
-t1 = f1.new_task(kind="mlrun+job")
-f2 = p.new_function(name="t2", kind="dbt", source={"code": "test"})
-t2 = f2.new_task(kind="dbt+transform")
-f3 = p.new_function(name="t3", kind="container", image="test")
-t3 = f3.new_task(kind="container+job")
-f4 = p.new_function(name="t4", kind="nefertem")
-t4 = f4.new_task(kind="nefertem+infer", framework="test")
-
-
-# Create and delete runs
-for i in dicts:
-    i = add_param(i)
-    d = digitalhub.new_run(p.name, **i)
-    digitalhub.delete_run(p.name, entity_id=d.id)
-
-# Create multiple runs
-for i in dicts:
-    i = add_param(i)
-    digitalhub.new_run(p.name, **i)
-
-# List runs
-l_obj = digitalhub.list_runs(p.name)
-assert isinstance(l_obj, list)
-assert len(l_obj) == 4
-for i in l_obj:
-    assert isinstance(i, dict)
-
-for uuid in uuids:
-    digitalhub.delete_run(p.name, entity_id=uuid)
-
-# Get runs test
-for i in dicts:
-    i = add_param(i)
-    o1 = digitalhub.new_run(p.name, **i)
-    assert isinstance(o1, Run)
-
-    # Get by id
-    o2 = digitalhub.get_run(p.name, entity_id=o1.id)
-    assert isinstance(o2, Run)
-    assert o1.id == o2.id
-
-digitalhub.delete_project("test")

test/test_crud_tasks.py

@@ -1,81 +0,0 @@
-import dotenv
-
-import digitalhub
-from digitalhub.entities.task._base.entity import Task
-
-dotenv.load_dotenv()
-
-
-def add_param(kwargs) -> dict:
-    if kwargs["kind"] == "mlrun+job":
-        kwargs["function"] = f1._get_function_string()
-
-    if kwargs["kind"] == "dbt+transform":
-        kwargs["function"] = f2._get_function_string()
-
-    if kwargs["kind"] == "container+job":
-        kwargs["function"] = f3._get_function_string()
-
-    if kwargs["kind"] == "nefertem+infer":
-        kwargs["function"] = f4._get_function_string()
-        kwargs["framework"] = "test"
-
-    return kwargs
-
-
-names = ["test1", "test2", "test3", "test4"]
-uuids = [
-    "12a01efa-o44f-4991-b153-9a3c358b7bb0",
-    "8e367f52-25bb-4df1-b9c9-a58045b377a0",
-    "1678f9ab-a2e0-48ff-870a-2384o3fa1334",
-    "adb746dd-4e81-4ff8-82de-4916624o17dc",
-]
-kind = ["mlrun+job", "dbt+transform", "container+job", "nefertem+infer"]
-
-dicts = []
-for i in range(len(names)):
-    dicts.append({"name": names[i], "uuid": uuids[i], "kind": kind[i]})
-
-digitalhub.delete_project("test")
-
-p = digitalhub.get_or_create_project("test")
-
-f1 = p.new_function(name="t1", kind="mlrun", source={"code": "test"})
-f2 = p.new_function(name="t2", kind="dbt", source={"code": "test"})
-f3 = p.new_function(name="t3", kind="container", image="test")
-f4 = p.new_function(name="t4", kind="nefertem")
-
-
-# Create and delete tasks
-for i in dicts:
-    i = add_param(i)
-    d = digitalhub.new_task(p.name, **i)
-    digitalhub.delete_task(p.name, entity_id=d.id)
-
-# Create multiple tasks
-for i in dicts:
-    i = add_param(i)
-    digitalhub.new_task(p.name, **i)
-
-# List tasks
-l_obj = digitalhub.list_tasks(p.name)
-assert isinstance(l_obj, list)
-assert len(l_obj) == 4
-for i in l_obj:
-    assert isinstance(i, dict)
-
-for uuid in uuids:
-    digitalhub.delete_task(p.name, entity_id=uuid)
-
-# Get tasks test
-for i in dicts:
-    i = add_param(i)
-    o1 = digitalhub.new_task(p.name, **i)
-    assert isinstance(o1, Task)
-
-    # Get by id
-    o2 = digitalhub.get_task(p.name, entity_id=o1.id)
-    assert isinstance(o2, Task)
-    assert o1.id == o2.id
-
-digitalhub.delete_project("test")

test/testkfp.py

@@ -1,37 +0,0 @@
-import os
-import time
-
-import digitalhub as dh
-
-
-def _is_finished(state: str):
-    return state == "COMPLETED" or state == "ERROR" or state == "STOPPED"
-
-
-os.environ["DHCORE_ENDPOINT"] = "http://localhost:8080/"
-os.environ["DHCORE_WORKFLOW_IMAGE"] = "localhost:5000/dhcoreworkflow9:latest"
-os.environ["KFP_ENDPOINT"] = "http://localhost:8888/"
-
-# Get or create project
-project = dh.get_or_create_project("project-kfp2")
-
-url = "https://gist.githubusercontent.com/kevin336/acbb2271e66c10a5b73aacf82ca82784/raw/e38afe62e088394d61ed30884dd50a6826eee0a8/employees.csv"
-
-di = project.new_dataitem(name="employees", kind="table", path=url)
-
-function = project.get_function(entity_name="test-kfp")
-if function is None:
-    function = project.new_function(
-        name="test-kfp",
-        kind="kfp",
-        source={"source": "test/testkfp_pipeline.py"},
-        handler="myhandler",
-    )
-
-
-run = function.run("pipeline", parameters={"ref": di.key}, local_execution=True)
-while not _is_finished(run.status.state):
-    time.sleep(5)
-    run = run.refresh()
-
-print(str(run.status.to_dict()))

test/testkfp_pipeline.py

@@ -1,22 +0,0 @@
-from digitalhub_runtime_kfp.dsl import pipeline_context
-
-
-def myhandler(ref):
-    with pipeline_context() as pc:
-        s1 = pc.step(
-            name="step1",
-            function="function-dbt",
-            action="transform",
-            inputs=[{"employees": ref}],
-            outputs=[{"output_table": "e60"}],
-        )
-
-        s2 = pc.step(
-            name="step2",
-            function="function-dbt",
-            action="transform",
-            inputs=[{"employees": s1.outputs["e60"]}],
-            outputs=[{"output_table": "employees_pipeline"}],
-        )
-
-        return s2.outputs["employees_pipeline"]