digitalhub 0.8.0__py3-none-any.whl → 0.8.0b1__py3-none-any.whl

digitalhub/entities/dataitem/crud.py

@@ -5,24 +5,24 @@ from pathlib import Path
 from typing import Any
 from urllib.parse import urlparse
 
+from digitalhub.context.builder import check_context
 from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_material_entity,
-    get_material_entity_versions,
-    import_context_entity,
-    list_material_entities,
-    new_context_entity,
+    delete_entity_api_ctx,
+    list_entity_api_ctx,
+    read_entity_api_ctx,
+    read_entity_api_ctx_versions,
 )
-from digitalhub.entities._base.entity._constructors.uuid import build_uuid
-from digitalhub.entities.utils.entity_types import EntityTypes
-from digitalhub.entities.utils.utils import build_log_path_from_filename, build_log_path_from_source, eval_local_source
-from digitalhub.factory.api import build_entity_from_params
-from digitalhub.readers.api import get_reader_by_object
-from digitalhub.stores.api import get_store
-from digitalhub.utils.generic_utils import slugify_string
+from digitalhub.entities._builders.uuid import build_uuid
+from digitalhub.entities.dataitem.builder import dataitem_from_dict, dataitem_from_parameters
+from digitalhub.entities.entity_types import EntityTypes
+from digitalhub.entities.utils import build_log_path_from_filename, build_log_path_from_source, eval_local_source
+from digitalhub.readers.builder import get_reader_by_object
+from digitalhub.stores.builder import get_store
+from digitalhub.utils.exceptions import EntityAlreadyExistsError
+from digitalhub.utils.io_utils import read_yaml
 
 if typing.TYPE_CHECKING:
-    from digitalhub.entities.dataitem._base.entity import Dataitem
+    from digitalhub.entities.dataitem.entity._base import Dataitem
 
 
 ENTITY_TYPE = EntityTypes.DATAITEM.value
@@ -35,7 +35,7 @@ def new_dataitem(
     uuid: str | None = None,
     description: str | None = None,
     labels: list[str] | None = None,
-    embedded: bool = False,
+    embedded: bool = True,
     path: str | None = None,
     **kwargs,
 ) -> Dataitem:
@@ -51,7 +51,7 @@ def new_dataitem(
     kind : str
         Kind the object.
     uuid : str
-        ID of the object.
+        ID of the object (UUID4, e.g. 40f25c4b-d26b-4221-b048-9527aff291e2).
     description : str
         Description of the object (human readable).
     labels : list[str]
@@ -75,7 +75,8 @@ def new_dataitem(
     >>>                    kind="dataitem",
     >>>                    path="s3://my-bucket/my-key")
     """
-    return new_context_entity(
+    check_context(project)
+    obj = dataitem_from_parameters(
         project=project,
         name=name,
         kind=kind,
@@ -86,6 +87,8 @@ def new_dataitem(
         path=path,
         **kwargs,
     )
+    obj.save()
+    return obj
 
 
 def log_dataitem(
@@ -149,14 +152,12 @@ def log_dataitem(
 
     # Case where data is provided
     else:
-        extension = extension if extension is not None else "parquet"
         if path is None:
             uuid = build_uuid()
             kwargs["uuid"] = uuid
-            slug = slugify_string(name) + f".{extension}"
-            path = build_log_path_from_filename(project, ENTITY_TYPE, name, uuid, slug)
+            path = build_log_path_from_filename(project, ENTITY_TYPE, name, uuid, "data.parquet")
 
-        obj = build_entity_from_params(project=project, name=name, kind=kind, path=path, **kwargs)
+        obj = dataitem_from_parameters(project=project, name=name, kind=kind, path=path, **kwargs)
         if kind == "table":
             dst = obj.write_df(df=data, extension=extension)
             reader = get_reader_by_object(data)
@@ -207,13 +208,16 @@ def get_dataitem(
     >>>                    project="my-project",
     >>>                    entity_id="my-dataitem-id")
     """
-    return get_material_entity(
-        identifier=identifier,
-        entity_type=ENTITY_TYPE,
+    obj = read_entity_api_ctx(
+        identifier,
+        ENTITY_TYPE,
         project=project,
         entity_id=entity_id,
         **kwargs,
     )
+    entity = dataitem_from_dict(obj)
+    entity._get_files_info()
+    return entity
 
 
 def get_dataitem_versions(
@@ -247,12 +251,18 @@ def get_dataitem_versions(
     >>> objs = get_dataitem_versions("my-dataitem-name",
     >>>                              project="my-project")
     """
-    return get_material_entity_versions(
-        identifier=identifier,
+    objs = read_entity_api_ctx_versions(
+        identifier,
         entity_type=ENTITY_TYPE,
         project=project,
         **kwargs,
     )
+    objects = []
+    for o in objs:
+        entity = dataitem_from_dict(o)
+        entity._get_files_info()
+        objects.append(entity)
+    return objects
 
 
 def list_dataitems(project: str, **kwargs) -> list[Dataitem]:
@@ -275,11 +285,17 @@ def list_dataitems(project: str, **kwargs) -> list[Dataitem]:
     --------
     >>> objs = list_dataitems(project="my-project")
     """
-    return list_material_entities(
+    objs = list_entity_api_ctx(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
     )
+    objects = []
+    for o in objs:
+        entity = dataitem_from_dict(o)
+        entity._get_files_info()
+        objects.append(entity)
+    return objects
 
 
 def import_dataitem(file: str) -> Dataitem:
@@ -300,7 +316,14 @@ def import_dataitem(file: str) -> Dataitem:
     --------
     >>> obj = import_dataitem("my-dataitem.yaml")
     """
-    return import_context_entity(file)
+    dict_obj: dict = read_yaml(file)
+    obj = dataitem_from_dict(dict_obj)
+    try:
+        obj.save()
+    except EntityAlreadyExistsError:
+        pass
+    finally:
+        return obj
 
 
 def update_dataitem(entity: Dataitem) -> Dataitem:
@@ -362,7 +385,7 @@ def delete_dataitem(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return delete_entity(
+    return delete_entity_api_ctx(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/dataitem/{_base/entity.py → entity/_base.py}

@@ -3,15 +3,15 @@ from __future__ import annotations
 import typing
 from pathlib import Path
 
-from digitalhub.entities._base.material.entity import MaterialEntity
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities._base.entity.material import MaterialEntity
+from digitalhub.entities.entity_types import EntityTypes
 from digitalhub.utils.exceptions import EntityError
 from digitalhub.utils.uri_utils import map_uri_scheme
 
 if typing.TYPE_CHECKING:
-    from digitalhub.entities._base.entity.metadata import Metadata
-    from digitalhub.entities.dataitem._base.spec import DataitemSpec
-    from digitalhub.entities.dataitem._base.status import DataitemStatus
+    from digitalhub.entities._base.metadata import Metadata
+    from digitalhub.entities.dataitem.spec import DataitemSpec
+    from digitalhub.entities.dataitem.status import DataitemStatus
 
 
 class Dataitem(MaterialEntity):

digitalhub/entities/dataitem/entity/dataitem.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem.entity._base import Dataitem
+
+
+class DataitemDataitem(Dataitem):
+    """
+    Dataitem dataitem.
+    """

digitalhub/entities/dataitem/entity/iceberg.py

@@ -0,0 +1,7 @@
+from digitalhub.entities.dataitem.entity._base import Dataitem
+
+
+class DataitemIceberg(Dataitem):
+    """
+    Iceberg dataitem.
+    """

digitalhub/entities/dataitem/{table/entity.py → entity/table.py}

@@ -1,41 +1,20 @@
 from __future__ import annotations
 
 import shutil
-import typing
 from pathlib import Path
 from typing import Any
 
-from digitalhub.datastores.api import get_datastore
-from digitalhub.entities.dataitem._base.entity import Dataitem
+from digitalhub.datastores.builder import get_datastore
+from digitalhub.entities.dataitem.entity._base import Dataitem
 from digitalhub.utils.uri_utils import check_local_path
 
-if typing.TYPE_CHECKING:
-    from digitalhub.entities._base.entity.metadata import Metadata
-    from digitalhub.entities.dataitem.table.spec import DataitemSpecTable
-    from digitalhub.entities.dataitem.table.status import DataitemStatusTable
-
 
 class DataitemTable(Dataitem):
+
     """
-    DataitemTable class.
+    Table dataitem.
     """
 
-    def __init__(
-        self,
-        project: str,
-        name: str,
-        uuid: str,
-        kind: str,
-        metadata: Metadata,
-        spec: DataitemSpecTable,
-        status: DataitemStatusTable,
-        user: str | None = None,
-    ) -> None:
-        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
-
-        self.spec: DataitemSpecTable
-        self.status: DataitemStatusTable
-
     def as_df(
         self,
         file_format: str | None = None,

digitalhub/entities/dataitem/spec.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from pydantic import Field
+
+from digitalhub.entities._base.spec.material import MaterialParams, MaterialSpec
+from digitalhub.entities.dataitem.models import TableSchema
+
+
+class DataitemSpec(MaterialSpec):
+    """
+    Dataitem specifications.
+    """
+
+
+class DataitemParams(MaterialParams):
+    """
+    Dataitem parameters.
+    """
+
+
+class DataitemSpecDataitem(DataitemSpec):
+    """
+    Dataitem dataitem specifications.
+    """
+
+
+class DataitemParamsDataitem(DataitemParams):
+    """
+    Dataitem dataitem parameters.
+    """
+
+
+class DataitemSpecTable(DataitemSpec):
+    """
+    Dataitem table specifications.
+    """
+
+    def __init__(self, path: str, schema: dict | None = None) -> None:
+        super().__init__(path)
+        self.schema = schema
+
+
+class DataitemParamsTable(DataitemParams):
+    """
+    Dataitem table parameters.
+    """
+
+    schema_: TableSchema = Field(default=None, alias="schema")
+    """The schema of the dataitem in table schema format."""
+
+
+class DataitemSpecIceberg(DataitemSpec):
+    """
+    Dataitem iceberg specifications.
+    """
+
+
+class DataitemParamsIceberg(DataitemParams):
+    """
+    Dataitem iceberg parameters.
+    """

digitalhub/entities/dataitem/status.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from digitalhub.entities._base.status.material import MaterialStatus
+
+
+class DataitemStatus(MaterialStatus):
+    """
+    Status class for dataitem entities.
+    """
+
+    def __init__(
+        self,
+        state: str,
+        message: str | None = None,
+        files: list[dict] | None = None,
+        preview: dict | None = None,
+        **kwargs,
+    ) -> None:
+        super().__init__(state, message, files)
+        self.preview = preview
+
+
+class DataitemStatusDataitem(DataitemStatus):
+    """
+    Status class for dataitem dataitem entities.
+    """
+
+
+class DataitemStatusTable(DataitemStatus):
+    """
+    Status class for dataitem table entities.
+    """
+
+
+class DataitemStatusIceberg(DataitemStatus):
+    """
+    Status class for dataitem iceberg entities.
+    """

digitalhub/entities/function/builder.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+from digitalhub.entities._builders.metadata import build_metadata
+from digitalhub.entities._builders.name import build_name
+from digitalhub.entities._builders.spec import build_spec
+from digitalhub.entities._builders.status import build_status
+from digitalhub.entities._builders.uuid import build_uuid
+from digitalhub.entities.function.entity import Function
+
+
+def function_from_parameters(
+    project: str,
+    name: str,
+    kind: str,
+    uuid: str | None = None,
+    description: str | None = None,
+    labels: list[str] | None = None,
+    embedded: bool = True,
+    **kwargs,
+) -> Function:
+    """
+    Create a new Function instance and persist it to the backend.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    name : str
+        Object name.
+    kind : str
+        Kind the object.
+    uuid : str
+        ID of the object (UUID4, e.g. 40f25c4b-d26b-4221-b048-9527aff291e2).
+    description : str
+        Description of the object (human readable).
+    labels : list[str]
+        List of labels.
+    embedded : bool
+        Flag to determine if object spec must be embedded in project spec.
+    **kwargs : dict
+        Spec keyword arguments.
+
+    Returns
+    -------
+    Function
+        Object instance.
+    """
+    name = build_name(name)
+    uuid = build_uuid(uuid)
+    spec = build_spec(kind, **kwargs)
+    metadata = build_metadata(
+        kind,
+        project=project,
+        name=name,
+        version=uuid,
+        description=description,
+        labels=labels,
+        embedded=embedded,
+    )
+    status = build_status(kind)
+    return Function(
+        project=project,
+        name=name,
+        uuid=uuid,
+        kind=kind,
+        metadata=metadata,
+        spec=spec,
+        status=status,
+    )
+
+
+def function_from_dict(obj: dict) -> Function:
+    """
+    Create a new object from dictionary.
+
+    Parameters
+    ----------
+    obj : dict
+        Dictionary to create object from.
+
+    Returns
+    -------
+    Function
+        Function object.
+    """
+    return Function.from_dict(obj)

digitalhub/entities/function/crud.py

@@ -2,18 +2,20 @@ from __future__ import annotations
 
 import typing
 
+from digitalhub.context.builder import check_context
 from digitalhub.entities._base.crud import (
-    delete_entity,
-    get_context_entity_versions,
-    get_versioned_entity,
-    import_executable_entity,
-    list_context_entities,
-    new_context_entity,
+    delete_entity_api_ctx,
+    list_entity_api_ctx,
+    read_entity_api_ctx,
+    read_entity_api_ctx_versions,
 )
-from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities.entity_types import EntityTypes
+from digitalhub.entities.function.builder import function_from_dict, function_from_parameters
+from digitalhub.utils.exceptions import EntityAlreadyExistsError
+from digitalhub.utils.io_utils import read_yaml
 
 if typing.TYPE_CHECKING:
-    from digitalhub.entities.function._base.entity import Function
+    from digitalhub.entities.function.entity import Function
 
 ENTITY_TYPE = EntityTypes.FUNCTION.value
 
@@ -25,7 +27,7 @@ def new_function(
     uuid: str | None = None,
     description: str | None = None,
     labels: list[str] | None = None,
-    embedded: bool = False,
+    embedded: bool = True,
     **kwargs,
 ) -> Function:
     """
@@ -40,7 +42,7 @@ def new_function(
     kind : str
         Kind the object.
     uuid : str
-        ID of the object.
+        ID of the object (UUID4, e.g. 40f25c4b-d26b-4221-b048-9527aff291e2).
     description : str
         Description of the object (human readable).
     labels : list[str]
@@ -63,7 +65,8 @@ def new_function(
     >>>                    code_src="function.py",
     >>>                    handler="function-handler")
     """
-    return new_context_entity(
+    check_context(project)
+    obj = function_from_parameters(
         project=project,
         name=name,
         kind=kind,
@@ -73,6 +76,8 @@ def new_function(
         embedded=embedded,
         **kwargs,
     )
+    obj.save()
+    return obj
 
 
 def get_function(
@@ -110,13 +115,14 @@ def get_function(
     >>>                    project="my-project",
     >>>                    entity_id="my-function-id")
     """
-    return get_versioned_entity(
+    obj = read_entity_api_ctx(
         identifier,
-        entity_type=ENTITY_TYPE,
+        ENTITY_TYPE,
         project=project,
         entity_id=entity_id,
         **kwargs,
     )
+    return function_from_dict(obj)
 
 
 def get_function_versions(
@@ -150,12 +156,13 @@ def get_function_versions(
     >>> obj = get_function_versions("my-function-name"
     >>>                             project="my-project")
     """
-    return get_context_entity_versions(
+    obj = read_entity_api_ctx_versions(
         identifier,
         entity_type=ENTITY_TYPE,
         project=project,
         **kwargs,
     )
+    return [function_from_dict(o) for o in obj]
 
 
 def list_functions(project: str, **kwargs) -> list[Function]:
@@ -178,11 +185,12 @@ def list_functions(project: str, **kwargs) -> list[Function]:
     --------
     >>> objs = list_functions(project="my-project")
     """
-    return list_context_entities(
+    objs = list_entity_api_ctx(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
     )
+    return [function_from_dict(obj) for obj in objs]
 
 
 def import_function(file: str) -> Function:
@@ -203,7 +211,25 @@ def import_function(file: str) -> Function:
     --------
     >>> obj = import_function("my-function.yaml")
     """
-    return import_executable_entity(file)
+    dict_obj: dict | list[dict] = read_yaml(file)
+    if isinstance(dict_obj, list):
+        fnc_dict = dict_obj[0]
+        tsk_dicts = dict_obj[1:]
+    else:
+        fnc_dict = dict_obj
+        tsk_dicts = []
+
+    check_context(fnc_dict.get("project"))
+    obj = function_from_dict(fnc_dict)
+
+    obj.import_tasks(tsk_dicts)
+
+    try:
+        obj.save()
+    except EntityAlreadyExistsError:
+        pass
+    finally:
+        return obj
 
 
 def update_function(entity: Function) -> Function:
@@ -268,7 +294,7 @@ def delete_function(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return delete_entity(
+    return delete_entity_api_ctx(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/function/{_base/entity.py → entity.py}

@@ -3,16 +3,16 @@ from __future__ import annotations
 import typing
 from concurrent.futures import ThreadPoolExecutor
 
-from digitalhub.entities._base.executable.entity import ExecutableEntity
-from digitalhub.entities.utils.entity_types import EntityTypes
-from digitalhub.factory.api import get_run_kind, get_task_kind_from_action
+from digitalhub.entities._base.entity.executable import ExecutableEntity
+from digitalhub.entities.entity_types import EntityTypes
+from digitalhub.runtimes.builder import get_kind_registry
 from digitalhub.utils.exceptions import BackendError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.entities._base.entity.metadata import Metadata
-    from digitalhub.entities.function._base.spec import FunctionSpec
-    from digitalhub.entities.function._base.status import FunctionStatus
-    from digitalhub.entities.run._base.entity import Run
+    from digitalhub.entities._base.metadata import Metadata
+    from digitalhub.entities.function.spec import FunctionSpec
+    from digitalhub.entities.function.status import FunctionStatus
+    from digitalhub.entities.run.entity import Run
 
 
 class Function(ExecutableEntity):
@@ -71,9 +71,12 @@ class Function(ExecutableEntity):
         Run
             Run instance.
         """
+        # Get kind registry
+        kind_reg = get_kind_registry(self.kind)
+
         # Get task and run kind
-        task_kind = get_task_kind_from_action(self.kind, action)
-        run_kind = get_run_kind(self.kind)
+        task_kind = kind_reg.get_task_kind_from_action(action=action)
+        run_kind = kind_reg.get_run_kind()
 
         # Create or update new task
         task = self._get_or_create_task(task_kind)

digitalhub/entities/function/{_base/models.py → models.py}

@@ -94,7 +94,7 @@ class SourceCodeStruct:
         return str(self.to_dict())
 
 
-class SourceCodeValidator(BaseModel):
+class SourceCodeParams(BaseModel):
     """
     Source code params.
     """