digitalhub 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

digitalhub/entities/_processors/utils.py

@@ -0,0 +1,158 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.client.api import get_client
+from digitalhub.context.api import get_context
+from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, EntityTypes
+from digitalhub.entities._commons.utils import get_project_from_key, parse_entity_key
+from digitalhub.factory.api import build_entity_from_dict
+from digitalhub.utils.exceptions import ContextError, EntityError, EntityNotExistsError
+
+if typing.TYPE_CHECKING:
+    from digitalhub.client._base.client import Client
+    from digitalhub.context.context import Context
+
+
+def parse_identifier(
+    identifier: str,
+    project: str | None = None,
+    entity_type: str | None = None,
+    entity_kind: str | None = None,
+    entity_id: str | None = None,
+) -> tuple[str, str, str | None, str | None, str | None]:
+    """
+    Parse entity identifier.
+
+    Parameters
+    ----------
+    identifier : str
+        Entity key (store://...) or entity name.
+    project : str
+        Project name.
+    entity_type : str
+        Entity type.
+    entity_id : str
+        Entity ID.
+
+    Returns
+    -------
+    tuple[str, str, str | None, str | None, str | None]
+        Project name, entity type, entity kind, entity name, entity ID.
+    """
+    if not identifier.startswith("store://"):
+        if project is None or entity_type is None:
+            raise ValueError("Project and entity type must be specified.")
+        return project, entity_type, entity_kind, identifier, entity_id
+    return parse_entity_key(identifier)
+
+
+def get_context_from_identifier(
+    identifier: str,
+    project: str | None = None,
+) -> Context:
+    """
+    Get context from project.
+
+    Parameters
+    ----------
+    identifier : str
+        Entity key (store://...) or entity name.
+    project : str
+        Project name.
+
+    Returns
+    -------
+    Context
+        Context.
+    """
+    if not identifier.startswith("store://"):
+        if project is None:
+            raise EntityError("Specify project if you do not specify entity key.")
+    else:
+        project = get_project_from_key(identifier)
+
+    return get_context_from_project(project)
+
+
+def get_context_from_project(
+    project: str,
+) -> Context:
+    """
+    Check if the given project is in the context.
+    Otherwise try to get the project from remote.
+    Finally return the client.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+
+    Returns
+    -------
+    Context
+        Context.
+    """
+    try:
+        return get_context(project)
+    except ContextError:
+        return get_context_from_remote(project)
+
+
+def get_context_from_remote(
+    project: str,
+) -> Client:
+    """
+    Get context from remote.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+
+    Returns
+    -------
+    Client
+        Client.
+    """
+    try:
+        client = get_client()
+        obj = _read_base_entity(client, EntityTypes.PROJECT.value, project)
+        build_entity_from_dict(obj)
+        return get_context(project)
+    except EntityNotExistsError:
+        raise ContextError(f"Project '{project}' not found.")
+
+
+def _read_base_entity(
+    client: Client,
+    entity_type: str,
+    entity_name: str,
+    **kwargs,
+) -> dict:
+    """
+    Read object from backend.
+
+    Parameters
+    ----------
+    client : Client
+        Client instance.
+    entity_type : str
+        Entity type.
+    entity_name : str
+        Entity name.
+    **kwargs : dict
+        Parameters to pass to the API call.
+
+    Returns
+    -------
+    dict
+        Object instance.
+    """
+    api = client.build_api(
+        ApiCategories.BASE.value,
+        BackendOperations.READ.value,
+        entity_type=entity_type,
+        entity_name=entity_name,
+    )
+    return client.read_object(api, **kwargs)

digitalhub/entities/artifact/artifact/spec.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import Optional
+
 from digitalhub.entities.artifact._base.spec import ArtifactSpec, ArtifactValidator
 
 
@@ -23,5 +25,5 @@ class ArtifactValidatorArtifact(ArtifactValidator):
     ArtifactValidatorArtifact validator.
     """
 
-    src_path: str = None
+    src_path: Optional[str] = None
     """Source path of the artifact."""

digitalhub/entities/artifact/crud.py

@@ -3,9 +3,10 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
+from digitalhub.entities._processors.context import context_processor
 from digitalhub.entities.artifact._base.entity import Artifact
 from digitalhub.entities.artifact.utils import eval_source, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.artifact._base.entity import Artifact
@@ -61,7 +62,7 @@ def new_artifact(
     >>>                    kind="artifact",
     >>>                    path="s3://my-bucket/my-key")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -78,7 +79,7 @@ def log_artifact(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str,
+    source: SourcesOrListOfSources,
     path: str | None = None,
     **kwargs,
 ) -> Artifact:
@@ -93,7 +94,7 @@ def log_artifact(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Artifact location on local path.
     path : str
         Destination path of the artifact. If not provided, it's generated.
@@ -114,7 +115,7 @@ def log_artifact(
     """
     eval_source(source)
     kwargs = process_kwargs(project, name, source=source, path=path, **kwargs)
-    return processor.log_material_entity(
+    return context_processor.log_material_entity(
         source=source,
         project=project,
         name=name,
@@ -158,7 +159,7 @@ def get_artifact(
     >>>                    project="my-project",
     >>>                    entity_id="my-artifact-id")
     """
-    return processor.read_material_entity(
+    return context_processor.read_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -198,7 +199,7 @@ def get_artifact_versions(
     >>> obj = get_artifact_versions("my-artifact-name"
     >>>                             project="my-project")
     """
-    return processor.read_material_entity_versions(
+    return context_processor.read_context_entity_versions(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -226,7 +227,7 @@ def list_artifacts(project: str, **kwargs) -> list[Artifact]:
     --------
     >>> objs = list_artifacts(project="my-project")
     """
-    return processor.list_material_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -251,7 +252,7 @@ def import_artifact(file: str) -> Artifact:
     --------
     >>> obj = import_artifact("my-artifact.yaml")
     """
-    return processor.import_context_entity(file)
+    return context_processor.import_context_entity(file)
 
 
 def load_artifact(file: str) -> Artifact:
@@ -272,7 +273,7 @@ def load_artifact(file: str) -> Artifact:
     --------
     >>> obj = load_artifact("my-artifact.yaml")
     """
-    return processor.load_context_entity(file)
+    return context_processor.load_context_entity(file)
 
 
 def update_artifact(entity: Artifact) -> Artifact:
@@ -293,7 +294,7 @@ def update_artifact(entity: Artifact) -> Artifact:
     --------
     >>> obj = update_artifact(obj)
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -339,7 +340,7 @@ def delete_artifact(
     >>>                  project="my-project",
     >>>                  delete_all_versions=True)
     """
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/artifact/utils.py

@@ -56,7 +56,7 @@ def process_kwargs(
     if path is None:
         uuid = build_uuid()
         kwargs["uuid"] = uuid
-        kwargs["path"] = build_log_path_from_source(project, EntityTypes.MODEL.value, name, uuid, source)
+        kwargs["path"] = build_log_path_from_source(project, EntityTypes.ARTIFACT.value, name, uuid, source)
     else:
         kwargs["path"] = path
     return kwargs

digitalhub/entities/builders.py

@@ -7,7 +7,7 @@ from digitalhub.entities.model.mlflow.builder import ModelModelBuilder
 from digitalhub.entities.project._base.builder import ProjectProjectBuilder
 from digitalhub.entities.secret._base.builder import SecretSecretBuilder
 
-entity_builders = (
+entity_builders: tuple = (
     (ProjectProjectBuilder.ENTITY_KIND, ProjectProjectBuilder),
     (SecretSecretBuilder.ENTITY_KIND, SecretSecretBuilder),
     (ArtifactArtifactBuilder.ENTITY_KIND, ArtifactArtifactBuilder),
@@ -17,17 +17,14 @@ entity_builders = (
 )
 
 ##############################
-#  Potential uninstalled entities
+#  Add custom entities here
 ##############################
 
 
 try:
     from digitalhub.entities.dataitem.iceberg.builder import DataitemIcebergBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (DataitemIcebergBuilder.ENTITY_KIND, DataitemIcebergBuilder),
-    )
+    entity_builders += ((DataitemIcebergBuilder.ENTITY_KIND, DataitemIcebergBuilder),)
 except ImportError:
     ...
 
@@ -35,29 +32,20 @@ except ImportError:
 try:
     from digitalhub.entities.model.model.builder import ModelMlflowBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelMlflowBuilder.ENTITY_KIND, ModelMlflowBuilder),
-    )
+    entity_builders += ((ModelMlflowBuilder.ENTITY_KIND, ModelMlflowBuilder),)
 except ImportError:
     ...
 
 try:
     from digitalhub.entities.model.sklearn.builder import ModelSklearnBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelSklearnBuilder.ENTITY_KIND, ModelSklearnBuilder),
-    )
+    entity_builders += ((ModelSklearnBuilder.ENTITY_KIND, ModelSklearnBuilder),)
 except ImportError:
     ...
 
 try:
     from digitalhub.entities.model.huggingface.builder import ModelHuggingfaceBuilder
 
-    entity_builders = (
-        *entity_builders,
-        (ModelHuggingfaceBuilder.ENTITY_KIND, ModelHuggingfaceBuilder),
-    )
+    entity_builders += ((ModelHuggingfaceBuilder.ENTITY_KIND, ModelHuggingfaceBuilder),)
 except ImportError:
     ...

digitalhub/entities/dataitem/_base/entity.py

@@ -1,13 +1,9 @@
 from __future__ import annotations
 
 import typing
-from pathlib import Path
 
 from digitalhub.entities._base.material.entity import MaterialEntity
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.readers._commons.enums import Extensions
-from digitalhub.utils.exceptions import EntityError
-from digitalhub.utils.uri_utils import has_sql_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -36,40 +32,3 @@ class Dataitem(MaterialEntity):
         super().__init__(project, name, uuid, kind, metadata, spec, status, user)
         self.spec: DataitemSpec
         self.status: DataitemStatus
-
-    ##############################
-    #  Helper methods
-    ##############################
-
-    @staticmethod
-    def _get_extension(path: str, file_format: str | None = None) -> str:
-        """
-        Get extension of path.
-
-        Parameters
-        ----------
-        path : str
-            Path to get extension from.
-        file_format : str
-            File format.
-
-        Returns
-        -------
-        str
-            File extension.
-
-        Raises
-        ------
-        EntityError
-            If file format is not supported.
-        """
-        if file_format is not None:
-            return file_format
-
-        if has_sql_scheme(path):
-            return Extensions.PARQUET.value
-
-        ext = Path(path).suffix[1:]
-        if ext is not None:
-            return ext
-        raise EntityError("Unknown file format. Only csv and parquet are supported.")

digitalhub/entities/dataitem/crud.py

@@ -4,8 +4,9 @@ import typing
 from typing import Any
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._operations.processor import processor
-from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_source, post_process, process_kwargs
+from digitalhub.entities._processors.context import context_processor
+from digitalhub.entities.dataitem.utils import clean_tmp_path, eval_data, eval_source, post_process, process_kwargs
+from digitalhub.utils.types import SourcesOrListOfSources
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.dataitem._base.entity import Dataitem
@@ -61,7 +62,7 @@ def new_dataitem(
     >>>                    kind="dataitem",
     >>>                    path="s3://my-bucket/my-key")
     """
-    return processor.create_context_entity(
+    return context_processor.create_context_entity(
         project=project,
         name=name,
         kind=kind,
@@ -78,9 +79,11 @@ def log_dataitem(
     project: str,
     name: str,
     kind: str,
-    source: list[str] | str | None = None,
+    source: SourcesOrListOfSources | None = None,
     data: Any | None = None,
     path: str | None = None,
+    file_format: str | None = None,
+    engine: str | None = "pandas",
     **kwargs,
 ) -> Dataitem:
     """
@@ -94,12 +97,16 @@ def log_dataitem(
         Object name.
     kind : str
         Kind the object.
-    source : str
+    source : SourcesOrListOfSources
         Dataitem location on local path.
     data : Any
         Dataframe to log. Alternative to source.
     path : str
         Destination path of the dataitem. If not provided, it's generated.
+    file_format : str
+        Extension of the file.
+    engine : str
+        Dataframe engine (pandas, polars, etc.).
     **kwargs : dict
         New dataitem spec parameters.
 
@@ -115,17 +122,22 @@ def log_dataitem(
     >>>                    kind="table",
     >>>                    data=df)
     """
+    cleanup = False
+    if data is not None:
+        cleanup = True
+
     source = eval_source(source, data, kind, name, project)
+    data = eval_data(project, kind, source, data, file_format, engine)
     kwargs = process_kwargs(project, name, kind, source=source, data=data, path=path, **kwargs)
-    obj = processor.log_material_entity(
+    obj = context_processor.log_material_entity(
         source=source,
         project=project,
         name=name,
         kind=kind,
         **kwargs,
     )
-    if data is not None:
-        obj = post_process(obj, data)
+    obj = post_process(obj, data)
+    if cleanup:
         clean_tmp_path(source)
     return obj
 
@@ -165,7 +177,7 @@ def get_dataitem(
     >>>                    project="my-project",
     >>>                    entity_id="my-dataitem-id")
     """
-    return processor.read_material_entity(
+    return context_processor.read_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -205,7 +217,7 @@ def get_dataitem_versions(
     >>> objs = get_dataitem_versions("my-dataitem-name",
     >>>                              project="my-project")
     """
-    return processor.read_material_entity_versions(
+    return context_processor.read_context_entity_versions(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
@@ -233,7 +245,7 @@ def list_dataitems(project: str, **kwargs) -> list[Dataitem]:
     --------
     >>> objs = list_dataitems(project="my-project")
     """
-    return processor.list_material_entities(
+    return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
         **kwargs,
@@ -258,7 +270,7 @@ def import_dataitem(file: str) -> Dataitem:
     --------
     >>> obj = import_dataitem("my-dataitem.yaml")
     """
-    return processor.import_context_entity(file)
+    return context_processor.import_context_entity(file)
 
 
 def load_dataitem(file: str) -> Dataitem:
@@ -279,7 +291,7 @@ def load_dataitem(file: str) -> Dataitem:
     --------
     >>> obj = load_dataitem("my-dataitem.yaml")
     """
-    return processor.load_context_entity(file)
+    return context_processor.load_context_entity(file)
 
 
 def update_dataitem(entity: Dataitem) -> Dataitem:
@@ -300,7 +312,7 @@ def update_dataitem(entity: Dataitem) -> Dataitem:
     --------
     >>> obj = update_dataitem(obj)
     """
-    return processor.update_context_entity(
+    return context_processor.update_context_entity(
         project=entity.project,
         entity_type=entity.ENTITY_TYPE,
         entity_id=entity.id,
@@ -346,7 +358,7 @@ def delete_dataitem(
     >>>                       project="my-project",
     >>>                       delete_all_versions=True)
     """
-    return processor.delete_context_entity(
+    return context_processor.delete_context_entity(
         identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,

digitalhub/entities/dataitem/table/entity.py

@@ -7,7 +7,7 @@ from typing import Any
 
 from digitalhub.entities.dataitem._base.entity import Dataitem
 from digitalhub.stores.api import get_store
-from digitalhub.utils.uri_utils import has_local_scheme
+from digitalhub.utils.uri_utils import has_sql_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -36,19 +36,39 @@ class DataitemTable(Dataitem):
         self.spec: DataitemSpecTable
         self.status: DataitemStatusTable
 
+        self._query: str | None = None
+
+    def query(self, query: str) -> DataitemTable:
+        """
+        Set query to execute.
+
+        Parameters
+        ----------
+        query : str
+            Query to execute.
+
+        Returns
+        -------
+        DataitemTable
+            Self object.
+        """
+        # to remove in future
+        if not has_sql_scheme(self.spec.path):
+            raise ValueError(
+                f"Dataitem path is not a SQL scheme: {self.spec.path}",
+                " Query can be made only on a SQL scheme.",
+            )
+        self._query = query
+        return self
+
     def as_df(
         self,
         file_format: str | None = None,
-        engine: str | None = None,
-        clean_tmp_path: bool = True,
+        engine: str | None = "pandas",
         **kwargs,
     ) -> Any:
         """
         Read dataitem file (csv or parquet) as a DataFrame from spec.path.
-        If the dataitem is not local, it will be downloaded to a temporary
-        folder named tmp_dir in the project context folder.
-        If clean_tmp_path is True, the temporary folder will be deleted after the
-        method is executed.
         It's possible to pass additional arguments to the this function. These
         keyword arguments will be passed to the DataFrame reader function such as
         pandas's read_csv or read_parquet.
@@ -56,11 +76,10 @@ class DataitemTable(Dataitem):
         Parameters
         ----------
         file_format : str
-            Format of the file. (Supported csv and parquet).
+            Format of the file to read. By default, it will be inferred from
+            the extension of the file.
         engine : str
             Dataframe framework, by default pandas.
-        clean_tmp_path : bool
-            If True, the temporary folder will be deleted.
         **kwargs : dict
             Keyword arguments passed to the read_df function.
 
@@ -69,30 +88,20 @@ class DataitemTable(Dataitem):
         Any
             DataFrame.
         """
-        try:
-            if has_local_scheme(self.spec.path):
-                tmp_dir = None
-                data_path = self.spec.path
-            else:
-                tmp_dir = self._context().root / "tmp_data"
-                tmp_dir.mkdir(parents=True, exist_ok=True)
-                data_path = self.download(destination=str(tmp_dir), overwrite=True)
-
-            if Path(data_path).is_dir():
-                files = [str(i) for i in Path(data_path).rglob("*") if i.is_file()]
-                checker = files[0]
-            else:
-                checker = data_path
-
-            extension = self._get_extension(checker, file_format)
-            return get_store("").read_df(data_path, extension, engine, **kwargs)
-
-        except Exception as e:
-            raise e
-
-        finally:
-            # Delete tmp folder
-            self._clean_tmp_path(tmp_dir, clean_tmp_path)
+        if self._query is not None:
+            df = get_store(self.project, self.spec.path).query(
+                self._query,
+                self.spec.path,
+                engine,
+            )
+            self._query = None
+            return df
+        return get_store(self.project, self.spec.path).read_df(
+            self.spec.path,
+            file_format,
+            engine,
+            **kwargs,
+        )
 
     def write_df(
         self,
@@ -119,7 +128,12 @@ class DataitemTable(Dataitem):
         str
             Path to the written dataframe.
         """
-        return get_store(self.spec.path).write_df(df, self.spec.path, extension=extension, **kwargs)
+        return get_store(self.project, self.spec.path).write_df(
+            df,
+            self.spec.path,
+            extension=extension,
+            **kwargs,
+        )
 
     @staticmethod
     def _clean_tmp_path(pth: Path | None, clean: bool) -> None: