digitalhub 0.7.0b2__py3-none-any.whl → 0.8.0__py3-none-any.whl

digitalhub/entities/dataitem/crud.py

@@ -0,0 +1,372 @@
+from __future__ import annotations
+
+import typing
+from pathlib import Path
+from typing import Any
+from urllib.parse import urlparse
+
+from digitalhub.entities._base.crud import (
+    delete_entity,
+    get_material_entity,
+    get_material_entity_versions,
+    import_context_entity,
+    list_material_entities,
+    new_context_entity,
+)
+from digitalhub.entities._base.entity._constructors.uuid import build_uuid
+from digitalhub.entities.utils.entity_types import EntityTypes
+from digitalhub.entities.utils.utils import build_log_path_from_filename, build_log_path_from_source, eval_local_source
+from digitalhub.factory.api import build_entity_from_params
+from digitalhub.readers.api import get_reader_by_object
+from digitalhub.stores.api import get_store
+from digitalhub.utils.generic_utils import slugify_string
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities.dataitem._base.entity import Dataitem
+
+
+ENTITY_TYPE = EntityTypes.DATAITEM.value
+
+
+def new_dataitem(
+    project: str,
+    name: str,
+    kind: str,
+    uuid: str | None = None,
+    description: str | None = None,
+    labels: list[str] | None = None,
+    embedded: bool = False,
+    path: str | None = None,
+    **kwargs,
+) -> Dataitem:
+    """
+    Create a new object.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    name : str
+        Object name.
+    kind : str
+        Kind the object.
+    uuid : str
+        ID of the object.
+    description : str
+        Description of the object (human readable).
+    labels : list[str]
+        List of labels.
+    embedded : bool
+        Flag to determine if object spec must be embedded in project spec.
+    path : str
+        Object path on local file system or remote storage. It is also the destination path of upload() method.
+    **kwargs : dict
+        Spec keyword arguments.
+
+    Returns
+    -------
+    Dataitem
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = new_dataitem(project="my-project",
+    >>>                    name="my-dataitem",
+    >>>                    kind="dataitem",
+    >>>                    path="s3://my-bucket/my-key")
+    """
+    return new_context_entity(
+        project=project,
+        name=name,
+        kind=kind,
+        uuid=uuid,
+        description=description,
+        labels=labels,
+        embedded=embedded,
+        path=path,
+        **kwargs,
+    )
+
+
+def log_dataitem(
+    project: str,
+    name: str,
+    kind: str,
+    source: list[str] | str | None = None,
+    data: Any | None = None,
+    extension: str | None = None,
+    path: str | None = None,
+    **kwargs,
+) -> Dataitem:
+    """
+    Log a dataitem to the project.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    name : str
+        Object name.
+    kind : str
+        Kind the object.
+    source : str
+        Dataitem location on local path.
+    data : Any
+        Dataframe to log. Alternative to source.
+    extension : str
+        Extension of the output dataframe.
+    path : str
+        Destination path of the dataitem. If not provided, it's generated.
+    **kwargs : dict
+        New dataitem spec parameters.
+
+    Returns
+    -------
+    Dataitem
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = log_dataitem(project="my-project",
+    >>>                    name="my-dataitem",
+    >>>                    kind="table",
+    >>>                    data=df)
+    """
+    if (source is None) == (data is None):
+        raise ValueError("You must provide source or data.")
+
+    # Case where source is provided
+    if source is not None:
+        eval_local_source(source)
+
+        if path is None:
+            uuid = build_uuid()
+            kwargs["uuid"] = uuid
+            path = build_log_path_from_source(project, ENTITY_TYPE, name, uuid, source)
+
+        obj = new_dataitem(project=project, name=name, kind=kind, path=path, **kwargs)
+        obj.upload(source)
+
+    # Case where data is provided
+    else:
+        extension = extension if extension is not None else "parquet"
+        if path is None:
+            uuid = build_uuid()
+            kwargs["uuid"] = uuid
+            slug = slugify_string(name) + f".{extension}"
+            path = build_log_path_from_filename(project, ENTITY_TYPE, name, uuid, slug)
+
+        obj = build_entity_from_params(project=project, name=name, kind=kind, path=path, **kwargs)
+        if kind == "table":
+            dst = obj.write_df(df=data, extension=extension)
+            reader = get_reader_by_object(data)
+            obj.spec.schema = reader.get_schema(data)
+            obj.status.preview = reader.get_preview(data)
+            store = get_store(obj.spec.path)
+            src = Path(urlparse(obj.spec.path).path).name
+            paths = [(dst, src)]
+            infos = store.get_file_info(paths)
+            obj.status.add_files_info(infos)
+        obj.save()
+
+    return obj
+
+
+def get_dataitem(
+    identifier: str,
+    project: str | None = None,
+    entity_id: str | None = None,
+    **kwargs,
+) -> Dataitem:
+    """
+    Get object from backend.
+
+    Parameters
+    ----------
+    identifier : str
+        Entity key (store://...) or entity name.
+    project : str
+        Project name.
+    entity_id : str
+        Entity ID.
+    **kwargs : dict
+        Parameters to pass to the API call.
+
+    Returns
+    -------
+    Dataitem
+        Object instance.
+
+    Examples
+    --------
+    Using entity key:
+    >>> obj = get_dataitem("store://my-dataitem-key")
+
+    Using entity name:
+    >>> obj = get_dataitem("my-dataitem-name"
+    >>>                    project="my-project",
+    >>>                    entity_id="my-dataitem-id")
+    """
+    return get_material_entity(
+        identifier=identifier,
+        entity_type=ENTITY_TYPE,
+        project=project,
+        entity_id=entity_id,
+        **kwargs,
+    )
+
+
+def get_dataitem_versions(
+    identifier: str,
+    project: str | None = None,
+    **kwargs,
+) -> list[Dataitem]:
+    """
+    Get object versions from backend.
+
+    Parameters
+    ----------
+    identifier : str
+        Entity key (store://...) or entity name.
+    project : str
+        Project name.
+    **kwargs : dict
+        Parameters to pass to the API call.
+
+    Returns
+    -------
+    list[Dataitem]
+        List of object instances.
+
+    Examples
+    --------
+    Using entity key:
+    >>> objs = get_dataitem_versions("store://my-dataitem-key")
+
+    Using entity name:
+    >>> objs = get_dataitem_versions("my-dataitem-name",
+    >>>                              project="my-project")
+    """
+    return get_material_entity_versions(
+        identifier=identifier,
+        entity_type=ENTITY_TYPE,
+        project=project,
+        **kwargs,
+    )
+
+
+def list_dataitems(project: str, **kwargs) -> list[Dataitem]:
+    """
+    List all latest version objects from backend.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+    **kwargs : dict
+        Parameters to pass to the API call.
+
+    Returns
+    -------
+    list[Dataitem]
+        List of object instances.
+
+    Examples
+    --------
+    >>> objs = list_dataitems(project="my-project")
+    """
+    return list_material_entities(
+        project=project,
+        entity_type=ENTITY_TYPE,
+        **kwargs,
+    )
+
+
+def import_dataitem(file: str) -> Dataitem:
+    """
+    Import object from a YAML file.
+
+    Parameters
+    ----------
+    file : str
+        Path to YAML file.
+
+    Returns
+    -------
+    Dataitem
+        Object instance.
+
+    Examples
+    --------
+    >>> obj = import_dataitem("my-dataitem.yaml")
+    """
+    return import_context_entity(file)
+
+
+def update_dataitem(entity: Dataitem) -> Dataitem:
+    """
+    Update object. Note that object spec are immutable.
+
+    Parameters
+    ----------
+    entity : Dataitem
+        Object to update.
+
+    Returns
+    -------
+    Dataitem
+        Entity updated.
+
+    Examples
+    --------
+    >>> obj = update_dataitem(obj)
+    """
+    return entity.save(update=True)
+
+
+def delete_dataitem(
+    identifier: str,
+    project: str | None = None,
+    entity_id: str | None = None,
+    delete_all_versions: bool = False,
+    **kwargs,
+) -> dict:
+    """
+    Delete object from backend.
+
+    Parameters
+    ----------
+    identifier : str
+        Entity key (store://...) or entity name.
+    project : str
+        Project name.
+    entity_id : str
+        Entity ID.
+    delete_all_versions : bool
+        Delete all versions of the named entity. If True, use entity name instead of entity key as identifier.
+    **kwargs : dict
+        Parameters to pass to the API call.
+
+    Returns
+    -------
+    dict
+        Response from backend.
+
+    Examples
+    --------
+    If delete_all_versions is False:
+    >>> obj = delete_dataitem("store://my-dataitem-key")
+
+    Otherwise:
+    >>> obj = delete_dataitem("my-dataitem-name",
+    >>>                       project="my-project",
+    >>>                       delete_all_versions=True)
+    """
+    return delete_entity(
+        identifier=identifier,
+        entity_type=ENTITY_TYPE,
+        project=project,
+        entity_id=entity_id,
+        delete_all_versions=delete_all_versions,
+        **kwargs,
+    )

digitalhub/entities/dataitem/dataitem/builder.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.builder import DataitemBuilder
+from digitalhub.entities.dataitem.dataitem.entity import DataitemDataitem
+from digitalhub.entities.dataitem.dataitem.spec import DataitemSpecDataitem, DataitemValidatorDataitem
+from digitalhub.entities.dataitem.dataitem.status import DataitemStatusDataitem
+
+
+class DataitemDataitemBuilder(DataitemBuilder):
+    """
+    DataitemDataitem builder.
+    """
+
+    ENTITY_CLASS = DataitemDataitem
+    ENTITY_SPEC_CLASS = DataitemSpecDataitem
+    ENTITY_SPEC_VALIDATOR = DataitemValidatorDataitem
+    ENTITY_STATUS_CLASS = DataitemStatusDataitem
+    ENTITY_KIND = "dataitem"

digitalhub/entities/dataitem/dataitem/entity.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.entities.dataitem._base.entity import Dataitem
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities.dataitem.dataitem.spec import DataitemSpecDataitem
+    from digitalhub.entities.dataitem.dataitem.status import DataitemStatusDataitem
+
+
+class DataitemDataitem(Dataitem):
+    """
+    DataitemDataitem class.
+    """
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: DataitemSpecDataitem,
+        status: DataitemStatusDataitem,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+
+        self.spec: DataitemSpecDataitem
+        self.status: DataitemStatusDataitem

digitalhub/entities/dataitem/dataitem/spec.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.spec import DataitemSpec, DataitemValidator
+
+
+class DataitemSpecDataitem(DataitemSpec):
+    """
+    DataitemSpecDataitem specifications.
+    """
+
+
+class DataitemValidatorDataitem(DataitemValidator):
+    """
+    DataitemValidatorDataitem validator.
+    """

digitalhub/entities/dataitem/dataitem/status.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.status import DataitemStatus
+
+
+class DataitemStatusDataitem(DataitemStatus):
+    """
+    DataitemStatusDataitem status.
+    """

digitalhub/entities/dataitem/iceberg/builder.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.builder import DataitemBuilder
+from digitalhub.entities.dataitem.iceberg.entity import DataitemIceberg
+from digitalhub.entities.dataitem.iceberg.spec import DataitemSpecIceberg, DataitemValidatorIceberg
+from digitalhub.entities.dataitem.iceberg.status import DataitemStatusIceberg
+
+
+class DataitemIcebergBuilder(DataitemBuilder):
+    """
+    DataitemIceberg builder.
+    """
+
+    ENTITY_CLASS = DataitemIceberg
+    ENTITY_SPEC_CLASS = DataitemSpecIceberg
+    ENTITY_SPEC_VALIDATOR = DataitemValidatorIceberg
+    ENTITY_STATUS_CLASS = DataitemStatusIceberg
+    ENTITY_KIND = "iceberg"

digitalhub/entities/dataitem/iceberg/entity.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import typing
+
+from digitalhub.entities.dataitem._base.entity import Dataitem
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities.dataitem.iceberg.spec import DataitemSpecIceberg
+    from digitalhub.entities.dataitem.iceberg.status import DataitemStatusIceberg
+
+
+class DataitemIceberg(Dataitem):
+    """
+    DataitemIceberg class.
+    """
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: DataitemSpecIceberg,
+        status: DataitemStatusIceberg,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+
+        self.spec: DataitemSpecIceberg
+        self.status: DataitemStatusIceberg

digitalhub/entities/dataitem/iceberg/spec.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.spec import DataitemSpec, DataitemValidator
+
+
+class DataitemSpecIceberg(DataitemSpec):
+    """
+    DataitemSpecIceberg specifications.
+    """
+
+
+class DataitemValidatorIceberg(DataitemValidator):
+    """
+    DataitemValidatorIceberg validator.
+    """

digitalhub/entities/dataitem/iceberg/status.py

@@ -0,0 +1,9 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.status import DataitemStatus
+
+
+class DataitemStatusIceberg(DataitemStatus):
+    """
+    DataitemStatusIceberg status.
+    """

digitalhub/entities/dataitem/table/builder.py

@@ -0,0 +1,18 @@
+from __future__ import annotations
+
+from digitalhub.entities.dataitem._base.builder import DataitemBuilder
+from digitalhub.entities.dataitem.table.entity import DataitemTable
+from digitalhub.entities.dataitem.table.spec import DataitemSpecTable, DataitemValidatorTable
+from digitalhub.entities.dataitem.table.status import DataitemStatusTable
+
+
+class DataitemTableBuilder(DataitemBuilder):
+    """
+    DataitemTable builder.
+    """
+
+    ENTITY_CLASS = DataitemTable
+    ENTITY_SPEC_CLASS = DataitemSpecTable
+    ENTITY_SPEC_VALIDATOR = DataitemValidatorTable
+    ENTITY_STATUS_CLASS = DataitemStatusTable
+    ENTITY_KIND = "table"

digitalhub/entities/dataitem/table/entity.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import shutil
+import typing
+from pathlib import Path
+from typing import Any
+
+from digitalhub.datastores.api import get_datastore
+from digitalhub.entities.dataitem._base.entity import Dataitem
+from digitalhub.utils.uri_utils import check_local_path
+
+if typing.TYPE_CHECKING:
+    from digitalhub.entities._base.entity.metadata import Metadata
+    from digitalhub.entities.dataitem.table.spec import DataitemSpecTable
+    from digitalhub.entities.dataitem.table.status import DataitemStatusTable
+
+
+class DataitemTable(Dataitem):
+    """
+    DataitemTable class.
+    """
+
+    def __init__(
+        self,
+        project: str,
+        name: str,
+        uuid: str,
+        kind: str,
+        metadata: Metadata,
+        spec: DataitemSpecTable,
+        status: DataitemStatusTable,
+        user: str | None = None,
+    ) -> None:
+        super().__init__(project, name, uuid, kind, metadata, spec, status, user)
+
+        self.spec: DataitemSpecTable
+        self.status: DataitemStatusTable
+
+    def as_df(
+        self,
+        file_format: str | None = None,
+        engine: str | None = None,
+        clean_tmp_path: bool = True,
+        **kwargs,
+    ) -> Any:
+        """
+        Read dataitem file (csv or parquet) as a DataFrame from spec.path.
+        If the dataitem is not local, it will be downloaded to a temporary
+        folder named tmp_dir in the project context folder.
+        If clean_tmp_path is True, the temporary folder will be deleted after the
+        method is executed.
+        It's possible to pass additional arguments to the this function. These
+        keyword arguments will be passed to the DataFrame reader function such as
+        pandas's read_csv or read_parquet.
+
+        Parameters
+        ----------
+        file_format : str
+            Format of the file. (Supported csv and parquet).
+        engine : str
+            Dataframe framework, by default pandas.
+        clean_tmp_path : bool
+            If True, the temporary folder will be deleted.
+        **kwargs : dict
+            Keyword arguments passed to the read_df function.
+
+        Returns
+        -------
+        Any
+            DataFrame.
+        """
+        if engine is None:
+            engine = "pandas"
+        try:
+            if check_local_path(self.spec.path):
+                tmp_dir = None
+                data_path = self.spec.path
+            else:
+                tmp_dir = self._context().root / "tmp_data"
+                tmp_dir.mkdir(parents=True, exist_ok=True)
+                data_path = self.download(destination=str(tmp_dir), overwrite=True)
+
+            if Path(data_path).is_dir():
+                files = [str(i) for i in Path(data_path).rglob("*") if i.is_file()]
+                checker = files[0]
+            else:
+                checker = data_path
+
+            extension = self._get_extension(checker, file_format)
+            datastore = get_datastore("")
+
+            return datastore.read_df(data_path, extension, engine, **kwargs)
+
+        except Exception as e:
+            raise e
+
+        finally:
+            # Delete tmp folder
+            self._clean_tmp_path(tmp_dir, clean_tmp_path)
+
+    def write_df(
+        self,
+        df: Any,
+        extension: str | None = None,
+        **kwargs,
+    ) -> str:
+        """
+        Write DataFrame as parquet/csv/table into dataitem spec.path.
+        keyword arguments will be passed to the DataFrame reader function such as
+        pandas's to_csv or to_parquet.
+
+        Parameters
+        ----------
+        df : Any
+            DataFrame to write.
+        extension : str
+            Extension of the file.
+        **kwargs : dict
+            Keyword arguments passed to the write_df function.
+
+        Returns
+        -------
+        str
+            Path to the written dataframe.
+        """
+        datastore = get_datastore(self.spec.path)
+        return datastore.write_df(df, self.spec.path, extension=extension, **kwargs)
+
+    @staticmethod
+    def _clean_tmp_path(pth: Path | None, clean: bool) -> None:
+        """
+        Clean temporary path.
+
+        Parameters
+        ----------
+        pth : Path | None
+            Path to clean.
+        clean : bool
+            If True, the path will be cleaned.
+
+        Returns
+        -------
+        None
+        """
+        if pth is not None and clean:
+            shutil.rmtree(pth)