digitalhub 0.14.0b1__tar.gz → 0.14.0b2__tar.gz

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.14.0b1
+Version: 0.14.0b2
 Summary: Python SDK for Digitalhub
 Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <digitalhub@fbk.eu>, Matteo Martini <mmartini@fbk.eu>

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/_base/executable/entity.py

@@ -13,7 +13,7 @@ from digitalhub.entities._processors.context import context_processor
 from digitalhub.entities.run.crud import list_runs
 from digitalhub.entities.task.crud import delete_task, list_tasks
 from digitalhub.entities.trigger.crud import list_triggers
-from digitalhub.factory.factory import factory
+from digitalhub.factory.entity import entity_factory
 from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError
 
 if typing.TYPE_CHECKING:
@@ -109,7 +109,7 @@ class ExecutableEntity(VersionedEntity):
             # Create a new object from dictionary.
             # the form in which tasks are stored in function
             # status
-            task_obj: Task = factory.build_entity_from_dict(task)
+            task_obj: Task = entity_factory.build_entity_from_dict(task)
 
             # Try to save it in backend to been able to use
             # it for launching runs. In fact, tasks must be
@@ -149,7 +149,7 @@ class ExecutableEntity(VersionedEntity):
         kwargs["kind"] = kind
 
         # Create object instance
-        task: Task = factory.build_entity_from_params(**kwargs)
+        task: Task = entity_factory.build_entity_from_params(**kwargs)
         task.save()
 
         self._tasks[kind] = task
@@ -225,7 +225,7 @@ class ExecutableEntity(VersionedEntity):
         kwargs["uuid"] = self._tasks[kind].id
 
         # Update task
-        task: Task = factory.build_entity_from_params(**kwargs)
+        task: Task = entity_factory.build_entity_from_params(**kwargs)
         task.save(update=True)
         self._tasks[kind] = task
         return task
@@ -361,10 +361,14 @@ class ExecutableEntity(VersionedEntity):
         Examples
         --------
         Using entity key:
-        >>> obj = executable.get_run("store://my-secret-key")
+        >>> obj = executable.get_run(
+        ...     "store://my-secret-key"
+        ... )
 
         Using entity ID:
-        >>> obj = executable.get_run("123")
+        >>> obj = executable.get_run(
+        ...     "123"
+        ... )
         """
         entities = self.list_runs(**kwargs)
         for entity in entities:
@@ -425,13 +429,13 @@ class ExecutableEntity(VersionedEntity):
             Run instance.
         """
         # Get task
-        task_kind = factory.get_task_kind_from_action(self.kind, action)
+        task_kind = entity_factory.get_task_kind_from_action(self.kind, action)
         task = self._get_or_create_task(task_kind)
         task_string = task._get_task_string()
 
         # Get run validator for building trigger template
-        run_kind = factory.get_run_kind_from_action(self.kind, action)
-        run_validator: SpecValidator = factory.get_spec_validator(run_kind)
+        run_kind = entity_factory.get_run_kind_from_action(self.kind, action)
+        run_validator: SpecValidator = entity_factory.get_spec_validator(run_kind)
         # Override kwargs
         kwargs["project"] = self.project
         kwargs["kind"] = trigger_kind
@@ -441,7 +445,7 @@ class ExecutableEntity(VersionedEntity):
         kwargs["template"] = run_validator(**kwargs).to_dict()
 
         # Create object instance
-        trigger: Trigger = factory.build_entity_from_params(**kwargs)
+        trigger: Trigger = entity_factory.build_entity_from_params(**kwargs)
         trigger.save()
         return trigger
 
@@ -468,10 +472,14 @@ class ExecutableEntity(VersionedEntity):
         Examples
         --------
         Using entity key:
-        >>> obj = executable.get_trigger("store://my-trigger-key")
+        >>> obj = executable.get_trigger(
+        ...     "store://my-trigger-key"
+        ... )
 
         Using entity ID:
-        >>> obj = executable.get_trigger("123")
+        >>> obj = executable.get_trigger(
+        ...     "123"
+        ... )
         """
         entities = self.list_triggers(**kwargs)
         for entity in entities:

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/_base/material/entity.py

@@ -150,12 +150,16 @@ class MaterialEntity(VersionedEntity):
         --------
         Upload a single file:
 
-        >>> entity.spec.path = 's3://bucket/data.csv'
-        >>> entity.upload('./data.csv')
+        >>> entity.spec.path = "s3://bucket/data.csv"
+        >>> entity.upload(
+        ...     "./data.csv"
+        ... )
 
         Upload a folder:
-        >>> entity.spec.path = 's3://bucket/data/'
-        >>> entity.upload('./data')
+        >>> entity.spec.path = "s3://bucket/data/"
+        >>> entity.upload(
+        ...     "./data"
+        ... )
         """
         # Get store and upload object
         store = get_store(self.spec.path)

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/_processors/base.py

@@ -9,7 +9,7 @@ from warnings import warn
 
 from digitalhub.context.api import delete_context
 from digitalhub.entities._commons.enums import ApiCategories, BackendOperations
-from digitalhub.factory.factory import factory
+from digitalhub.factory.entity import entity_factory
 from digitalhub.stores.client.api import get_client
 from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError, EntityNotExistsError
 from digitalhub.utils.io_utils import read_yaml
@@ -100,10 +100,10 @@ class BaseEntityOperationsProcessor:
             obj = _entity
         else:
             client = get_client(kwargs.get("local"))
-            obj = factory.build_entity_from_params(**kwargs)
+            obj = entity_factory.build_entity_from_params(**kwargs)
         ent = self._create_base_entity(client, obj.ENTITY_TYPE, obj.to_dict())
         ent["local"] = client.is_local()
-        return factory.build_entity_from_dict(ent)
+        return entity_factory.build_entity_from_dict(ent)
 
     def _read_base_entity(
         self,
@@ -172,7 +172,7 @@ class BaseEntityOperationsProcessor:
         client = get_client(kwargs.pop("local", False))
         obj = self._read_base_entity(client, entity_type, entity_name, **kwargs)
         obj["local"] = client.is_local()
-        return factory.build_entity_from_dict(obj)
+        return entity_factory.build_entity_from_dict(obj)
 
     def import_project_entity(
         self,
@@ -207,7 +207,7 @@ class BaseEntityOperationsProcessor:
         obj: dict = read_yaml(file)
         obj["status"] = {}
         obj["local"] = client.is_local()
-        ent: Project = factory.build_entity_from_dict(obj)
+        ent: Project = entity_factory.build_entity_from_dict(obj)
         reset_id = kwargs.pop("reset_id", False)
 
         try:
@@ -253,7 +253,7 @@ class BaseEntityOperationsProcessor:
         client = get_client(kwargs.pop("local", False))
         obj: dict = read_yaml(file)
         obj["local"] = client.is_local()
-        ent: Project = factory.build_entity_from_dict(obj)
+        ent: Project = entity_factory.build_entity_from_dict(obj)
 
         try:
             self._update_base_entity(ent._client, ent.ENTITY_TYPE, ent.name, ent.to_dict())
@@ -328,7 +328,7 @@ class BaseEntityOperationsProcessor:
         entities = []
         for obj in objs:
             obj["local"] = client.is_local()
-            ent = factory.build_entity_from_dict(obj)
+            ent = entity_factory.build_entity_from_dict(obj)
             entities.append(ent)
         return entities
 
@@ -405,7 +405,7 @@ class BaseEntityOperationsProcessor:
         client = get_client(kwargs.pop("local", False))
         obj = self._update_base_entity(client, entity_type, entity_name, entity_dict, **kwargs)
         obj["local"] = client.is_local()
-        return factory.build_entity_from_dict(obj)
+        return entity_factory.build_entity_from_dict(obj)
 
     def _delete_base_entity(
         self,

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/_processors/context.py

@@ -15,7 +15,7 @@ from digitalhub.entities._processors.utils import (
     get_context_from_project,
     parse_identifier,
 )
-from digitalhub.factory.factory import factory
+from digitalhub.factory.entity import entity_factory
 from digitalhub.utils.exceptions import EntityAlreadyExistsError, EntityError, EntityNotExistsError
 from digitalhub.utils.io_utils import read_yaml
 from digitalhub.utils.types import SourcesOrListOfSources
@@ -108,9 +108,9 @@ class ContextEntityOperationsProcessor:
             obj = _entity
         else:
             context = get_context_from_project(kwargs["project"])
-            obj: ContextEntity = factory.build_entity_from_params(**kwargs)
+            obj: ContextEntity = entity_factory.build_entity_from_params(**kwargs)
         new_obj = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
-        return factory.build_entity_from_dict(new_obj)
+        return entity_factory.build_entity_from_dict(new_obj)
 
     def log_material_entity(
         self,
@@ -143,12 +143,12 @@ class ContextEntityOperationsProcessor:
         """
         source: SourcesOrListOfSources = kwargs.pop("source")
         context = get_context_from_project(kwargs["project"])
-        obj = factory.build_entity_from_params(**kwargs)
+        obj = entity_factory.build_entity_from_params(**kwargs)
         if context.is_running:
             obj.add_relationship(Relationship.PRODUCEDBY.value, context.get_run_ctx())
 
         new_obj: MaterialEntity = self._create_context_entity(context, obj.ENTITY_TYPE, obj.to_dict())
-        new_obj = factory.build_entity_from_dict(new_obj)
+        new_obj = entity_factory.build_entity_from_dict(new_obj)
 
         new_obj.status.state = State.UPLOADING.value
         new_obj = self._update_material_entity(new_obj)
@@ -285,7 +285,7 @@ class ContextEntityOperationsProcessor:
             entity_id=entity_id,
             **kwargs,
         )
-        entity = factory.build_entity_from_dict(obj)
+        entity = entity_factory.build_entity_from_dict(obj)
         return self._post_process_get(entity)
 
     def read_unversioned_entity(
@@ -382,14 +382,14 @@ class ContextEntityOperationsProcessor:
             context = dict_obj["project"]
 
         ctx = get_context_from_project(context)
-        obj = factory.build_entity_from_dict(dict_obj)
+        obj = entity_factory.build_entity_from_dict(dict_obj)
         if reset_id:
             new_id = build_uuid()
             obj.id = new_id
             obj.metadata.version = new_id
         try:
             bck_obj = self._create_context_entity(ctx, obj.ENTITY_TYPE, obj.to_dict())
-            new_obj: ContextEntity = factory.build_entity_from_dict(bck_obj)
+            new_obj: ContextEntity = entity_factory.build_entity_from_dict(bck_obj)
         except EntityAlreadyExistsError:
             raise EntityError(f"Entity {obj.name} already exists. If you want to update it, use load instead.")
         return new_obj
@@ -451,7 +451,7 @@ class ContextEntityOperationsProcessor:
             context = exec_dict["project"]
 
         ctx = get_context_from_project(context)
-        obj: ExecutableEntity = factory.build_entity_from_dict(exec_dict)
+        obj: ExecutableEntity = entity_factory.build_entity_from_dict(exec_dict)
 
         if reset_id:
             new_id = build_uuid()
@@ -460,7 +460,7 @@ class ContextEntityOperationsProcessor:
 
         try:
             bck_obj = self._create_context_entity(ctx, obj.ENTITY_TYPE, obj.to_dict())
-            new_obj: ExecutableEntity = factory.build_entity_from_dict(bck_obj)
+            new_obj: ExecutableEntity = entity_factory.build_entity_from_dict(bck_obj)
         except EntityAlreadyExistsError:
             raise EntityError(f"Entity {obj.name} already exists. If you want to update it, use load instead.")
 
@@ -491,7 +491,7 @@ class ContextEntityOperationsProcessor:
         """
         dict_obj: dict = read_yaml(file)
         context = get_context_from_project(dict_obj["project"])
-        obj: ContextEntity = factory.build_entity_from_dict(dict_obj)
+        obj: ContextEntity = entity_factory.build_entity_from_dict(dict_obj)
         try:
             self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
         except EntityNotExistsError:
@@ -529,7 +529,7 @@ class ContextEntityOperationsProcessor:
             tsk_dicts = []
 
         context = get_context_from_project(exec_dict["project"])
-        obj: ExecutableEntity = factory.build_entity_from_dict(exec_dict)
+        obj: ExecutableEntity = entity_factory.build_entity_from_dict(exec_dict)
 
         try:
             self._update_context_entity(context, obj.ENTITY_TYPE, obj.id, obj.to_dict())
@@ -631,7 +631,7 @@ class ContextEntityOperationsProcessor:
         )
         objects = []
         for o in objs:
-            entity: ContextEntity = factory.build_entity_from_dict(o)
+            entity: ContextEntity = entity_factory.build_entity_from_dict(o)
             entity = self._post_process_get(entity)
             objects.append(entity)
         return objects
@@ -704,7 +704,7 @@ class ContextEntityOperationsProcessor:
         objs = self._list_context_entities(context, entity_type, **kwargs)
         objects = []
         for o in objs:
-            entity: ContextEntity = factory.build_entity_from_dict(o)
+            entity: ContextEntity = entity_factory.build_entity_from_dict(o)
             entity = self._post_process_get(entity)
             objects.append(entity)
         return objects
@@ -819,7 +819,7 @@ class ContextEntityOperationsProcessor:
             entity_dict,
             **kwargs,
         )
-        return factory.build_entity_from_dict(obj)
+        return entity_factory.build_entity_from_dict(obj)
 
     def _delete_context_entity(
         self,

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/_processors/utils.py

@@ -9,7 +9,7 @@ import typing
 from digitalhub.context.api import get_context
 from digitalhub.entities._commons.enums import ApiCategories, BackendOperations, EntityTypes
 from digitalhub.entities._commons.utils import get_project_from_key, is_valid_key, parse_entity_key
-from digitalhub.factory.factory import factory
+from digitalhub.factory.entity import entity_factory
 from digitalhub.stores.client.api import get_client
 from digitalhub.utils.exceptions import ContextError, EntityError, EntityNotExistsError
 
@@ -161,7 +161,7 @@ def get_context_from_remote(
     try:
         client = get_client()
         obj = _read_base_entity(client, EntityTypes.PROJECT.value, project)
-        factory.build_entity_from_dict(obj)
+        entity_factory.build_entity_from_dict(obj)
         return get_context(project)
     except EntityNotExistsError:
         raise ContextError(f"Project '{project}' not found.")

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/artifact/crud.py

@@ -156,7 +156,9 @@ def get_artifact(
     Examples
     --------
     Using entity key:
-    >>> obj = get_artifact("store://my-artifact-key")
+    >>> obj = get_artifact(
+    ...     "store://my-artifact-key"
+    ... )
 
     Using entity name:
     >>> obj = get_artifact("my-artifact-name"
@@ -197,7 +199,9 @@ def get_artifact_versions(
     Examples
     --------
     Using entity key:
-    >>> obj = get_artifact_versions("store://my-artifact-key")
+    >>> obj = get_artifact_versions(
+    ...     "store://my-artifact-key"
+    ... )
 
     Using entity name:
     >>> obj = get_artifact_versions("my-artifact-name"
@@ -229,7 +233,9 @@ def list_artifacts(project: str, **kwargs) -> list[Artifact]:
 
     Examples
     --------
-    >>> objs = list_artifacts(project="my-project")
+    >>> objs = list_artifacts(
+    ...     project="my-project"
+    ... )
     """
     return context_processor.list_context_entities(
         project=project,
@@ -265,7 +271,9 @@ def import_artifact(
 
     Examples
     --------
-    >>> obj = import_artifact("my-artifact.yaml")
+    >>> obj = import_artifact(
+    ...     "my-artifact.yaml"
+    ... )
     """
     return context_processor.import_context_entity(
         file,
@@ -291,7 +299,9 @@ def load_artifact(file: str) -> Artifact:
 
     Examples
     --------
-    >>> obj = load_artifact("my-artifact.yaml")
+    >>> obj = load_artifact(
+    ...     "my-artifact.yaml"
+    ... )
     """
     return context_processor.load_context_entity(file)
 
@@ -312,7 +322,11 @@ def update_artifact(entity: Artifact) -> Artifact:
 
     Examples
     --------
-    >>> obj = update_artifact(obj)
+    >>> obj = (
+    ...     update_artifact(
+    ...         obj
+    ...     )
+    ... )
     """
     return context_processor.update_context_entity(
         project=entity.project,
@@ -356,7 +370,9 @@ def delete_artifact(
     Examples
     --------
     If delete_all_versions is False:
-    >>> delete_artifact("store://my-artifact-key")
+    >>> delete_artifact(
+    ...     "store://my-artifact-key"
+    ... )
 
     Otherwise:
     >>> delete_artifact("my-artifact-name",

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/dataitem/crud.py

@@ -87,6 +87,7 @@ def log_dataitem(
     data: Any | None = None,
     path: str | None = None,
     file_format: str | None = None,
+    read_df_params: dict | None = None,
     engine: str | None = "pandas",
     **kwargs,
 ) -> Dataitem:
@@ -108,7 +109,9 @@ def log_dataitem(
     path : str
         Destination path of the dataitem. If not provided, it's generated.
     file_format : str
-        Extension of the file.
+        Extension of the file source (parquet, csv, json, xlsx, txt).
+    read_df_params : dict
+        Parameters to pass to the dataframe reader.
     engine : str
         Dataframe engine (pandas, polars, etc.).
     **kwargs : dict
@@ -131,7 +134,7 @@ def log_dataitem(
         cleanup = True
 
     source = eval_source(source, data, kind, name, project)
-    data = eval_data(kind, source, data, file_format, engine)
+    data = eval_data(kind, source, data, file_format, read_df_params, engine)
     kwargs = process_kwargs(
         project,
         name,
@@ -182,7 +185,9 @@ def get_dataitem(
     Examples
     --------
     Using entity key:
-    >>> obj = get_dataitem("store://my-dataitem-key")
+    >>> obj = get_dataitem(
+    ...     "store://my-dataitem-key"
+    ... )
 
     Using entity name:
     >>> obj = get_dataitem("my-dataitem-name"
@@ -223,7 +228,9 @@ def get_dataitem_versions(
     Examples
     --------
     Using entity key:
-    >>> objs = get_dataitem_versions("store://my-dataitem-key")
+    >>> objs = get_dataitem_versions(
+    ...     "store://my-dataitem-key"
+    ... )
 
     Using entity name:
     >>> objs = get_dataitem_versions("my-dataitem-name",
@@ -255,7 +262,9 @@ def list_dataitems(project: str, **kwargs) -> list[Dataitem]:
 
     Examples
     --------
-    >>> objs = list_dataitems(project="my-project")
+    >>> objs = list_dataitems(
+    ...     project="my-project"
+    ... )
     """
     return context_processor.list_context_entities(
         project=project,
@@ -291,7 +300,9 @@ def import_dataitem(
 
     Examples
     --------
-    >>> obj = import_dataitem("my-dataitem.yaml")
+    >>> obj = import_dataitem(
+    ...     "my-dataitem.yaml"
+    ... )
     """
     return context_processor.import_context_entity(
         file,
@@ -317,7 +328,9 @@ def load_dataitem(file: str) -> Dataitem:
 
     Examples
     --------
-    >>> obj = load_dataitem("my-dataitem.yaml")
+    >>> obj = load_dataitem(
+    ...     "my-dataitem.yaml"
+    ... )
     """
     return context_processor.load_context_entity(file)
 
@@ -338,7 +351,11 @@ def update_dataitem(entity: Dataitem) -> Dataitem:
 
     Examples
     --------
-    >>> obj = update_dataitem(obj)
+    >>> obj = (
+    ...     update_dataitem(
+    ...         obj
+    ...     )
+    ... )
     """
     return context_processor.update_context_entity(
         project=entity.project,
@@ -382,7 +399,9 @@ def delete_dataitem(
     Examples
     --------
     If delete_all_versions is False:
-    >>> obj = delete_dataitem("store://my-dataitem-key")
+    >>> obj = delete_dataitem(
+    ...     "store://my-dataitem-key"
+    ... )
 
     Otherwise:
     >>> obj = delete_dataitem("my-dataitem-name",

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/dataitem/table/entity.py

@@ -115,13 +115,17 @@ class DataitemTable(Dataitem):
         Write DataFrame as parquet/csv/table into dataitem spec.path.
         keyword arguments will be passed to the DataFrame reader function such as
         pandas's to_csv or to_parquet.
+        Note that by default the index is dropped when writing the dataframe. To
+        keep the index, you can pass index=True as a keyword argument.
+        If the dataitem path is a SQL scheme, the dataframe will be written to the
+        table specified in the path (sql://<database_name>(/<schema_name>)/<table_name>).
 
         Parameters
         ----------
         df : Any
             DataFrame to write.
         extension : str
-            Extension of the file.
+            Extension of the file (supported parquet and csv).
         **kwargs : dict
             Keyword arguments passed to the write_df function.
 
@@ -129,6 +133,29 @@ class DataitemTable(Dataitem):
         -------
         str
             Path to the written dataframe.
+
+        Examples
+        --------
+        >>> import digitalhub as dh
+        >>> import pandas as pd
+        >>>
+        >>> p = dh.get_project(
+        ...     "my_project"
+        ... )
+        >>> df = pd.read_df(
+        ...     "data/my_data.csv"
+        ... )
+        >>> di = p.new_dataitem(
+        ...     name="my_dataitem",
+        ...     kind="table",
+        ...     path="s3://my-bucket/my-data.parquet",
+        ... )
+        >>> di.write_df(
+        ...     df,
+        ...     extension="parquet",
+        ...     index=True,
+        ... )
+        's3://my-bucket/my-data.parquet'
         """
         return get_store(self.spec.path).write_df(
             df,

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/dataitem/utils.py

@@ -89,6 +89,7 @@ def eval_data(
     source: SourcesOrListOfSources,
     data: Any | None = None,
     file_format: str | None = None,
+    read_df_params: dict | None = None,
     engine: str | None = None,
 ) -> Any:
     """
@@ -121,10 +122,13 @@ def eval_data(
     """
     if kind == EntityKinds.DATAITEM_TABLE.value:
         if data is None:
+            if read_df_params is None:
+                read_df_params = {}
             return get_store(source).read_df(
                 source,
                 file_format=file_format,
                 engine=engine,
+                **read_df_params,
             )
     return data
 

{digitalhub-0.14.0b1 → digitalhub-0.14.0b2}/digitalhub/entities/function/_base/entity.py

@@ -9,7 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
 from digitalhub.entities._commons.enums import EntityTypes, Relationship
-from digitalhub.factory.factory import factory
+from digitalhub.factory.entity import entity_factory
 from digitalhub.utils.exceptions import BackendError
 
 if typing.TYPE_CHECKING:
@@ -76,8 +76,8 @@ class Function(ExecutableEntity):
             Run instance.
         """
         # Get task and run kind
-        task_kind = factory.get_task_kind_from_action(self.kind, action)
-        run_kind = factory.get_run_kind_from_action(self.kind, action)
+        task_kind = entity_factory.get_task_kind_from_action(self.kind, action)
+        run_kind = entity_factory.get_run_kind_from_action(self.kind, action)
 
         # Create or update new task
         task = self._get_or_create_task(task_kind)