digitalhub 0.13.0b3__py3-none-any.whl → 0.14.9__py3-none-any.whl

digitalhub/entities/dataitem/utils.py

@@ -9,9 +9,9 @@ import typing
 from typing import Any
 
 from digitalhub.context.api import get_context
-from digitalhub.entities._base.entity._constructors.uuid import build_uuid
 from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
 from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
+from digitalhub.entities._constructors.uuid import build_uuid
 from digitalhub.stores.data.api import get_store
 from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.enums import FileExtensions
@@ -33,16 +33,39 @@ def eval_source(
     project: str | None = None,
 ) -> Any:
     """
-    Evaluate if source is local.
+    Evaluate and process data source for dataitem creation.
+
+    Determines the appropriate source handling based on whether a source
+    path or data object is provided. For table dataitems with data objects,
+    writes the data to a Parquet file and returns the file path.
 
     Parameters
     ----------
     source : SourcesOrListOfSources
-        Source(s).
+        The source specification(s) for the dataitem. Can be file paths,
+        URLs, or other source identifiers.
+    data : Any
+        The data object to process (e.g., DataFrame). Alternative to source.
+        Exactly one of source or data must be provided.
+    kind : str
+        The kind of dataitem being created (e.g., 'table').
+    name : str
+        The name of the dataitem, used for generating file paths.
+    project : str
+        The project name, used for context and path generation.
 
     Returns
     -------
-    None
+    Any
+        The processed source. Returns the original source if provided,
+        or the path to a generated file if data was processed.
+
+    Raises
+    ------
+    ValueError
+        If both source and data are provided or both are None.
+    NotImplementedError
+        If the specified kind is not supported for data processing.
     """
     if (source is None) == (data is None):
         raise ValueError("You must provide source or data.")
@@ -66,34 +89,46 @@ def eval_data(
     source: SourcesOrListOfSources,
     data: Any | None = None,
     file_format: str | None = None,
+    read_df_params: dict | None = None,
     engine: str | None = None,
 ) -> Any:
     """
-    Evaluate data is loaded.
+    Evaluate and load data from source or return provided data.
+
+    For table dataitems, loads data from the source using the appropriate
+    store and reader. For other kinds, returns the data as-is.
 
     Parameters
     ----------
-    project : str
-        Project name.
-    source : str
-        Source(s).
+    kind : str
+        The kind of dataitem (e.g., 'table') that determines
+        how data should be processed.
+    source : SourcesOrListOfSources
+        The source specification(s) to load data from.
     data : Any
-        Dataframe to log. Alternative to source.
+        Pre-loaded data object. If provided, may be returned directly
+        depending on the dataitem kind.
     file_format : str
-        Extension of the file.
+        The file format specification for reading the source
+        (e.g., 'parquet', 'csv').
     engine : str
-        Engine to use.
+        The engine to use for reading the data (e.g., 'pandas', 'polars').
 
     Returns
     -------
-    None
+    Any
+        The loaded data object for table dataitems, or the original
+        data parameter for other kinds.
     """
     if kind == EntityKinds.DATAITEM_TABLE.value:
         if data is None:
+            if read_df_params is None:
+                read_df_params = {}
             return get_store(source).read_df(
                 source,
                 file_format=file_format,
                 engine=engine,
+                **read_df_params,
             )
     return data
 
@@ -108,29 +143,37 @@ def process_kwargs(
     **kwargs,
 ) -> dict:
     """
-    Process spec kwargs.
+    Process and enhance specification parameters for dataitem creation.
+
+    Processes the keyword arguments for dataitem specification, handling
+    schema extraction for table dataitems and path generation. Extracts
+    schema information from data objects when available.
 
     Parameters
     ----------
     project : str
-        Project name.
+        The name of the project.
     name : str
-        Object name.
+        The name of the dataitem entity.
     kind : str
-        Kind the object.
+        The kind of dataitem being created (e.g., 'table').
     source : SourcesOrListOfSources
-        Source(s).
+        The source specification(s) for the dataitem content.
     data : Any
-        Dataframe to log. Alternative to source.
+        The data object for schema extraction and processing.
+        Used as an alternative to source for table dataitems.
     path : str
-        Destination path of the entity. If not provided, it's generated.
+        The destination path for the dataitem entity.
+        If None, a path will be automatically generated.
     **kwargs : dict
-        Spec parameters.
+        Additional specification parameters to be processed
+        and passed to the dataitem creation.
 
     Returns
     -------
     dict
-        Kwargs updated.
+        The updated kwargs dictionary with processed path,
+        UUID, and schema information included.
     """
     if data is not None:
         if kind == EntityKinds.DATAITEM_TABLE.value:
@@ -147,16 +190,17 @@ def process_kwargs(
 
 def clean_tmp_path(pth: SourcesOrListOfSources) -> None:
     """
-    Clean temporary path.
+    Clean up temporary files and directories.
+
+    Removes temporary files or directories created during dataitem
+    processing. Handles both single paths and lists of paths,
+    ignoring any errors that occur during cleanup.
 
     Parameters
     ----------
     pth : SourcesOrListOfSources
-        Path to clean.
-
-    Returns
-    -------
-    None
+        The path or list of paths to clean up. Can be file paths
+        or directory paths that need to be removed.
     """
     if isinstance(pth, list):
         for p in pth:
@@ -167,19 +211,25 @@ def clean_tmp_path(pth: SourcesOrListOfSources) -> None:
 
 def post_process(obj: Dataitem, data: Any) -> Dataitem:
     """
-    Post process object.
+    Post-process dataitem object with additional metadata and previews.
+
+    Enhances the dataitem object with additional information extracted
+    from the data. For table dataitems, generates and stores a data
+    preview in the object's status.
 
     Parameters
     ----------
     obj : Dataitem
-        The object.
+        The dataitem object to post-process and enhance.
     data : Any
-        The data.
+        The data object used to generate previews and extract
+        additional metadata information.
 
     Returns
     -------
     Dataitem
-        The object.
+        The enhanced dataitem object with updated status information
+        and saved changes.
     """
     if obj.kind == EntityKinds.DATAITEM_TABLE.value:
         reader = get_reader_by_object(data)

digitalhub/entities/function/_base/entity.py

@@ -9,8 +9,7 @@ from concurrent.futures import ThreadPoolExecutor
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
 from digitalhub.entities._commons.enums import EntityTypes, Relationship
-from digitalhub.factory.factory import factory
-from digitalhub.utils.exceptions import BackendError
+from digitalhub.factory.entity import entity_factory
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -76,8 +75,8 @@ class Function(ExecutableEntity):
             Run instance.
         """
         # Get task and run kind
-        task_kind = factory.get_task_kind_from_action(self.kind, action)
-        run_kind = factory.get_run_kind(self.kind)
+        task_kind = entity_factory.get_task_kind_from_action(self.kind, action)
+        run_kind = entity_factory.get_run_kind_from_action(self.kind, action)
 
         # Create or update new task
         task = self._get_or_create_task(task_kind)
@@ -91,8 +90,6 @@ class Function(ExecutableEntity):
 
         # If execution is done by DHCore backend, return the object
         if not local_execution:
-            if self._context().local:
-                raise BackendError("Cannot run remote function with local backend.")
             if wait:
                 return run.wait(log_info=log_info)
             return run

digitalhub/entities/function/crud.py

@@ -7,7 +7,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities._processors.context import context_processor
+from digitalhub.entities._processors.processors import context_processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities.function._base.entity import Function
@@ -76,7 +76,6 @@ def get_function(
     identifier: str,
     project: str | None = None,
     entity_id: str | None = None,
-    **kwargs,
 ) -> Function:
     """
     Get object from backend.
@@ -89,8 +88,6 @@ def get_function(
         Project name.
     entity_id : str
         Entity ID.
-    **kwargs : dict
-        Parameters to pass to the API call.
 
     Returns
     -------
@@ -108,18 +105,16 @@ def get_function(
     >>>                    entity_id="my-function-id")
     """
     return context_processor.read_context_entity(
-        identifier,
+        identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
         entity_id=entity_id,
-        **kwargs,
     )
 
 
 def get_function_versions(
     identifier: str,
     project: str | None = None,
-    **kwargs,
 ) -> list[Function]:
     """
     Get object versions from backend.
@@ -130,8 +125,6 @@ def get_function_versions(
         Entity key (store://...) or entity name.
     project : str
         Project name.
-    **kwargs : dict
-        Parameters to pass to the API call.
 
     Returns
     -------
@@ -148,14 +141,23 @@ def get_function_versions(
     >>>                             project="my-project")
     """
     return context_processor.read_context_entity_versions(
-        identifier,
+        identifier=identifier,
         entity_type=ENTITY_TYPE,
         project=project,
-        **kwargs,
     )
 
 
-def list_functions(project: str, **kwargs) -> list[Function]:
+def list_functions(
+    project: str,
+    q: str | None = None,
+    name: str | None = None,
+    kind: str | None = None,
+    user: str | None = None,
+    state: str | None = None,
+    created: str | None = None,
+    updated: str | None = None,
+    versions: str | None = None,
+) -> list[Function]:
     """
     List all latest version objects from backend.
 
@@ -163,8 +165,22 @@ def list_functions(project: str, **kwargs) -> list[Function]:
     ----------
     project : str
         Project name.
-    **kwargs : dict
-        Parameters to pass to the API call.
+    q : str
+        Query string to filter objects.
+    name : str
+        Object name.
+    kind : str
+        Kind of the object.
+    user : str
+        User that created the object.
+    state : str
+        Object state.
+    created : str
+        Creation date filter.
+    updated : str
+        Update date filter.
+    versions : str
+        Object version, default is latest.
 
     Returns
     -------
@@ -178,18 +194,36 @@ def list_functions(project: str, **kwargs) -> list[Function]:
     return context_processor.list_context_entities(
         project=project,
         entity_type=ENTITY_TYPE,
-        **kwargs,
+        q=q,
+        name=name,
+        kind=kind,
+        user=user,
+        state=state,
+        created=created,
+        updated=updated,
+        versions=versions,
     )
 
 
-def import_function(file: str) -> Function:
+def import_function(
+    file: str | None = None,
+    key: str | None = None,
+    reset_id: bool = False,
+    context: str | None = None,
+) -> Function:
     """
-    Import object from a YAML file and create a new object into the backend.
+    Import an object from a YAML file or from a storage key.
 
     Parameters
     ----------
     file : str
-        Path to YAML file.
+        Path to the YAML file.
+    key : str
+        Entity key (store://...).
+    reset_id : bool
+        Flag to determine if the ID of executable entities should be reset.
+    context : str
+        Project name to use for context resolution.
 
     Returns
     -------
@@ -200,7 +234,7 @@ def import_function(file: str) -> Function:
     --------
     >>> obj = import_function("my-function.yaml")
     """
-    return context_processor.import_executable_entity(file)
+    return context_processor.import_executable_entity(file, key, reset_id, context)
 
 
 def load_function(file: str) -> Function:
@@ -256,7 +290,6 @@ def delete_function(
     entity_id: str | None = None,
     delete_all_versions: bool = False,
     cascade: bool = True,
-    **kwargs,
 ) -> dict:
     """
     Delete object from backend.
@@ -270,11 +303,10 @@ def delete_function(
     entity_id : str
         Entity ID.
     delete_all_versions : bool
-        Delete all versions of the named entity. If True, use entity name instead of entity key as identifier.
+        Delete all versions of the named entity.
+        If True, use entity name instead of entity key as identifier.
     cascade : bool
         Cascade delete.
-    **kwargs : dict
-        Parameters to pass to the API call.
 
     Returns
     -------
@@ -298,5 +330,4 @@ def delete_function(
         entity_id=entity_id,
         delete_all_versions=delete_all_versions,
         cascade=cascade,
-        **kwargs,
     )

digitalhub/entities/model/_base/entity.py

@@ -9,7 +9,7 @@ import typing
 from digitalhub.entities._base.material.entity import MaterialEntity
 from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.entities._commons.metrics import MetricType, set_metrics, validate_metric_value
-from digitalhub.entities._processors.context import context_processor
+from digitalhub.entities._processors.processors import context_processor
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -81,10 +81,6 @@ class Model(MaterialEntity):
         single_value : bool
             If True, value is a single value.
 
-        Returns
-        -------
-        None
-
         Examples
         --------
         Log a new value in a list
@@ -94,13 +90,27 @@ class Model(MaterialEntity):
         >>> entity.log_metric("loss", 0.0019)
 
         Log a list of values and append them to existing metric:
-        >>> entity.log_metric("loss", [0.0018, 0.0015])
+        >>> entity.log_metric(
+        ...     "loss",
+        ...     [
+        ...         0.0018,
+        ...         0.0015,
+        ...     ],
+        ... )
 
         Log a single value (not represented as list):
-        >>> entity.log_metric("accuracy", 0.9, single_value=True)
+        >>> entity.log_metric(
+        ...     "accuracy",
+        ...     0.9,
+        ...     single_value=True,
+        ... )
 
         Log a list of values and overwrite existing metric:
-        >>> entity.log_metric("accuracy", [0.8, 0.9], overwrite=True)
+        >>> entity.log_metric(
+        ...     "accuracy",
+        ...     [0.8, 0.9],
+        ...     overwrite=True,
+        ... )
         """
         self._set_metrics(key, value, overwrite, single_value)
         context_processor.update_metric(self.project, self.ENTITY_TYPE, self.id, key, self.status.metrics[key])
@@ -121,19 +131,59 @@ class Model(MaterialEntity):
         overwrite : bool
             If True, overwrite existing metrics.
 
-        Returns
-        -------
-        None
+        Examples
+        --------
+        Log multiple metrics at once
+        >>> entity.log_metrics(
+        ...     {
+        ...         "loss": 0.002,
+        ...         "accuracy": 0.95,
+        ...     }
+        ... )
+
+        Log metrics with lists and single values
+        >>> entity.log_metrics(
+        ...     {
+        ...         "loss": [
+        ...             0.1,
+        ...             0.05,
+        ...         ],
+        ...         "epoch": 10,
+        ...     }
+        ... )
+
+        Append to existing metrics (default behavior)
+        >>> entity.log_metrics(
+        ...     {
+        ...         "loss": 0.001,
+        ...         "accuracy": 0.96,
+        ...     }
+        ... )  # Appends to existing
+
+        Overwrite existing metrics
+        >>> entity.log_metrics(
+        ...     {
+        ...         "loss": 0.0005,
+        ...         "accuracy": 0.98,
+        ...     },
+        ...     overwrite=True,
+        ... )
 
         See also
         --------
         log_metric
         """
         for key, value in metrics.items():
+            # For lists, use log_metric which handles appending correctly
             if isinstance(value, list):
                 self.log_metric(key, value, overwrite)
+
+            # For single values, check if we should append or create new
             else:
-                self.log_metric(key, value, overwrite, single_value=True)
+                if not overwrite and key in self.status.metrics:
+                    self.log_metric(key, value)
+                else:
+                    self.log_metric(key, value, overwrite, single_value=True)
 
     ##############################
     # Helper methods
@@ -142,10 +192,6 @@ class Model(MaterialEntity):
     def _get_metrics(self) -> None:
         """
         Get model metrics from backend.
-
-        Returns
-        -------
-        None
         """
         self.status.metrics = context_processor.read_metrics(
             project=self.project,
@@ -173,10 +219,6 @@ class Model(MaterialEntity):
             If True, overwrite existing metric.
         single_value : bool
             If True, value is a single value.
-
-        Returns
-        -------
-        None
         """
         value = validate_metric_value(value)
         self.status.metrics = set_metrics(