digitalhub 0.9.0__py3-none-any.whl → 0.9.0b0__py3-none-any.whl

digitalhub/entities/task/_base/spec.py

@@ -22,8 +22,6 @@ class TaskSpecFunction(TaskSpec):
         envs: list[dict] | None = None,
         secrets: list[str] | None = None,
         profile: str | None = None,
-        runtime_class: str | None = None,
-        priority_class: str | None = None,
         **kwargs,
     ) -> None:
         self.function = function
@@ -35,8 +33,6 @@ class TaskSpecFunction(TaskSpec):
         self.envs = envs
         self.secrets = secrets
         self.profile = profile
-        self.runtime_class = runtime_class
-        self.priority_class = priority_class
 
 
 class TaskSpecWorkflow(TaskSpec):
@@ -53,8 +49,6 @@ class TaskSpecWorkflow(TaskSpec):
         envs: list[dict] | None = None,
         secrets: list[str] | None = None,
         profile: str | None = None,
-        runtime_class: str | None = None,
-        priority_class: str | None = None,
         **kwargs,
     ) -> None:
         self.workflow = workflow

digitalhub/entities/workflow/_base/entity.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import typing
 
 from digitalhub.entities._base.executable.entity import ExecutableEntity
-from digitalhub.entities._commons.enums import EntityTypes, Relationship
+from digitalhub.entities._commons.enums import EntityTypes
 from digitalhub.factory.api import get_run_kind, get_task_kind_from_action
 from digitalhub.utils.exceptions import BackendError
 
@@ -44,7 +44,7 @@ class Workflow(ExecutableEntity):
     def run(
         self,
         action: str,
-        wait: bool = False,
+        wait: bool = True,
         log_info: bool = True,
         **kwargs,
     ) -> Run:
@@ -55,10 +55,7 @@ class Workflow(ExecutableEntity):
         ----------
         action : str
             Action to execute.
-        wait : bool
-            Flag to wait for execution to finish.
-        log_info : bool
-            Flag to log information while waiting.
+
         **kwargs : dict
             Keyword arguments passed to Run builder.
 
@@ -79,12 +76,7 @@ class Workflow(ExecutableEntity):
             raise BackendError("Cannot run workflow with local backend.")
 
         # Run task
-        run = task.run(run_kind, save=False, local_execution=False, **kwargs)
-
-        # Set as run's parent
-        run.add_relationship(Relationship.RUN_OF.value, self.key)
-        run.save()
-
+        run = task.run(run_kind, local_execution=False, **kwargs)
         if wait:
             return run.wait(log_info=log_info)
         return run

digitalhub/stores/_base/store.py

@@ -52,17 +52,13 @@ class Store:
         """
 
     @abstractmethod
-    def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
         """
         Method to upload artifact to storage.
         """
 
     @abstractmethod
-    def get_file_info(
-        self,
-        root: str,
-        paths: list[tuple[str, str]],
-    ) -> list[dict]:
+    def get_file_info(self, paths: list[str]) -> list[dict]:
         """
         Method to get file metadata.
         """

digitalhub/stores/local/store.py

@@ -61,7 +61,7 @@ class LocalStore(Store):
         """
         raise StoreError("Local store does not support download.")
 
-    def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
         """
         Upload an artifact to storage.
 
@@ -72,11 +72,7 @@ class LocalStore(Store):
         """
         raise StoreError("Local store does not support upload.")
 
-    def get_file_info(
-        self,
-        root: str,
-        paths: list[tuple[str, str]],
-    ) -> list[dict]:
+    def get_file_info(self, paths: list[str]) -> list[dict]:
         """
         Method to get file metadata.
 

digitalhub/stores/remote/store.py

@@ -69,7 +69,7 @@ class RemoteStore(Store):
 
         return self._download_file(root, dst, overwrite)
 
-    def upload(self, src: str | list[str], dst: str) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
         """
         Upload an artifact to storage.
 
@@ -80,11 +80,7 @@ class RemoteStore(Store):
         """
         raise StoreError("Remote HTTP store does not support upload.")
 
-    def get_file_info(
-        self,
-        root: str,
-        paths: list[tuple[str, str]],
-    ) -> list[dict]:
+    def get_file_info(self, paths: list[str]) -> list[dict]:
         """
         Get file information from HTTP(s) storage.
 

digitalhub/stores/s3/store.py

@@ -13,7 +13,6 @@ from digitalhub.readers.api import get_reader_by_object
 from digitalhub.stores._base.store import Store, StoreConfig
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
-from digitalhub.utils.s3_utils import get_bucket_name
 
 # Type aliases
 S3Client = Type["botocore.client.S3"]
@@ -77,7 +76,7 @@ class S3Store(Store):
         str
             Destination path of the downloaded artifact.
         """
-        client, bucket = self._check_factory(root)
+        client, bucket = self._check_factory()
 
         # Build destination directory
         if dst.suffix == "":
@@ -127,18 +126,14 @@ class S3Store(Store):
                 return str(Path(dst, trees[0]))
         return str(dst)
 
-    def upload(
-        self,
-        src: str | list[str],
-        dst: str,
-    ) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
         """
         Upload an artifact to storage.
 
         Parameters
         ----------
-        src : str | list[str]
-            Source(s).
+        src : str
+            List of sources.
         dst : str
             The destination of the artifact on storage.
 
@@ -148,11 +143,18 @@ class S3Store(Store):
             Returns the list of destination and source paths of the uploaded artifacts.
         """
         # Destination handling
-        key = self._get_key(dst)
 
-        # Source handling (files list, dir or single file)
-        src_is_list = isinstance(src, list)
-        if not src_is_list:
+        # If no destination is provided, build key from source
+        # Otherwise build key from destination
+        if dst is None:
+            raise StoreError(
+                "Destination must be provided. If source is a list of files or a directory, destination must be a partition, e.g. 's3://bucket/partition/' otherwise a destination key, e.g. 's3://bucket/key'"
+            )
+        else:
+            dst = self._get_key(dst)
+
+        # Source handling
+        if not isinstance(src, list):
             self._check_local_src(src)
             src_is_dir = Path(src).is_dir()
         else:
@@ -163,31 +165,21 @@ class S3Store(Store):
                 src = src[0]
 
         # If source is a directory, destination must be a partition
-        if (src_is_dir or src_is_list) and not dst.endswith("/"):
-            raise StoreError(
-                "If source is a list of files or a directory, "
-                "destination must be a partition, e.g. 's3://bucket/partition/'"
-            )
-
-        # S3 client
-        client, bucket = self._check_factory(dst)
+        if (src_is_dir or isinstance(src, list)) and not dst.endswith("/"):
+            raise StoreError("Destination must be a partition if the source is a directory or a list of files.")
 
         # Directory
         if src_is_dir:
-            return self._upload_dir(src, key, client, bucket)
+            return self._upload_dir(src, dst)
 
         # List of files
-        elif src_is_list:
-            return self._upload_file_list(src, key, client, bucket)
+        elif isinstance(src, list):
+            return self._upload_file_list(src, dst)
 
         # Single file
-        return self._upload_single_file(src, key, client, bucket)
+        return self._upload_single_file(src, dst)
 
-    def upload_fileobject(
-        self,
-        src: BytesIO,
-        dst: str,
-    ) -> str:
+    def upload_fileobject(self, src: BytesIO, dst: str) -> str:
         """
         Upload an BytesIO to S3 based storage.
 
@@ -196,23 +188,18 @@ class S3Store(Store):
         src : BytesIO
             The source object to be persisted.
         dst : str
-            The destination path of the artifact.
+            The destination partition for the artifact.
 
         Returns
         -------
         str
             S3 key of the uploaded artifact.
         """
-        client, bucket = self._check_factory(dst)
-        key = self._get_key(dst)
-        self._upload_fileobject(src, key, client, bucket)
-        return f"s3://{bucket}/{key}"
+        client, bucket = self._check_factory()
+        self._upload_fileobject(src, dst, client, bucket)
+        return f"s3://{bucket}/{dst}"
 
-    def get_file_info(
-        self,
-        root: str,
-        paths: list[tuple[str, str]],
-    ) -> list[dict]:
+    def get_file_info(self, paths: list[tuple[str, str]]) -> list[dict]:
         """
         Method to get file metadata.
 
@@ -226,7 +213,7 @@ class S3Store(Store):
         list[dict]
             Returns files metadata.
         """
-        client, bucket = self._check_factory(root)
+        client, bucket = self._check_factory()
 
         infos = []
         for i in paths:
@@ -277,13 +264,7 @@ class S3Store(Store):
         # Download file
         client.download_file(bucket, key, dst_pth)
 
-    def _upload_dir(
-        self,
-        src: str,
-        dst: str,
-        client: S3Client,
-        bucket: str,
-    ) -> list[tuple[str, str]]:
+    def _upload_dir(self, src: str, dst: str) -> list[tuple[str, str]]:
         """
         Upload directory to storage.
 
@@ -293,16 +274,14 @@ class S3Store(Store):
             List of sources.
         dst : str
             The destination of the artifact on storage.
-        client : S3Client
-            The S3 client object.
-        bucket : str
-            The name of the S3 bucket.
 
         Returns
         -------
         list[tuple[str, str]]
             Returns the list of destination and source paths of the uploaded artifacts.
         """
+        client, bucket = self._check_factory()
+
         # Get list of files
         src_pth = Path(src)
         files = [i for i in src_pth.rglob("*") if i.is_file()]
@@ -320,13 +299,7 @@ class S3Store(Store):
             paths.append((k, str(f.relative_to(src_pth))))
         return paths
 
-    def _upload_file_list(
-        self,
-        src: list[str],
-        dst: str,
-        client: S3Client,
-        bucket: str,
-    ) -> list[tuple[str, str]]:
+    def _upload_file_list(self, src: list[str], dst: str) -> list[tuple[str, str]]:
         """
         Upload list of files to storage.
 
@@ -336,16 +309,13 @@ class S3Store(Store):
             List of sources.
         dst : str
             The destination of the artifact on storage.
-        client : S3Client
-            The S3 client object.
-        bucket : str
-            The name of the S3 bucket.
 
         Returns
         -------
         list[tuple[str, str]]
             Returns the list of destination and source paths of the uploaded artifacts.
         """
+        client, bucket = self._check_factory()
         files = src
         keys = []
         for i in files:
@@ -360,13 +330,7 @@ class S3Store(Store):
             paths.append((k, Path(f).name))
         return paths
 
-    def _upload_single_file(
-        self,
-        src: str,
-        dst: str,
-        client: S3Client,
-        bucket: str,
-    ) -> str:
+    def _upload_single_file(self, src: str, dst: str) -> str:
         """
         Upload a single file to storage.
 
@@ -376,16 +340,14 @@ class S3Store(Store):
             List of sources.
         dst : str
             The destination of the artifact on storage.
-        client : S3Client
-            The S3 client object.
-        bucket : str
-            The name of the S3 bucket.
 
         Returns
         -------
         str
             Returns the list of destination and source paths of the uploaded artifacts.
         """
+        client, bucket = self._check_factory()
+
         if dst.endswith("/"):
             dst = f"{dst.removeprefix('/')}{Path(src).name}"
 
@@ -395,12 +357,7 @@ class S3Store(Store):
         return [(dst, name)]
 
     @staticmethod
-    def _upload_file(
-        src: str,
-        key: str,
-        client: S3Client,
-        bucket: str,
-    ) -> None:
+    def _upload_file(src: str, key: str, client: S3Client, bucket: str) -> None:
         """
         Upload a file to S3 based storage. The function checks if the
         bucket is accessible.
@@ -427,12 +384,7 @@ class S3Store(Store):
         client.upload_file(Filename=src, Bucket=bucket, Key=key, ExtraArgs=extra_args)
 
     @staticmethod
-    def _upload_fileobject(
-        fileobj: BytesIO,
-        key: str,
-        client: S3Client,
-        bucket: str,
-    ) -> None:
+    def _upload_fileobject(fileobj: BytesIO, key: str, client: S3Client, bucket: str) -> None:
         """
         Upload a fileobject to S3 based storage. The function checks if the bucket is accessible.
 
@@ -457,13 +409,7 @@ class S3Store(Store):
     # Datastore methods
     ##############################
 
-    def write_df(
-        self,
-        df: Any,
-        dst: str,
-        extension: str | None = None,
-        **kwargs,
-    ) -> str:
+    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
         """
         Write a dataframe to S3 based storage. Kwargs are passed to df.to_parquet().
 
@@ -473,8 +419,6 @@ class S3Store(Store):
             The dataframe.
         dst : str
             The destination path on S3 based storage.
-        extension : str
-            The extension of the file.
         **kwargs : dict
             Keyword arguments.
 
@@ -486,13 +430,15 @@ class S3Store(Store):
         fileobj = BytesIO()
         reader = get_reader_by_object(df)
         reader.write_df(df, fileobj, extension=extension, **kwargs)
-        return self.upload_fileobject(fileobj, dst)
+
+        key = self._get_key(dst)
+        return self.upload_fileobject(fileobj, key)
 
     ##############################
     # Helper methods
     ##############################
 
-    def _get_bucket(self, root: str) -> str:
+    def _get_bucket(self) -> str:
         """
         Get the name of the S3 bucket from the URI.
 
@@ -501,7 +447,7 @@ class S3Store(Store):
         str
             The name of the S3 bucket.
         """
-        return get_bucket_name(root)
+        return str(self.config.bucket_name)
 
     def _get_client(self) -> S3Client:
         """
@@ -519,7 +465,7 @@ class S3Store(Store):
         }
         return boto3.client("s3", **cfg)
 
-    def _check_factory(self, root: str) -> tuple[S3Client, str]:
+    def _check_factory(self) -> tuple[S3Client, str]:
         """
         Check if the S3 bucket is accessible by sending a head_bucket request.
 
@@ -529,7 +475,7 @@ class S3Store(Store):
             A tuple containing the S3 client object and the name of the S3 bucket.
         """
         client = self._get_client()
-        bucket = self._get_bucket(root)
+        bucket = self._get_bucket()
         self._check_access_to_storage(client, bucket)
         return client, bucket
 

digitalhub/stores/sql/store.py

@@ -99,11 +99,7 @@ class SqlStore(Store):
         table = self._get_table_name(root)
         return self._download_table(schema, table, str(dst))
 
-    def upload(
-        self,
-        src: str | list[str],
-        dst: str,
-    ) -> list[tuple[str, str]]:
+    def upload(self, src: str | list[str], dst: str | None = None) -> list[tuple[str, str]]:
         """
         Upload an artifact to storage.
 
@@ -114,11 +110,7 @@ class SqlStore(Store):
         """
         raise StoreError("SQL store does not support upload.")
 
-    def get_file_info(
-        self,
-        root: str,
-        paths: list[tuple[str, str]],
-    ) -> list[dict]:
+    def get_file_info(self, paths: list[str]) -> list[dict]:
         """
         Get file information from SQL based storage.
 

digitalhub/utils/s3_utils.py

@@ -7,23 +7,6 @@ from urllib.parse import urlparse
 from boto3 import client as boto3_client
 
 
-def get_bucket_name(path: str) -> str:
-    """
-    Get bucket name from path.
-
-    Parameters
-    ----------
-    path : str
-        The source path to get the key from.
-
-    Returns
-    -------
-    str
-        The bucket name.
-    """
-    return urlparse(path).netloc
-
-
 def get_bucket_and_key(path: str) -> tuple[str, str]:
     """
     Get bucket and key from path.

{digitalhub-0.9.0.dist-info → digitalhub-0.9.0b0.dist-info}/LICENSE.txt

@@ -186,7 +186,7 @@ APPENDIX: How to apply the Apache License to your work.
    same "printed page" as the copyright notice for easier
    identification within third-party archives.
 
-Copyright 2024 DSLab, Fondazione Bruno Kessler
+Copyright [yyyy] [name of copyright owner]
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{digitalhub-0.9.0.dist-info → digitalhub-0.9.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: digitalhub
-Version: 0.9.0
+Version: 0.9.0b0
 Summary: Python SDK for Digitalhub
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
@@ -191,7 +191,7 @@ License:                               Apache License
            same "printed page" as the copyright notice for easier
            identification within third-party archives.
         
-        Copyright 2024 DSLab, Fondazione Bruno Kessler
+        Copyright [yyyy] [name of copyright owner]
         
         Licensed under the Apache License, Version 2.0 (the "License");
         you may not use this file except in compliance with the License.