digitalhub 0.9.1__py3-none-any.whl → 0.10.0b0__py3-none-any.whl

digitalhub/stores/s3/models.py

@@ -0,0 +1,21 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class S3StoreConfig(BaseModel):
+    """
+    S3 store configuration class.
+    """
+
+    endpoint_url: str
+    """S3 endpoint URL."""
+
+    aws_access_key_id: str
+    """AWS access key ID."""
+
+    aws_secret_access_key: str
+    """AWS secret access key."""
+
+    bucket_name: str
+    """S3 bucket name."""

digitalhub/stores/s3/store.py

@@ -10,42 +10,26 @@ import botocore.client  # pylint: disable=unused-import
 from botocore.exceptions import ClientError
 
 from digitalhub.readers.api import get_reader_by_object
-from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.stores._base.store import Store
+from digitalhub.stores.s3.configurator import S3StoreConfigurator
+from digitalhub.stores.s3.utils import get_bucket_name
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
-from digitalhub.utils.s3_utils import get_bucket_name
 
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
 
-class S3StoreConfig(StoreConfig):
-    """
-    S3 store configuration class.
-    """
-
-    endpoint_url: str
-    """S3 endpoint URL."""
-
-    aws_access_key_id: str
-    """AWS access key ID."""
-
-    aws_secret_access_key: str
-    """AWS secret access key."""
-
-    bucket_name: str
-    """S3 bucket name."""
-
-
 class S3Store(Store):
     """
     S3 store class. It implements the Store interface and provides methods to fetch and persist
     artifacts on S3 based storage.
     """
 
-    def __init__(self, name: str, store_type: str, config: S3StoreConfig) -> None:
-        super().__init__(name, store_type)
-        self.config = config
+    def __init__(self, config: dict | None = None) -> None:
+        super().__init__()
+        self._configurator = S3StoreConfigurator()
+        self._configurator.configure(config)
 
     ##############################
     # I/O methods
@@ -512,11 +496,7 @@ class S3Store(Store):
         S3Client
             Returns a client object that interacts with the S3 storage service.
         """
-        cfg = {
-            "endpoint_url": self.config.endpoint_url,
-            "aws_access_key_id": self.config.aws_access_key_id,
-            "aws_secret_access_key": self.config.aws_secret_access_key,
-        }
+        cfg = self._configurator.get_s3_creds()
         return boto3.client("s3", **cfg)
 
     def _check_factory(self, root: str) -> tuple[S3Client, str]:

digitalhub/{utils/s3_utils.py → stores/s3/utils.py}

@@ -6,6 +6,10 @@ from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
+from digitalhub.stores.s3.enums import S3StoreEnv
+
+DEFAULT_BUCKET = "datalake"
+
 
 def get_bucket_name(path: str) -> str:
     """
@@ -59,11 +63,11 @@ def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     -------
     None
     """
-    s3 = boto3_client("s3", endpoint_url=os.getenv("S3_ENDPOINT_URL"))
+    s3 = boto3_client("s3", endpoint_url=os.getenv(S3StoreEnv.ENDPOINT_URL.value))
     s3.download_file(bucket, key, filename)
 
 
-def get_s3_bucket() -> str | None:
+def get_s3_bucket_from_env() -> str | None:
     """
     Function to get S3 bucket name.
 
@@ -72,4 +76,4 @@ def get_s3_bucket() -> str | None:
     str
         The S3 bucket name.
     """
-    return os.getenv("S3_BUCKET_NAME", "datalake")
+    return os.getenv(S3StoreEnv.BUCKET_NAME.value, DEFAULT_BUCKET)

digitalhub/stores/sql/configurator.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from digitalhub.configurator.configurator import configurator
+from digitalhub.stores.sql.enums import SqlStoreEnv
+from digitalhub.stores.sql.models import SqlStoreConfig
+from digitalhub.utils.exceptions import StoreError
+
+
+class SqlStoreConfigurator:
+    """
+    Configure the store by getting the credentials from user
+    provided config or from environment.
+    """
+
+    def __init__(self, config: dict | None = None) -> None:
+        self.configure(config)
+
+    ##############################
+    # Configuration methods
+    ##############################
+
+    def configure(self, config: dict | None = None) -> None:
+        """
+        Configure the store by getting the credentials from user
+        provided config or from environment.
+
+        Parameters
+        ----------
+        config : dict
+            Configuration dictionary.
+
+        Returns
+        -------
+        None
+        """
+        if config is None:
+            self._get_config()
+        else:
+            config: SqlStoreConfig = SqlStoreConfig(**config)
+            for pair in [
+                (SqlStoreEnv.USER.value, config.user),
+                (SqlStoreEnv.PASSWORD.value, config.password),
+                (SqlStoreEnv.HOST.value, config.host),
+                (SqlStoreEnv.PORT.value, config.port),
+                (SqlStoreEnv.DATABASE.value, config.database),
+            ]:
+                configurator.set_credential(*pair)
+
+    def get_sql_conn_string(self) -> str:
+        """
+        Get the connection string from environment variables.
+
+        Returns
+        -------
+        str
+            The connection string.
+        """
+        creds = configurator.get_all_cred()
+        try:
+            user = creds[SqlStoreEnv.USER.value]
+            password = creds[SqlStoreEnv.PASSWORD.value]
+            host = creds[SqlStoreEnv.HOST.value]
+            port = creds[SqlStoreEnv.PORT.value]
+            database = creds[SqlStoreEnv.DATABASE.value]
+            return f"postgresql://{user}:{password}@{host}:{port}/{database}"
+        except KeyError as e:
+            raise StoreError(f"Missing credentials for SQL store. {str(e)}")
+
+    def _get_config(self) -> None:
+        """
+        Get the credentials from environment variables.
+
+        Returns
+        -------
+        None
+        """
+        user = configurator.load_var(SqlStoreEnv.USER.value)
+        password = configurator.load_var(SqlStoreEnv.PASSWORD.value)
+        host = configurator.load_var(SqlStoreEnv.HOST.value)
+        port = configurator.load_var(SqlStoreEnv.PORT.value)
+        database = configurator.load_var(SqlStoreEnv.DATABASE.value)
+        if user is None or password is None or host is None or port is None or database is None:
+            raise StoreError("Missing credentials for SQL store.")
+        configurator.set_credential(SqlStoreEnv.USER.value, user)
+        configurator.set_credential(SqlStoreEnv.PASSWORD.value, password)
+        configurator.set_credential(SqlStoreEnv.HOST.value, host)
+        configurator.set_credential(SqlStoreEnv.PORT.value, port)
+        configurator.set_credential(SqlStoreEnv.DATABASE.value, database)

digitalhub/stores/sql/enums.py

@@ -0,0 +1,16 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class SqlStoreEnv(Enum):
+    """
+    SqlStore environment
+    """
+
+    HOST = "DB_HOST"
+    PORT = "DB_PORT"
+    USER = "DB_USER"
+    PASSWORD = "DB_PASSWORD"
+    DATABASE = "DB_DATABASE"
+    PG_SCHEMA = "DB_SCHEMA"

digitalhub/stores/sql/models.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pydantic import BaseModel
+
+
+class SqlStoreConfig(BaseModel):
+    """
+    SQL store configuration class.
+    """
+
+    host: str
+    """SQL host."""
+
+    port: int
+    """SQL port."""
+
+    user: str
+    """SQL user."""
+
+    password: str
+    """SQL password."""
+
+    database: str
+    """SQL database name."""

digitalhub/stores/sql/store.py

@@ -1,39 +1,22 @@
 from __future__ import annotations
 
+import typing
 from pathlib import Path
 from typing import Any
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from sqlalchemy import MetaData, Table, create_engine
+from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.engine import Engine
-from sqlalchemy.engine.row import LegacyRow
 from sqlalchemy.exc import SQLAlchemyError
 
 from digitalhub.readers.api import get_reader_by_object
-from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.stores._base.store import Store
+from digitalhub.stores.sql.configurator import SqlStoreConfigurator
 from digitalhub.utils.exceptions import StoreError
 
-
-class SQLStoreConfig(StoreConfig):
-    """
-    SQL store configuration class.
-    """
-
-    host: str
-    """SQL host."""
-
-    port: int
-    """SQL port."""
-
-    user: str
-    """SQL user."""
-
-    password: str
-    """SQL password."""
-
-    database: str
-    """SQL database name."""
+if typing.TYPE_CHECKING:
+    from sqlalchemy.engine.row import Row
 
 
 class SqlStore(Store):
@@ -42,9 +25,10 @@ class SqlStore(Store):
     artifacts on SQL based storage.
     """
 
-    def __init__(self, name: str, store_type: str, config: SQLStoreConfig) -> None:
-        super().__init__(name, store_type)
-        self.config = config
+    def __init__(self, config: dict | None = None) -> None:
+        super().__init__()
+        self._configurator = SqlStoreConfigurator()
+        self._configurator.configure(config)
 
     ##############################
     # I/O methods
@@ -160,12 +144,12 @@ class SqlStore(Store):
 
         # Read the table from the database
         sa_table = Table(table, MetaData(), autoload_with=engine)
-        query = sa_table.select()
+        stm = select(sa_table)
         with engine.begin() as conn:
-            result: list[LegacyRow] = conn.execute(query).fetchall()
+            result: list[Row] = conn.execute(stm).fetchall()
 
         # Parse the result
-        data = self._parse_result(result)
+        data = {col: [row[idx] for row in result] for idx, col in enumerate(sa_table.columns.keys())}
 
         # Convert the result to a pyarrow table and
         # write the pyarrow table to a Parquet file
@@ -245,10 +229,7 @@ class SqlStore(Store):
         str
             The connection string.
         """
-        return (
-            f"postgresql://{self.config.user}:{self.config.password}@"
-            f"{self.config.host}:{self.config.port}/{self.config.database}"
-        )
+        return self._configurator.get_sql_conn_string()
 
     def _get_engine(self, schema: str | None = None) -> Engine:
         """
@@ -377,27 +358,3 @@ class SqlStore(Store):
         except SQLAlchemyError:
             engine.dispose()
             raise StoreError("No access to db!")
-
-    @staticmethod
-    def _parse_result(result: list[LegacyRow]) -> dict:
-        """
-        Convert a list of list of tuples to a dict.
-
-        Parameters
-        ----------
-        result : list[LegacyRow]
-            The data to convert.
-
-        Returns
-        -------
-        dict
-            The converted data.
-        """
-        data_list = [row.items() for row in result]
-        data = {}
-        for row in data_list:
-            for column_name, value in row:
-                if column_name not in data:
-                    data[column_name] = []
-                data[column_name].append(value)
-        return data

digitalhub/utils/exceptions.py

@@ -71,3 +71,9 @@ class ReaderError(Exception):
     """
     Raised when incontered errors on readers.
     """
+
+
+class ClientError(Exception):
+    """
+    Raised when incontered errors on clients.
+    """

digitalhub/utils/generic_utils.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import base64
 import importlib.util as imputil
 import json
-from datetime import datetime
+from datetime import date, datetime, time
 from enum import Enum
 from pathlib import Path
 from typing import Any, Callable
@@ -120,14 +120,14 @@ def extract_archive(path: Path, filename: Path) -> None:
         zip_file.extractall(path)
 
 
-class MyEncoder(json.JSONEncoder):
+class CustomJsonEncoder(json.JSONEncoder):
     """
-    Custom JSON encoder to handle numpy types.
+    Custom JSON encoder to handle json dumps.
     """
 
     def default(self, obj: Any) -> Any:
         """
-        Convert numpy types to json.
+        Convert an object to json.
 
         Parameters
         ----------
@@ -147,11 +147,12 @@ class MyEncoder(json.JSONEncoder):
             return float(obj)
         elif isinstance(obj, np.ndarray):
             return obj.tolist()
-        else:
-            return str(obj)
+        elif isinstance(obj, (datetime, date, time)):
+            return obj.isoformat()
+        return str(obj)
 
 
-def dict_to_json(struct: dict) -> str:
+def dump_json(struct: Any) -> str:
     """
     Convert a dict to json.
 
@@ -165,7 +166,7 @@ def dict_to_json(struct: dict) -> str:
     str
         The json string.
     """
-    return json.dumps(struct, cls=MyEncoder)
+    return json.dumps(struct, cls=CustomJsonEncoder).encode("utf-8")
 
 
 def slugify_string(filename: str) -> str:

digitalhub/utils/uri_utils.py

@@ -99,7 +99,7 @@ def map_uri_scheme(uri: str) -> str:
     if scheme in list_enum(LocalSchemes):
         return SchemeCategory.LOCAL.value
     if scheme in list_enum(InvalidLocalSchemes):
-        raise ValueError("For local uri, do not use any scheme.")
+        raise ValueError("For local URI, do not use any scheme.")
     if scheme in list_enum(RemoteSchemes):
         return SchemeCategory.REMOTE.value
     if scheme in list_enum(S3Schemes):

{digitalhub-0.9.1.dist-info → digitalhub-0.10.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: digitalhub
-Version: 0.9.1
+Version: 0.10.0b0
 Summary: Python SDK for Digitalhub
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
@@ -231,7 +231,7 @@ Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: boto3
 Requires-Dist: pydantic
-Requires-Dist: sqlalchemy<2
+Requires-Dist: sqlalchemy
 Requires-Dist: pyarrow
 Requires-Dist: numpy<2
 Requires-Dist: requests
@@ -241,10 +241,10 @@ Requires-Dist: GitPython>=3
 Requires-Dist: psycopg2-binary
 Requires-Dist: python-slugify
 Provides-Extra: full
-Requires-Dist: pandas<2.2,>=1.2; extra == "full"
+Requires-Dist: pandas; extra == "full"
 Requires-Dist: mlflow; extra == "full"
 Provides-Extra: pandas
-Requires-Dist: pandas<2.2,>=1.2; extra == "pandas"
+Requires-Dist: pandas; extra == "pandas"
 Provides-Extra: mlflow
 Requires-Dist: mlflow; extra == "mlflow"
 Provides-Extra: dev
@@ -260,7 +260,6 @@ Requires-Dist: numpydoc>=1.6; extra == "docs"
 
 # Digitalhub Library
 
-The Digitalhub SDK library is used to manage entities and executions in Digitalhub from Python.
-It comes with a suite of tools to help you manage your projects and executions. It exposes CRUD methods to create, read, update and delete entities, and objects methods to excute functions or workflows, collect or store execution results and data.
+The Digitalhub library is a python tool for managing projects, entities and executions in Digitalhub. It exposes CRUD methods to create, read, update and delete entities, tools to execute functions or workflows, collect or store execution results and data.
 
 A more detailed description of the library can be found in the [official documentation](https://scc-digitalhub.github.io/sdk-docs/).