digitalhub 0.10.0b7__py3-none-any.whl → 0.10.2__py3-none-any.whl

digitalhub/stores/data/s3/configurator.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+from botocore.config import Config
+
+from digitalhub.stores.configurator.configurator import configurator
+from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.data.s3.enums import S3StoreEnv
+from digitalhub.utils.exceptions import StoreError
+
+
+class S3StoreConfigurator:
+    """
+    Configure the store by getting the credentials from user
+    provided config or from environment.
+    """
+
+    required_vars = [
+        S3StoreEnv.ENDPOINT_URL,
+        S3StoreEnv.ACCESS_KEY_ID,
+        S3StoreEnv.SECRET_ACCESS_KEY,
+        S3StoreEnv.BUCKET_NAME,
+    ]
+    optional_vars = [
+        S3StoreEnv.REGION,
+        S3StoreEnv.SIGNATURE_VERSION,
+        S3StoreEnv.SESSION_TOKEN,
+    ]
+
+    def __init__(self, config: dict | None = None) -> None:
+        self.configure(config)
+
+    ##############################
+    # Configuration methods
+    ##############################
+
+    def configure(self, config: dict | None = None) -> None:
+        """
+        Configure the store by getting the credentials from user
+        provided config or from environment.
+
+        Parameters
+        ----------
+        config : dict
+            Configuration dictionary.
+
+        Returns
+        -------
+        None
+        """
+        self._get_env_config()
+
+    def get_boto3_client_config(self, origin: str) -> dict:
+        """
+        Get S3 credentials (access key, secret key,
+        session token and other config).
+
+        Parameters
+        ----------
+        origin : str
+            The origin of the credentials.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        if origin == CredsOrigin.ENV.value:
+            creds = self._get_env_config()
+        elif origin == CredsOrigin.FILE.value:
+            creds = self._get_file_config()
+        else:
+            raise StoreError(f"Unknown origin: {origin}")
+        return {
+            "endpoint_url": creds[S3StoreEnv.ENDPOINT_URL.value],
+            "aws_access_key_id": creds[S3StoreEnv.ACCESS_KEY_ID.value],
+            "aws_secret_access_key": creds[S3StoreEnv.SECRET_ACCESS_KEY.value],
+            "aws_session_token": creds[S3StoreEnv.SESSION_TOKEN.value],
+            "config": Config(
+                region_name=creds[S3StoreEnv.REGION.value],
+                signature_version=creds[S3StoreEnv.SIGNATURE_VERSION.value],
+            ),
+        }
+
+    def _get_env_config(self) -> dict:
+        """
+        Get the store configuration from environment variables.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        credentials = {
+            var.value: configurator.load_from_env(var.value) for var in self.required_vars + self.optional_vars
+        }
+        self._set_credentials(credentials)
+        return credentials
+
+    def _get_file_config(self) -> dict:
+        """
+        Get the store configuration from file.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        credentials = {
+            var.value: configurator.load_from_file(var.value) for var in self.required_vars + self.optional_vars
+        }
+        self._set_credentials(credentials)
+        return credentials
+
+    def _check_credentials(self, credentials: dict) -> None:
+        """
+        Check for missing credentials.
+
+        Parameters
+        ----------
+        credentials : dict
+            The credentials.
+
+        Returns
+        -------
+        None
+        """
+        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
+        if missing_vars:
+            raise StoreError(f"Missing credentials for S3 store: {', '.join(missing_vars)}")
+
+    def _set_credentials(self, credentials: dict) -> None:
+        """
+        Set the store credentials into the configurator.
+
+        Parameters
+        ----------
+        credentials : dict
+            The credentials.
+
+        Returns
+        -------
+        None
+        """
+        # Set credentials
+        for key, value in credentials.items():
+            configurator.set_credential(key, value)

digitalhub/stores/{s3 → data/s3}/store.py

@@ -10,10 +10,11 @@ import boto3
 import botocore.client  # pylint: disable=unused-import
 from botocore.exceptions import ClientError
 
-from digitalhub.readers.data.api import get_reader_by_object
-from digitalhub.stores._base.store import Store
-from digitalhub.stores.s3.configurator import S3StoreConfigurator
-from digitalhub.stores.s3.utils import get_bucket_name
+from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.data._base.store import Store
+from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
+from digitalhub.stores.data.s3.utils import get_bucket_name
+from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
@@ -591,16 +592,20 @@ class S3Store(Store):
         """
         return get_bucket_name(root)
 
-    def _get_client(self) -> S3Client:
+    def _get_client(self, cfg: dict) -> S3Client:
         """
         Get an S3 client object.
 
+        Parameters
+        ----------
+        cfg : dict
+            The configuration of the S3 client.
+
         Returns
         -------
         S3Client
             Returns a client object that interacts with the S3 storage service.
         """
-        cfg = self._configurator.get_boto3_client_config()
         return boto3.client("s3", **cfg)
 
     def _check_factory(self, root: str) -> tuple[S3Client, str]:
@@ -612,9 +617,20 @@ class S3Store(Store):
         tuple[S3Client, str]
             A tuple containing the S3 client object and the name of the S3 bucket.
         """
-        client = self._get_client()
         bucket = self._get_bucket(root)
-        self._check_access_to_storage(client, bucket)
+
+        # Try to get client from environment variables
+        try:
+            cfg = self._configurator.get_boto3_client_config(CredsOrigin.ENV.value)
+            client = self._get_client(cfg)
+            self._check_access_to_storage(client, bucket)
+
+        # Fallback to file
+        except StoreError:
+            cfg = self._configurator.get_boto3_client_config(CredsOrigin.FILE.value)
+            client = self._get_client(cfg)
+            self._check_access_to_storage(client, bucket)
+
         return client, bucket
 
     def _check_access_to_storage(self, client: S3Client, bucket: str) -> None:
@@ -639,8 +655,8 @@ class S3Store(Store):
         """
         try:
             client.head_bucket(Bucket=bucket)
-        except ClientError as e:
-            raise ClientError("No access to s3 bucket!") from e
+        except ClientError:
+            raise StoreError("No access to s3 bucket!")
 
     @staticmethod
     def _get_key(path: str) -> str:

digitalhub/stores/{s3 → data/s3}/utils.py

@@ -6,7 +6,7 @@ from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
-from digitalhub.stores.s3.enums import S3StoreEnv
+from digitalhub.stores.data.s3.enums import S3StoreEnv
 
 DEFAULT_BUCKET = "datalake"
 

digitalhub/stores/data/sql/configurator.py

@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+from digitalhub.stores.configurator.configurator import configurator
+from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.data.sql.enums import SqlStoreEnv
+from digitalhub.utils.exceptions import StoreError
+
+
+class SqlStoreConfigurator:
+    """
+    Configure the store by getting the credentials from user
+    provided config or from environment.
+    """
+
+    variables = [SqlStoreEnv.USERNAME, SqlStoreEnv.PASSWORD, SqlStoreEnv.HOST, SqlStoreEnv.PORT, SqlStoreEnv.DATABASE]
+
+    def __init__(self, config: dict | None = None) -> None:
+        self.configure(config)
+
+    ##############################
+    # Configuration methods
+    ##############################
+
+    def configure(self, config: dict | None = None) -> None:
+        """
+        Configure the store by getting the credentials from user
+        provided config or from environment.
+
+        Parameters
+        ----------
+        config : dict
+            Configuration dictionary.
+
+        Returns
+        -------
+        None
+        """
+        self._get_env_config()
+
+    def get_sql_conn_string(self, origin: str) -> str:
+        """
+        Get the connection string from environment variables.
+
+        Parameters
+        ----------
+        origin : str
+            The origin of the credentials.
+
+        Returns
+        -------
+        str
+            The connection string.
+        """
+        if origin == CredsOrigin.ENV.value:
+            creds = self._get_env_config()
+        elif origin == CredsOrigin.FILE.value:
+            creds = self._get_file_config()
+        else:
+            raise StoreError(f"Unknown origin: {origin}")
+
+        user = creds[SqlStoreEnv.USERNAME.value]
+        password = creds[SqlStoreEnv.PASSWORD.value]
+        host = creds[SqlStoreEnv.HOST.value]
+        port = creds[SqlStoreEnv.PORT.value]
+        database = creds[SqlStoreEnv.DATABASE.value]
+        return f"postgresql://{user}:{password}@{host}:{port}/{database}"
+
+    def _get_env_config(self) -> dict:
+        """
+        Get the store configuration from environment variables.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        credentials = {var.value: configurator.load_from_env(var.value) for var in self.variables}
+        self._set_credentials(credentials)
+        return credentials
+
+    def _get_file_config(self) -> dict:
+        """
+        Get the store configuration from file.
+
+        Returns
+        -------
+        dict
+            The credentials.
+        """
+        credentials = {var.value: configurator.load_from_file(var.value) for var in self.variables}
+        self._set_credentials(credentials)
+        return credentials
+
+    def _check_credentials(self, credentials: dict) -> None:
+        """
+        Check for missing credentials.
+
+        Parameters
+        ----------
+        credentials : dict
+            The credentials.
+
+        Returns
+        -------
+        None
+        """
+        missing_vars = [key for key, value in credentials.items() if value is None and key in self.required_vars]
+        if missing_vars:
+            raise StoreError(f"Missing credentials for SQL store: {', '.join(missing_vars)}")
+
+    def _set_credentials(self, credentials: dict) -> None:
+        """
+        Set the store credentials into the configurator.
+
+        Parameters
+        ----------
+        credentials : dict
+            The credentials.
+
+        Returns
+        -------
+        None
+        """
+        # Set credentials
+        for key, value in credentials.items():
+            configurator.set_credential(key, value)

digitalhub/stores/{sql → data/sql}/store.py

@@ -10,9 +10,10 @@ from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.engine import Engine
 from sqlalchemy.exc import SQLAlchemyError
 
-from digitalhub.readers.data.api import get_reader_by_object
-from digitalhub.stores._base.store import Store
-from digitalhub.stores.sql.configurator import SqlStoreConfigurator
+from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.data._base.store import Store
+from digitalhub.stores.data.sql.configurator import SqlStoreConfigurator
+from digitalhub.stores.readers.data.api import get_reader_by_object
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.types import SourcesOrListOfSources
 
@@ -284,23 +285,30 @@ class SqlStore(Store):
     # Helper methods
     ##############################
 
-    def _get_connection_string(self) -> str:
+    def _get_connection_string(self, origin: str) -> str:
         """
         Get the connection string.
 
+        Parameters
+        ----------
+        origin : str
+            The origin of the credentials.
+
         Returns
         -------
         str
             The connection string.
         """
-        return self._configurator.get_sql_conn_string()
+        return self._configurator.get_sql_conn_string(origin)
 
-    def _get_engine(self, schema: str | None = None) -> Engine:
+    def _get_engine(self, origin: str, schema: str | None = None) -> Engine:
         """
         Create engine from connection string.
 
         Parameters
         ----------
+        origin : str
+            The origin of the credentials.
         schema : str
             The schema.
 
@@ -309,7 +317,7 @@ class SqlStore(Store):
         Engine
             An SQLAlchemy engine.
         """
-        connection_string = self._get_connection_string()
+        connection_string = self._get_connection_string(origin)
         if not isinstance(connection_string, str):
             raise StoreError("Connection string must be a string.")
         try:
@@ -334,8 +342,12 @@ class SqlStore(Store):
         Engine
             The database engine.
         """
-        engine = self._get_engine(schema)
-        self._check_access_to_storage(engine)
+        try:
+            engine = self._get_engine(CredsOrigin.ENV.value, schema)
+            self._check_access_to_storage(engine)
+        except StoreError:
+            engine = self._get_engine(CredsOrigin.FILE.value, schema)
+            self._check_access_to_storage(engine)
         return engine
 
     @staticmethod

digitalhub/{readers → stores/readers}/data/_base/builder.py

@@ -6,7 +6,7 @@ from abc import abstractmethod
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers.data._base.reader import DataframeReader
+    from digitalhub.stores.readers.data._base.reader import DataframeReader
 
 
 class ReaderBuilder:

digitalhub/{readers → stores/readers}/data/api.py

@@ -3,11 +3,11 @@ from __future__ import annotations
 import typing
 from typing import Any
 
-from digitalhub.readers.data.factory import factory
+from digitalhub.stores.readers.data.factory import factory
 from digitalhub.utils.exceptions import ReaderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers.data._base.reader import DataframeReader
+    from digitalhub.stores.readers.data._base.reader import DataframeReader
 
 
 def get_reader_by_engine(engine: str | None = None) -> DataframeReader:

digitalhub/{readers → stores/readers}/data/factory.py

@@ -5,8 +5,8 @@ import typing
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers.data._base.builder import ReaderBuilder
-    from digitalhub.readers.data._base.reader import DataframeReader
+    from digitalhub.stores.readers.data._base.builder import ReaderBuilder
+    from digitalhub.stores.readers.data._base.reader import DataframeReader
 
 
 class ReaderFactory:
@@ -126,7 +126,7 @@ class ReaderFactory:
 factory = ReaderFactory()
 
 try:
-    from digitalhub.readers.data.pandas.builder import ReaderBuilderPandas
+    from digitalhub.stores.readers.data.pandas.builder import ReaderBuilderPandas
 
     factory.add_builder(
         ReaderBuilderPandas.ENGINE,

digitalhub/{readers → stores/readers}/data/pandas/builder.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from digitalhub.readers.data._base.builder import ReaderBuilder
-from digitalhub.readers.data.pandas.reader import DataframeReaderPandas
+from digitalhub.stores.readers.data._base.builder import ReaderBuilder
+from digitalhub.stores.readers.data.pandas.reader import DataframeReaderPandas
 
 
 class ReaderBuilderPandas(ReaderBuilder):

digitalhub/{readers → stores/readers}/data/pandas/reader.py

@@ -9,7 +9,7 @@ import pandas as pd
 from pandas.errors import ParserError
 
 from digitalhub.entities.dataitem.table.utils import check_preview_size, finalize_preview, prepare_data, prepare_preview
-from digitalhub.readers.data._base.reader import DataframeReader
+from digitalhub.stores.readers.data._base.reader import DataframeReader
 from digitalhub.utils.enums import FileExtensions
 from digitalhub.utils.exceptions import ReaderError
 from digitalhub.utils.generic_utils import CustomJsonEncoder

{digitalhub-0.10.0b7.dist-info → digitalhub-0.10.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.10.0b7
+Version: 0.10.2
 Summary: Python SDK for Digitalhub
 Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
@@ -226,14 +226,14 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Requires-Python: <3.13,>=3.9
 Requires-Dist: boto3
 Requires-Dist: gitpython>=3
-Requires-Dist: numpy<2
+Requires-Dist: numpy
 Requires-Dist: psycopg2-binary
 Requires-Dist: pyarrow
 Requires-Dist: pydantic
-Requires-Dist: python-dotenv
 Requires-Dist: python-slugify
 Requires-Dist: pyyaml
 Requires-Dist: requests
@@ -249,7 +249,6 @@ Requires-Dist: pandas; extra == 'full'
 Provides-Extra: mlflow
 Requires-Dist: mlflow; extra == 'mlflow'
 Provides-Extra: pandas
-Requires-Dist: fsspec; extra == 'pandas'
 Requires-Dist: pandas; extra == 'pandas'
 Description-Content-Type: text/markdown