digitalhub 0.10.2__py3-none-any.whl → 0.11.0__py3-none-any.whl

digitalhub/stores/configurator/configurator.py

@@ -3,7 +3,8 @@ from __future__ import annotations
 import os
 
 from digitalhub.stores.configurator.credentials_store import CredentialsStore
-from digitalhub.stores.configurator.ini_module import load_from_file, write_config
+from digitalhub.stores.configurator.enums import SetCreds
+from digitalhub.stores.configurator.ini_module import load_from_file, read_env_from_file, set_current_env, write_config
 
 
 class EnvConfigurator:
@@ -16,7 +17,7 @@ class EnvConfigurator:
         self._creds_store = CredentialsStore()
 
         # Current credentials set (__default by default)
-        self._environment = "__default"
+        self._environment = os.getenv(SetCreds.DH_ENV.value, SetCreds.DEFAULT.value)
 
     ##############################
     # Public methods
@@ -36,6 +37,7 @@ class EnvConfigurator:
         None
         """
         self._environment = creds_set
+        set_current_env(creds_set)
 
     def get_current_env(self) -> str:
         """
@@ -105,6 +107,7 @@ class EnvConfigurator:
         str | None
             Environment variable value.
         """
+        self._environment = read_env_from_file()
         return load_from_file(var)
 
     def write_env(self, key_to_include: list[str] | None = None) -> None:

digitalhub/stores/configurator/enums.py

@@ -10,3 +10,12 @@ class CredsOrigin(Enum):
 
     ENV = "env"
     FILE = "file"
+
+
+class SetCreds(Enum):
+    """
+    List supported environments.
+    """
+
+    DEFAULT = "__default"
+    DH_ENV = "DH_NAME"

digitalhub/stores/configurator/ini_module.py

@@ -13,6 +13,23 @@ from digitalhub.utils.exceptions import ClientError
 ENV_FILE = Path.home() / ".dhcore.ini"
 
 
+def load_file() -> ConfigParser:
+    """
+    Load current credentials set from the .dhcore.ini file.
+
+    Returns
+    -------
+    ConfigParser
+        Credentials set name.
+    """
+    try:
+        file = ConfigParser()
+        file.read(ENV_FILE)
+        return file
+    except Exception as e:
+        raise ClientError(f"Failed to read env file: {e}")
+
+
 def load_from_file(var: str) -> str | None:
     """
     Load variable from config file.
@@ -29,9 +46,8 @@ def load_from_file(var: str) -> str | None:
     str | None
         Environment variable value.
     """
-    cfg = ConfigParser()
-    cfg.read(ENV_FILE)
     try:
+        cfg = load_file()
         profile = cfg["DEFAULT"]["current_environment"]
         return cfg[profile].get(var)
     except KeyError:
@@ -55,8 +71,7 @@ def write_config(creds: dict, environment: str) -> None:
     None
     """
     try:
-        cfg = ConfigParser()
-        cfg.read(ENV_FILE)
+        cfg = load_file()
 
         sections = cfg.sections()
         if environment not in sections:
@@ -72,3 +87,42 @@ def write_config(creds: dict, environment: str) -> None:
 
     except Exception as e:
         raise ClientError(f"Failed to write env file: {e}")
+
+
+def set_current_env(environment: str) -> None:
+    """
+    Set the current credentials set.
+
+    Parameters
+    ----------
+    environment : str
+        Credentials set name.
+
+    Returns
+    -------
+    None
+    """
+    try:
+        cfg = load_file()
+        cfg["DEFAULT"]["current_environment"] = environment
+        with open(ENV_FILE, "w") as inifile:
+            cfg.write(inifile)
+
+    except Exception as e:
+        raise ClientError(f"Failed to write env file: {e}")
+
+
+def read_env_from_file() -> str | None:
+    """
+    Read the current credentials set from the .dhcore.ini file.
+
+    Returns
+    -------
+    str
+        Credentials set name.
+    """
+    try:
+        cfg = load_file()
+        return cfg["DEFAULT"]["current_environment"]
+    except Exception:
+        return None

digitalhub/stores/data/api.py

@@ -8,7 +8,7 @@ if typing.TYPE_CHECKING:
     from digitalhub.stores.data._base.store import Store
 
 
-def get_store(project: str, uri: str, config: dict | None = None) -> Store:
+def get_store(project: str, uri: str) -> Store:
     """
     Get store instance by URI.
 
@@ -26,4 +26,4 @@ def get_store(project: str, uri: str, config: dict | None = None) -> Store:
     Store
         Store instance.
     """
-    return store_builder.get(uri, config)
+    return store_builder.get(project, uri)

digitalhub/stores/data/builder.py

@@ -44,9 +44,9 @@ class StoreBuilder:
     """
 
     def __init__(self) -> None:
-        self._instances: dict[str, Store] = {}
+        self._instances: dict[str, dict[str, Store]] = {}
 
-    def build(self, store_type: str, config: dict | None = None) -> None:
+    def build(self, project: str, store_type: str) -> None:
         """
         Build a store instance and register it.
 
@@ -55,7 +55,6 @@ class StoreBuilder:
         store_type : str
             Store type.
         config : dict
-            Store configuration.
 
         Returns
         -------
@@ -64,9 +63,9 @@ class StoreBuilder:
         env = get_current_env()
         if env not in self._instances:
             self._instances[env] = {}
-        self._instances[env][store_type] = _get_class_from_type(store_type)(config)
+        self._instances[env][store_type] = _get_class_from_type(store_type)()
 
-    def get(self, uri: str, config: dict | None = None) -> Store:
+    def get(self, project: str, uri: str) -> Store:
         """
         Get a store instance by URI.
 
@@ -87,7 +86,7 @@ class StoreBuilder:
         try:
             return self._instances[env][store_type]
         except KeyError:
-            self.build(store_type, config)
+            self.build(project, store_type)
             return self._instances[env][store_type]
 
 

digitalhub/stores/data/enums.py

@@ -0,0 +1,11 @@
+from __future__ import annotations
+
+from enum import Enum
+
+
+class StoreEnv(Enum):
+    """
+    Environment variables for data stores.
+    """
+
+    DEFAULT_FILES_STORE = "DHCORE_DEFAULT_FILES_STORE"

digitalhub/stores/data/local/store.py

@@ -17,9 +17,6 @@ class LocalStore(Store):
     artifacts on local filesystem based storage.
     """
 
-    def __init__(self, config: dict | None = None) -> None:
-        super().__init__()
-
     ##############################
     # I/O methods
     ##############################

digitalhub/stores/data/remote/store.py

@@ -16,9 +16,6 @@ class RemoteStore(Store):
     artifacts from remote HTTP based storage.
     """
 
-    def __init__(self, config: dict | None = None) -> None:
-        super().__init__()
-
     ##############################
     # I/O methods
     ##############################

digitalhub/stores/data/s3/configurator.py

@@ -18,7 +18,6 @@ class S3StoreConfigurator:
         S3StoreEnv.ENDPOINT_URL,
         S3StoreEnv.ACCESS_KEY_ID,
         S3StoreEnv.SECRET_ACCESS_KEY,
-        S3StoreEnv.BUCKET_NAME,
     ]
     optional_vars = [
         S3StoreEnv.REGION,
@@ -26,29 +25,10 @@ class S3StoreConfigurator:
         S3StoreEnv.SESSION_TOKEN,
     ]
 
-    def __init__(self, config: dict | None = None) -> None:
-        self.configure(config)
-
     ##############################
     # Configuration methods
     ##############################
 
-    def configure(self, config: dict | None = None) -> None:
-        """
-        Configure the store by getting the credentials from user
-        provided config or from environment.
-
-        Parameters
-        ----------
-        config : dict
-            Configuration dictionary.
-
-        Returns
-        -------
-        None
-        """
-        self._get_env_config()
-
     def get_boto3_client_config(self, origin: str) -> dict:
         """
         Get S3 credentials (access key, secret key,

digitalhub/stores/data/s3/enums.py

@@ -8,10 +8,9 @@ class S3StoreEnv(Enum):
     S3Store environment
     """
 
-    ENDPOINT_URL = "S3_ENDPOINT_URL"
+    ENDPOINT_URL = "AWS_ENDPOINT_URL"
     ACCESS_KEY_ID = "AWS_ACCESS_KEY_ID"
     SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
     SESSION_TOKEN = "AWS_SESSION_TOKEN"
-    BUCKET_NAME = "S3_BUCKET"
-    REGION = "S3_REGION"
+    REGION = "AWS_REGION"
     SIGNATURE_VERSION = "S3_SIGNATURE_VERSION"

digitalhub/stores/data/s3/store.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import typing
 from io import BytesIO
 from pathlib import Path
 from typing import Any, Type
@@ -8,7 +7,7 @@ from urllib.parse import urlparse
 
 import boto3
 import botocore.client  # pylint: disable=unused-import
-from botocore.exceptions import ClientError
+from botocore.exceptions import ClientError, NoCredentialsError
 
 from digitalhub.stores.configurator.enums import CredsOrigin
 from digitalhub.stores.data._base.store import Store
@@ -19,9 +18,6 @@ from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
 from digitalhub.utils.types import SourcesOrListOfSources
 
-if typing.TYPE_CHECKING:
-    pass
-
 # Type aliases
 S3Client = Type["botocore.client.S3"]
 
@@ -32,10 +28,8 @@ class S3Store(Store):
     artifacts on S3 based storage.
     """
 
-    def __init__(self, config: dict | None = None) -> None:
-        super().__init__()
+    def __init__(self) -> None:
         self._configurator = S3StoreConfigurator()
-        self._configurator.configure(config)
 
     ##############################
     # I/O methods
@@ -655,8 +649,8 @@ class S3Store(Store):
         """
         try:
             client.head_bucket(Bucket=bucket)
-        except ClientError:
-            raise StoreError("No access to s3 bucket!")
+        except (ClientError, NoCredentialsError) as err:
+            raise StoreError(f"No access to s3 bucket! Error: {err}")
 
     @staticmethod
     def _get_key(path: str) -> str:

digitalhub/stores/data/s3/utils.py

@@ -1,14 +1,13 @@
 from __future__ import annotations
 
-import os
 from pathlib import Path
 from urllib.parse import urlparse
 
 from boto3 import client as boto3_client
 
-from digitalhub.stores.data.s3.enums import S3StoreEnv
-
-DEFAULT_BUCKET = "datalake"
+from digitalhub.stores.configurator.enums import CredsOrigin
+from digitalhub.stores.data.s3.configurator import S3StoreConfigurator
+from digitalhub.utils.exceptions import StoreError
 
 
 def get_bucket_name(path: str) -> str:
@@ -63,17 +62,13 @@ def get_s3_source(bucket: str, key: str, filename: Path) -> None:
     -------
     None
     """
-    s3 = boto3_client("s3", endpoint_url=os.getenv(S3StoreEnv.ENDPOINT_URL.value))
-    s3.download_file(bucket, key, filename)
-
-
-def get_s3_bucket_from_env() -> str | None:
-    """
-    Function to get S3 bucket name.
-
-    Returns
-    -------
-    str
-        The S3 bucket name.
-    """
-    return os.getenv(S3StoreEnv.BUCKET_NAME.value, DEFAULT_BUCKET)
+    # Try to get client from environment variables
+    try:
+        cfg = S3StoreConfigurator().get_boto3_client_config(CredsOrigin.ENV.value)
+        s3 = boto3_client("s3", **cfg)
+        s3.download_file(bucket, key, filename)
+
+    # Fallback to file
+    except StoreError:
+        cfg = S3StoreConfigurator().get_boto3_client_config(CredsOrigin.FILE.value)
+        s3.download_file(bucket, key, filename)

digitalhub/stores/data/sql/configurator.py

@@ -12,31 +12,18 @@ class SqlStoreConfigurator:
     provided config or from environment.
     """
 
-    variables = [SqlStoreEnv.USERNAME, SqlStoreEnv.PASSWORD, SqlStoreEnv.HOST, SqlStoreEnv.PORT, SqlStoreEnv.DATABASE]
-
-    def __init__(self, config: dict | None = None) -> None:
-        self.configure(config)
+    required_vars = [
+        SqlStoreEnv.USERNAME,
+        SqlStoreEnv.PASSWORD,
+        SqlStoreEnv.HOST,
+        SqlStoreEnv.PORT,
+        SqlStoreEnv.DATABASE,
+    ]
 
     ##############################
     # Configuration methods
     ##############################
 
-    def configure(self, config: dict | None = None) -> None:
-        """
-        Configure the store by getting the credentials from user
-        provided config or from environment.
-
-        Parameters
-        ----------
-        config : dict
-            Configuration dictionary.
-
-        Returns
-        -------
-        None
-        """
-        self._get_env_config()
-
     def get_sql_conn_string(self, origin: str) -> str:
         """
         Get the connection string from environment variables.
@@ -74,7 +61,7 @@ class SqlStoreConfigurator:
         dict
             The credentials.
         """
-        credentials = {var.value: configurator.load_from_env(var.value) for var in self.variables}
+        credentials = {var.value: configurator.load_from_env(var.value) for var in self.required_vars}
         self._set_credentials(credentials)
         return credentials
 
@@ -87,7 +74,7 @@ class SqlStoreConfigurator:
         dict
             The credentials.
         """
-        credentials = {var.value: configurator.load_from_file(var.value) for var in self.variables}
+        credentials = {var.value: configurator.load_from_file(var.value) for var in self.required_vars}
         self._set_credentials(credentials)
         return credentials
 

digitalhub/stores/data/sql/store.py

@@ -27,10 +27,8 @@ class SqlStore(Store):
     artifacts on SQL based storage.
     """
 
-    def __init__(self, config: dict | None = None) -> None:
-        super().__init__()
+    def __init__(self) -> None:
         self._configurator = SqlStoreConfigurator()
-        self._configurator.configure(config)
 
     ##############################
     # I/O methods

digitalhub/stores/data/utils.py

@@ -0,0 +1,34 @@
+from __future__ import annotations
+
+from digitalhub.context.api import get_context
+from digitalhub.stores.configurator.configurator import configurator
+from digitalhub.stores.data.enums import StoreEnv
+
+
+def get_default_store(project: str) -> str:
+    """
+    Get default store URI.
+
+    Parameters
+    ----------
+    project : str
+        Project name.
+
+    Returns
+    -------
+    str
+        Default store URI.
+    """
+    context = get_context(project)
+    store = context.config.get(StoreEnv.DEFAULT_FILES_STORE.value.lower())
+    if store is not None:
+        return store
+    store = configurator.load_var(StoreEnv.DEFAULT_FILES_STORE.value)
+    if store is None or store == "":
+        raise ValueError(
+            "No default store found. "
+            "Please set a default store "
+            f"in your environment (e.g. export {StoreEnv.DEFAULT_FILES_STORE.value}=) "
+            "or set it in project config."
+        )
+    return store

digitalhub/utils/file_utils.py

@@ -190,7 +190,7 @@ def get_file_info_from_s3(path: str, metadata: dict) -> None | dict:
         if size < file_size_limit_multipart:
             file_hash = "md5:" + file_hash
         else:
-            file_hash = "LiteralETag:" + file_hash
+            file_hash = "ETag:" + file_hash
 
         name = get_path_name(path)
         content_type = metadata["ContentType"]

digitalhub/utils/generic_utils.py

@@ -8,6 +8,7 @@ from enum import Enum, EnumMeta
 from pathlib import Path
 from types import MappingProxyType
 from typing import Any, Callable
+from warnings import warn
 from zipfile import ZipFile
 
 import numpy as np
@@ -235,3 +236,39 @@ def list_enum(enum: EnumMeta) -> list[Any]:
     """
     vals: MappingProxyType[str, Enum] = enum.__members__
     return [member.value for member in vals.values()]
+
+
+def carriage_return_warn(string: str) -> None:
+    """
+    Print a warning message if string contains a carriage return (\r\n).
+
+    Parameters
+    ----------
+    string : str
+        The string to check.
+
+    Returns
+    -------
+    None
+    """
+    if "\r\n" in string:
+        warn("String contains a carriage return. It may not be parsed correctly from remote runtimes.")
+
+
+def read_source(path: str) -> str:
+    """
+    Read a file.
+
+    Parameters
+    ----------
+    path : Path
+        Path to the file.
+
+    Returns
+    -------
+    str
+        File content.
+    """
+    text = read_text(path)
+    carriage_return_warn(text)
+    return encode_string(text)

digitalhub/utils/uri_utils.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from enum import Enum
 from urllib.parse import unquote, urlparse
 
@@ -95,6 +96,10 @@ def map_uri_scheme(uri: str) -> str:
     ValueError
         If the scheme is unknown or invalid.
     """
+    # Check for Windows paths (e.g. C:\path\to\file or \\network\share)
+    if re.match(r"^[a-zA-Z]:\\", uri) or uri.startswith(r"\\"):
+        return SchemeCategory.LOCAL.value
+
     scheme = urlparse(uri).scheme
     if scheme in list_enum(LocalSchemes):
         return SchemeCategory.LOCAL.value

{digitalhub-0.10.2.dist-info → digitalhub-0.11.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.10.2
+Version: 0.11.0
 Summary: Python SDK for Digitalhub
 Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>