digitalhub 0.9.1__py3-none-any.whl → 0.10.0__py3-none-any.whl

digitalhub/stores/sql/store.py

@@ -1,39 +1,23 @@
 from __future__ import annotations
 
+import typing
 from pathlib import Path
 from typing import Any
 
 import pyarrow as pa
 import pyarrow.parquet as pq
-from sqlalchemy import MetaData, Table, create_engine
+from sqlalchemy import MetaData, Table, create_engine, select
 from sqlalchemy.engine import Engine
-from sqlalchemy.engine.row import LegacyRow
 from sqlalchemy.exc import SQLAlchemyError
 
-from digitalhub.readers.api import get_reader_by_object
-from digitalhub.stores._base.store import Store, StoreConfig
+from digitalhub.readers.data.api import get_reader_by_object
+from digitalhub.stores._base.store import Store
+from digitalhub.stores.sql.configurator import SqlStoreConfigurator
 from digitalhub.utils.exceptions import StoreError
+from digitalhub.utils.types import SourcesOrListOfSources
 
-
-class SQLStoreConfig(StoreConfig):
-    """
-    SQL store configuration class.
-    """
-
-    host: str
-    """SQL host."""
-
-    port: int
-    """SQL port."""
-
-    user: str
-    """SQL user."""
-
-    password: str
-    """SQL password."""
-
-    database: str
-    """SQL database name."""
+if typing.TYPE_CHECKING:
+    from sqlalchemy.engine.row import Row
 
 
 class SqlStore(Store):
@@ -42,9 +26,10 @@ class SqlStore(Store):
     artifacts on SQL based storage.
     """
 
-    def __init__(self, name: str, store_type: str, config: SQLStoreConfig) -> None:
-        super().__init__(name, store_type)
-        self.config = config
+    def __init__(self, config: dict | None = None) -> None:
+        super().__init__()
+        self._configurator = SqlStoreConfigurator()
+        self._configurator.configure(config)
 
     ##############################
     # I/O methods
@@ -101,7 +86,7 @@ class SqlStore(Store):
 
     def upload(
         self,
-        src: str | list[str],
+        src: SourcesOrListOfSources,
         dst: str,
     ) -> list[tuple[str, str]]:
         """
@@ -135,50 +120,75 @@ class SqlStore(Store):
         return []
 
     ##############################
-    # Private I/O methods
+    # Datastore methods
     ##############################
 
-    def _download_table(self, schema: str, table: str, dst: str) -> str:
+    def read_df(
+        self,
+        path: SourcesOrListOfSources,
+        file_format: str | None = None,
+        engine: str | None = None,
+        **kwargs,
+    ) -> Any:
         """
-        Download a table from SQL based storage.
+        Read DataFrame from path.
 
         Parameters
         ----------
-        schema : str
-            The origin schema.
-        table : str
-            The origin table.
-        dst : str
-            The destination path.
+        path : SourcesOrListOfSources
+            Path(s) to read DataFrame from.
+        file_format : str
+            Extension of the file.
+        engine : str
+            Dataframe engine (pandas, polars, etc.).
+        **kwargs : dict
+            Keyword arguments.
 
         Returns
         -------
-        str
-            The destination path.
+        Any
+            DataFrame.
         """
-        engine = self._check_factory(schema=schema)
+        if isinstance(path, list):
+            raise StoreError("SQL store can only read a single DataFrame at a time.")
 
-        # Read the table from the database
-        sa_table = Table(table, MetaData(), autoload_with=engine)
-        query = sa_table.select()
-        with engine.begin() as conn:
-            result: list[LegacyRow] = conn.execute(query).fetchall()
+        reader = self._get_reader(engine)
+        schema = self._get_schema(path)
+        table = self._get_table_name(path)
+        sql_engine = self._check_factory(schema=schema)
 
-        # Parse the result
-        data = self._parse_result(result)
+        sa_table = Table(table, MetaData(), autoload_with=sql_engine)
+        stm = select(sa_table)
 
-        # Convert the result to a pyarrow table and
-        # write the pyarrow table to a Parquet file
-        arrow_table = pa.Table.from_pydict(data)
-        pq.write_table(arrow_table, dst)
+        return reader.read_table(stm, sql_engine, **kwargs)
 
-        engine.dispose()
+    def query(
+        self,
+        query: str,
+        path: str,
+        engine: str | None = None,
+    ) -> Any:
+        """
+        Query data from database.
 
-        return dst
+        Parameters
+        ----------
+        query : str
+            The query to execute.
+        path : str
+            Path to the database.
+        engine : str
+            Dataframe engine (pandas, polars, etc.).
 
-    ##############################
-    # Datastore methods
-    ##############################
+        Returns
+        -------
+        Any
+            DataFrame.
+        """
+        reader = self._get_reader(engine)
+        schema = self._get_schema(path)
+        sql_engine = self._check_factory(schema=schema)
+        return reader.read_table(query, sql_engine)
 
     def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
         """
@@ -203,9 +213,47 @@ class SqlStore(Store):
         return self._upload_table(df, schema, table, **kwargs)
 
     ##############################
-    # Private Datastore methods
+    # Private I/O methods
     ##############################
 
+    def _download_table(self, schema: str, table: str, dst: str) -> str:
+        """
+        Download a table from SQL based storage.
+
+        Parameters
+        ----------
+        schema : str
+            The origin schema.
+        table : str
+            The origin table.
+        dst : str
+            The destination path.
+
+        Returns
+        -------
+        str
+            The destination path.
+        """
+        engine = self._check_factory(schema=schema)
+
+        # Read the table from the database
+        sa_table = Table(table, MetaData(), autoload_with=engine)
+        stm = select(sa_table)
+        with engine.begin() as conn:
+            result: list[Row] = conn.execute(stm).fetchall()
+
+        # Parse the result
+        data = {col: [row[idx] for row in result] for idx, col in enumerate(sa_table.columns.keys())}
+
+        # Convert the result to a pyarrow table and
+        # write the pyarrow table to a Parquet file
+        arrow_table = pa.Table.from_pydict(data)
+        pq.write_table(arrow_table, dst)
+
+        engine.dispose()
+
+        return dst
+
     def _upload_table(self, df: Any, schema: str, table: str, **kwargs) -> str:
         """
         Upload a table to SQL based storage.
@@ -245,10 +293,7 @@ class SqlStore(Store):
         str
             The connection string.
         """
-        return (
-            f"postgresql://{self.config.user}:{self.config.password}@"
-            f"{self.config.host}:{self.config.port}/{self.config.database}"
-        )
+        return self._configurator.get_sql_conn_string()
 
     def _get_engine(self, schema: str | None = None) -> Engine:
         """
@@ -377,27 +422,3 @@ class SqlStore(Store):
         except SQLAlchemyError:
             engine.dispose()
             raise StoreError("No access to db!")
-
-    @staticmethod
-    def _parse_result(result: list[LegacyRow]) -> dict:
-        """
-        Convert a list of list of tuples to a dict.
-
-        Parameters
-        ----------
-        result : list[LegacyRow]
-            The data to convert.
-
-        Returns
-        -------
-        dict
-            The converted data.
-        """
-        data_list = [row.items() for row in result]
-        data = {}
-        for row in data_list:
-            for column_name, value in row:
-                if column_name not in data:
-                    data[column_name] = []
-                data[column_name].append(value)
-        return data

digitalhub/{readers/_commons → utils}/enums.py

@@ -3,11 +3,15 @@ from __future__ import annotations
 from enum import Enum
 
 
-class Extensions(Enum):
+class FileExtensions(Enum):
     """
     Supported file extensions.
     """
 
     CSV = "csv"
     PARQUET = "parquet"
+    JSON = "json"
+    EXCEL_OLD = "xls"
+    EXCEL = "xlsx"
+    TXT = "txt"
     FILE = "file"

digitalhub/utils/exceptions.py

@@ -71,3 +71,9 @@ class ReaderError(Exception):
     """
     Raised when incontered errors on readers.
     """
+
+
+class ClientError(Exception):
+    """
+    Raised when incontered errors on clients.
+    """

digitalhub/utils/file_utils.py

@@ -4,6 +4,7 @@ from datetime import datetime
 from hashlib import sha256
 from mimetypes import guess_type
 from pathlib import Path
+from typing import Optional
 
 from pydantic import BaseModel
 
@@ -13,12 +14,12 @@ class FileInfo(BaseModel):
     File info class.
     """
 
-    path: str = None
-    name: str = None
-    content_type: str = None
-    size: int = None
-    hash: str = None
-    last_modified: str = None
+    path: Optional[str] = None
+    name: Optional[str] = None
+    content_type: Optional[str] = None
+    size: Optional[int] = None
+    hash: Optional[str] = None
+    last_modified: Optional[str] = None
 
     def to_dict(self):
         return self.model_dump()
@@ -60,7 +61,7 @@ def get_file_size(data_path: str) -> int:
     return Path(data_path).stat().st_size
 
 
-def get_file_mime_type(data_path: str) -> str:
+def get_file_mime_type(data_path: str) -> str | None:
     """
     Get the mime type of a file.
 

digitalhub/utils/generic_utils.py

@@ -3,9 +3,10 @@ from __future__ import annotations
 import base64
 import importlib.util as imputil
 import json
-from datetime import datetime
-from enum import Enum
+from datetime import date, datetime, time
+from enum import Enum, EnumMeta
 from pathlib import Path
+from types import MappingProxyType
 from typing import Any, Callable
 from zipfile import ZipFile
 
@@ -120,14 +121,14 @@ def extract_archive(path: Path, filename: Path) -> None:
         zip_file.extractall(path)
 
 
-class MyEncoder(json.JSONEncoder):
+class CustomJsonEncoder(json.JSONEncoder):
     """
-    Custom JSON encoder to handle numpy types.
+    Custom JSON encoder to handle json dumps.
     """
 
     def default(self, obj: Any) -> Any:
         """
-        Convert numpy types to json.
+        Convert an object to json.
 
         Parameters
         ----------
@@ -147,11 +148,12 @@ class MyEncoder(json.JSONEncoder):
             return float(obj)
         elif isinstance(obj, np.ndarray):
             return obj.tolist()
-        else:
-            return str(obj)
+        elif isinstance(obj, (datetime, date, time)):
+            return obj.isoformat()
+        return str(obj)
 
 
-def dict_to_json(struct: dict) -> str:
+def dump_json(struct: Any) -> str:
     """
     Convert a dict to json.
 
@@ -165,7 +167,7 @@ def dict_to_json(struct: dict) -> str:
     str
         The json string.
     """
-    return json.dumps(struct, cls=MyEncoder)
+    return json.dumps(struct, cls=CustomJsonEncoder)
 
 
 def slugify_string(filename: str) -> str:
@@ -202,23 +204,34 @@ def import_function(path: Path, handler: str) -> Callable:
         Function.
     """
     spec = imputil.spec_from_file_location(path.stem, path)
+    if spec is None:
+        raise RuntimeError(f"Error loading function source from {str(path)}.")
+
     mod = imputil.module_from_spec(spec)
+    if spec.loader is None:
+        raise RuntimeError(f"Error getting module loader from {str(path)}.")
+
     spec.loader.exec_module(mod)
-    return getattr(mod, handler)
+    func = getattr(mod, handler)
+    if not callable(func):
+        raise RuntimeError(f"Handler '{handler}' is not a callable.")
+
+    return func
 
 
-def list_enum(enum: Enum) -> list:
+def list_enum(enum: EnumMeta) -> list[Any]:
     """
     Get all values of an enum.
 
     Parameters
     ----------
-    enum : Enum
-        Enum to get values from.
+    enum : EnumMeta
+        Enum class to get values from.
 
     Returns
     -------
-    list
+    list[Any]
         List of enum values.
     """
-    return [e.value for e in enum]
+    vals: MappingProxyType[str, Enum] = enum.__members__
+    return [member.value for member in vals.values()]

digitalhub/utils/git_utils.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import shutil
 import warnings
+from enum import Enum
 from pathlib import Path
 from urllib.parse import urlparse
 
@@ -13,6 +14,16 @@ except ImportError as e:
         warnings.warn("git is not installed. Please install git and try again.", RuntimeWarning)
 
 
+class GitCredentialsType(Enum):
+    """
+    Supported git credentials types.
+    """
+
+    USERNAME = "GIT_USERNAME"
+    PASSWORD = "GIT_PASSWORD"
+    TOKEN = "GIT_TOKEN"
+
+
 def clone_repository(path: Path, url: str) -> None:
     """
     Clone git repository.
@@ -90,12 +101,8 @@ def get_git_username_password_from_token(token: str) -> tuple[str, str]:
     """
     # Mutued from mlrun
     if token.startswith("github_pat_") or token.startswith("glpat"):
-        username = "oauth2"
-        password = token
-    else:
-        username = token
-        password = "x-oauth-basic"
-    return username, password
+        return "oauth2", token
+    return token, "x-oauth-basic"
 
 
 def add_credentials_git_remote_url(url: str) -> str:
@@ -115,9 +122,9 @@ def add_credentials_git_remote_url(url: str) -> str:
     url_obj = urlparse(url)
 
     # Get credentials from environment variables
-    username = os.getenv("GIT_USERNAME")
-    password = os.getenv("GIT_PASSWORD")
-    token = os.getenv("GIT_TOKEN")
+    username = os.getenv(GitCredentialsType.USERNAME.value)
+    password = os.getenv(GitCredentialsType.PASSWORD.value)
+    token = os.getenv(GitCredentialsType.TOKEN.value)
 
     # Get credentials from token. Override username and password
     if token is not None:

digitalhub/utils/types.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from typing import Union
+
+SourcesOrListOfSources = Union[str, list[str]]

digitalhub/utils/uri_utils.py

@@ -93,13 +93,13 @@ def map_uri_scheme(uri: str) -> str:
     Raises
     ------
     ValueError
-        If the scheme is unknown.
+        If the scheme is unknown or invalid.
     """
     scheme = urlparse(uri).scheme
     if scheme in list_enum(LocalSchemes):
         return SchemeCategory.LOCAL.value
     if scheme in list_enum(InvalidLocalSchemes):
-        raise ValueError("For local uri, do not use any scheme.")
+        raise ValueError("For local URI, do not use any scheme.")
     if scheme in list_enum(RemoteSchemes):
         return SchemeCategory.REMOTE.value
     if scheme in list_enum(S3Schemes):

{digitalhub-0.9.1.dist-info → digitalhub-0.10.0.dist-info}/METADATA

@@ -1,7 +1,8 @@
-Metadata-Version: 2.2
+Metadata-Version: 2.4
 Name: digitalhub
-Version: 0.9.1
+Version: 0.10.0
 Summary: Python SDK for Digitalhub
+Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
                                 Version 2.0, January 2004
@@ -219,48 +220,41 @@ License:                               Apache License
         
         (SIL OPEN FONT LICENSE Version 1.1) The Inter font family (https://github.com/rsms/inter)
         (SIL OPEN FONT LICENSE Version 1.1) The Fira Code font family (https://github.com/tonsky/FiraCode)
-        
-Project-URL: Homepage, https://github.com/scc-digitalhub/digitalhub-sdk
+License-File: LICENSE.txt
 Keywords: data,dataops,kubernetes
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-License-File: LICENSE.txt
+Requires-Python: <3.13,>=3.9
 Requires-Dist: boto3
-Requires-Dist: pydantic
-Requires-Dist: sqlalchemy<2
-Requires-Dist: pyarrow
+Requires-Dist: gitpython>=3
 Requires-Dist: numpy<2
-Requires-Dist: requests
-Requires-Dist: PyYAML
-Requires-Dist: python-dotenv
-Requires-Dist: GitPython>=3
 Requires-Dist: psycopg2-binary
+Requires-Dist: pyarrow
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv
 Requires-Dist: python-slugify
+Requires-Dist: pyyaml
+Requires-Dist: requests
+Requires-Dist: sqlalchemy
+Provides-Extra: dev
+Requires-Dist: bumpver; extra == 'dev'
+Requires-Dist: jsonschema; extra == 'dev'
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-cov; extra == 'dev'
 Provides-Extra: full
-Requires-Dist: pandas<2.2,>=1.2; extra == "full"
-Requires-Dist: mlflow; extra == "full"
-Provides-Extra: pandas
-Requires-Dist: pandas<2.2,>=1.2; extra == "pandas"
+Requires-Dist: mlflow; extra == 'full'
+Requires-Dist: pandas; extra == 'full'
 Provides-Extra: mlflow
-Requires-Dist: mlflow; extra == "mlflow"
-Provides-Extra: dev
-Requires-Dist: black; extra == "dev"
-Requires-Dist: pytest; extra == "dev"
-Requires-Dist: bumpver; extra == "dev"
-Requires-Dist: ruff; extra == "dev"
-Requires-Dist: moto; extra == "dev"
-Provides-Extra: docs
-Requires-Dist: Sphinx>=7; extra == "docs"
-Requires-Dist: pydata-sphinx-theme>=0.15; extra == "docs"
-Requires-Dist: numpydoc>=1.6; extra == "docs"
+Requires-Dist: mlflow; extra == 'mlflow'
+Provides-Extra: pandas
+Requires-Dist: fsspec; extra == 'pandas'
+Requires-Dist: pandas; extra == 'pandas'
+Description-Content-Type: text/markdown
 
 # Digitalhub Library
 
-The Digitalhub SDK library is used to manage entities and executions in Digitalhub from Python.
-It comes with a suite of tools to help you manage your projects and executions. It exposes CRUD methods to create, read, update and delete entities, and objects methods to excute functions or workflows, collect or store execution results and data.
+The Digitalhub library is a python tool for managing projects, entities and executions in Digitalhub. It exposes CRUD methods to create, read, update and delete entities, tools to execute functions or workflows, collect or store execution results and data.
 
 A more detailed description of the library can be found in the [official documentation](https://scc-digitalhub.github.io/sdk-docs/).