digitalhub 0.8.1__py3-none-any.whl → 0.9.0b0__py3-none-any.whl

digitalhub/stores/s3/store.py

@@ -2,13 +2,14 @@ from __future__ import annotations
 
 from io import BytesIO
 from pathlib import Path
-from typing import Type
+from typing import Any, Type
 from urllib.parse import urlparse
 
 import boto3
 import botocore.client  # pylint: disable=unused-import
 from botocore.exceptions import ClientError
 
+from digitalhub.readers.api import get_reader_by_object
 from digitalhub.stores._base.store import Store, StoreConfig
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.file_utils import get_file_info_from_s3, get_file_mime_type
@@ -46,7 +47,7 @@ class S3Store(Store):
         self.config = config
 
     ##############################
-    # IO methods
+    # I/O methods
     ##############################
 
     def download(
@@ -141,16 +142,13 @@ class S3Store(Store):
         list[tuple[str, str]]
             Returns the list of destination and source paths of the uploaded artifacts.
         """
-
         # Destination handling
 
         # If no destination is provided, build key from source
         # Otherwise build key from destination
         if dst is None:
             raise StoreError(
-                "Destination must be provided. " + "If source is a list of files or a directory, "
-                "destination must be a partition, e.g. 's3://bucket/partition/', ",
-                "otherwise a destination key, e.g. 's3://bucket/key'",
+                "Destination must be provided. If source is a list of files or a directory, destination must be a partition, e.g. 's3://bucket/partition/' otherwise a destination key, e.g. 's3://bucket/key'"
             )
         else:
             dst = self._get_key(dst)
@@ -408,7 +406,36 @@ class S3Store(Store):
         client.put_object(Bucket=bucket, Key=key, Body=fileobj.getvalue())
 
     ##############################
-    # Private helper methods
+    # Datastore methods
+    ##############################
+
+    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
+        """
+        Write a dataframe to S3 based storage. Kwargs are passed to df.to_parquet().
+
+        Parameters
+        ----------
+        df : Any
+            The dataframe.
+        dst : str
+            The destination path on S3 based storage.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        str
+            The S3 path where the dataframe was saved.
+        """
+        fileobj = BytesIO()
+        reader = get_reader_by_object(df)
+        reader.write_df(df, fileobj, extension=extension, **kwargs)
+
+        key = self._get_key(dst)
+        return self.upload_fileobject(fileobj, key)
+
+    ##############################
+    # Helper methods
     ##############################
 
     def _get_bucket(self) -> str:

digitalhub/stores/sql/store.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 from pathlib import Path
+from typing import Any
 
 import pyarrow as pa
 import pyarrow.parquet as pq
@@ -9,6 +10,7 @@ from sqlalchemy.engine import Engine
 from sqlalchemy.engine.row import LegacyRow
 from sqlalchemy.exc import SQLAlchemyError
 
+from digitalhub.readers.api import get_reader_by_object
 from digitalhub.stores._base.store import Store, StoreConfig
 from digitalhub.utils.exceptions import StoreError
 
@@ -45,7 +47,7 @@ class SqlStore(Store):
         self.config = config
 
     ##############################
-    # IO methods
+    # I/O methods
     ##############################
 
     def download(
@@ -112,15 +114,118 @@ class SqlStore(Store):
         """
         Get file information from SQL based storage.
 
-        Raises
-        ------
-        NotImplementedError
-            This method is not implemented.
+        Parameters
+        ----------
+        paths : list[str]
+            List of source paths.
+
+        Returns
+        -------
+        list[dict]
+            Returns files metadata.
         """
-        raise NotImplementedError("SQL store does not support upload.")
+        return []
 
     ##############################
-    # Private helper methods
+    # Private I/O methods
+    ##############################
+
+    def _download_table(self, schema: str, table: str, dst: str) -> str:
+        """
+        Download a table from SQL based storage.
+
+        Parameters
+        ----------
+        schema : str
+            The origin schema.
+        table : str
+            The origin table.
+        dst : str
+            The destination path.
+
+        Returns
+        -------
+        str
+            The destination path.
+        """
+        engine = self._check_factory(schema=schema)
+
+        # Read the table from the database
+        sa_table = Table(table, MetaData(), autoload_with=engine)
+        query = sa_table.select()
+        with engine.begin() as conn:
+            result: list[LegacyRow] = conn.execute(query).fetchall()
+
+        # Parse the result
+        data = self._parse_result(result)
+
+        # Convert the result to a pyarrow table and
+        # write the pyarrow table to a Parquet file
+        arrow_table = pa.Table.from_pydict(data)
+        pq.write_table(arrow_table, dst)
+
+        engine.dispose()
+
+        return dst
+
+    ##############################
+    # Datastore methods
+    ##############################
+
+    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
+        """
+        Write a dataframe to a database. Kwargs are passed to df.to_sql().
+
+        Parameters
+        ----------
+        df : Any
+            The dataframe to write.
+        dst : str
+            The destination of the dataframe.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        str
+            Path of written dataframe.
+        """
+        schema = self._get_schema(dst)
+        table = self._get_table_name(dst)
+        return self._upload_table(df, schema, table, **kwargs)
+
+    ##############################
+    # Private Datastore methods
+    ##############################
+
+    def _upload_table(self, df: Any, schema: str, table: str, **kwargs) -> str:
+        """
+        Upload a table to SQL based storage.
+
+        Parameters
+        ----------
+        df : DataFrame
+            The dataframe.
+        schema : str
+            Destination schema.
+        table : str
+            Destination table.
+        **kwargs : dict
+            Keyword arguments.
+
+        Returns
+        -------
+        str
+            The SQL URI where the dataframe was saved.
+        """
+        reader = get_reader_by_object(df)
+        engine = self._check_factory()
+        reader.write_table(df, table, engine, schema, **kwargs)
+        engine.dispose()
+        return f"sql://{engine.url.database}/{schema}/{table}"
+
+    ##############################
+    # Helper methods
     ##############################
 
     def _get_connection_string(self) -> str:
@@ -265,44 +370,6 @@ class SqlStore(Store):
             engine.dispose()
             raise StoreError("No access to db!")
 
-    def _download_table(self, schema: str, table: str, dst: str) -> str:
-        """
-        Download a table from SQL based storage.
-
-        Parameters
-        ----------
-        schema : str
-            The origin schema.
-        table : str
-            The origin table.
-        dst : str
-            The destination path.
-
-        Returns
-        -------
-        str
-            The destination path.
-        """
-        engine = self._check_factory(schema=schema)
-
-        # Read the table from the database
-        sa_table = Table(table, MetaData(), autoload_with=engine)
-        query = sa_table.select()
-        with engine.begin() as conn:
-            result: list[LegacyRow] = conn.execute(query).fetchall()
-
-        # Parse the result
-        data = self._parse_result(result)
-
-        # Convert the result to a pyarrow table and
-        # write the pyarrow table to a Parquet file
-        arrow_table = pa.Table.from_pydict(data)
-        pq.write_table(arrow_table, dst)
-
-        engine.dispose()
-
-        return dst
-
     @staticmethod
     def _parse_result(result: list[LegacyRow]) -> dict:
         """

digitalhub/utils/exceptions.py

@@ -65,3 +65,9 @@ class ContextError(Exception):
     """
     Raised when context errors.
     """
+
+
+class ReaderError(Exception):
+    """
+    Raised when incontered errors on readers.
+    """

digitalhub/utils/file_utils.py

@@ -20,6 +20,9 @@ class FileInfo(BaseModel):
     hash: str = None
     last_modified: str = None
 
+    def to_dict(self):
+        return self.model_dump()
+
 
 def calculate_blob_hash(data_path: str) -> str:
     """
@@ -157,7 +160,7 @@ def get_file_info_from_local(path: str, src_path: str) -> None | dict:
             size=size,
             hash=hash,
             last_modified=last_modified,
-        ).dict()
+        ).to_dict()
     except Exception:
         return None
 
@@ -199,6 +202,61 @@ def get_file_info_from_s3(path: str, metadata: dict) -> None | dict:
             size=size,
             hash=file_hash,
             last_modified=last_modified,
-        ).dict()
+        ).to_dict()
     except Exception:
         return None
+
+
+def eval_zip_type(source: str) -> bool:
+    """
+    Evaluate zip type.
+
+    Parameters
+    ----------
+    source : str
+        Source.
+
+    Returns
+    -------
+    bool
+        True if path is zip.
+    """
+    extension = source.endswith(".zip")
+    mime_zip = get_file_mime_type(source) == "application/zip"
+    return extension or mime_zip
+
+
+def eval_text_type(source: str) -> bool:
+    """
+    Evaluate text type.
+
+    Parameters
+    ----------
+    source : str
+        Source.
+
+    Returns
+    -------
+    bool
+        True if path is text.
+    """
+    return get_file_mime_type(source) == "text/plain"
+
+
+def eval_py_type(source: str) -> bool:
+    """
+    Evaluate python type.
+
+    Parameters
+    ----------
+    source : str
+        Source.
+
+    Returns
+    -------
+    bool
+        True if path is python.
+    """
+    extension = source.endswith(".py")
+    mime_py = get_file_mime_type(source) == "text/x-python"
+    return extension or mime_py

digitalhub/utils/generic_utils.py

@@ -1,14 +1,16 @@
 from __future__ import annotations
 
 import base64
+import importlib.util as imputil
 import json
 from datetime import datetime
+from enum import Enum
 from pathlib import Path
-from typing import Any
+from typing import Any, Callable
 from zipfile import ZipFile
 
 import numpy as np
-from requests import get as requests_get
+import requests
 from slugify import slugify
 
 from digitalhub.utils.io_utils import read_text
@@ -26,7 +28,7 @@ def get_timestamp() -> str:
     return datetime.now().astimezone().isoformat()
 
 
-def decode_string(string: str) -> str:
+def decode_base64_string(string: str) -> str:
     """
     Decode a string from base64.
 
@@ -92,7 +94,7 @@ def requests_chunk_download(source: str, filename: Path) -> None:
     -------
     None
     """
-    with requests_get(source, stream=True) as r:
+    with requests.get(source, stream=True) as r:
         r.raise_for_status()
         with filename.open("wb") as f:
             for chunk in r.iter_content(chunk_size=8192):
@@ -181,3 +183,42 @@ def slugify_string(filename: str) -> str:
         The sanitized filename.
     """
     return slugify(filename, max_length=255)
+
+
+def import_function(path: Path, handler: str) -> Callable:
+    """
+    Import a function from a module.
+
+    Parameters
+    ----------
+    path : Path
+        Path where the function source is located.
+    handler : str
+        Function name.
+
+    Returns
+    -------
+    Callable
+        Function.
+    """
+    spec = imputil.spec_from_file_location(path.stem, path)
+    mod = imputil.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return getattr(mod, handler)
+
+
+def list_enum(enum: Enum) -> list:
+    """
+    Get all values of an enum.
+
+    Parameters
+    ----------
+    enum : Enum
+        Enum to get values from.
+
+    Returns
+    -------
+    list
+        List of enum values.
+    """
+    return [e.value for e in enum]

digitalhub/utils/io_utils.py

@@ -32,6 +32,24 @@ def write_yaml(filepath: str | Path, obj: dict | list[dict]) -> None:
             yaml.dump(obj, out_file, sort_keys=False)
 
 
+def write_text(filepath: Path, text: str) -> None:
+    """
+    Write text to a file.
+
+    Parameters
+    ----------
+    filepath : Path
+        The file path to write.
+    text : str
+        The text to write.
+
+    Returns
+    -------
+    None
+    """
+    filepath.write_text(text, encoding="utf-8")
+
+
 ##############################
 # Readers
 ##############################

digitalhub/utils/uri_utils.py

@@ -1,7 +1,78 @@
 from __future__ import annotations
 
+from enum import Enum
 from urllib.parse import urlparse
 
+from digitalhub.utils.generic_utils import list_enum
+
+
+class S3Schemes(Enum):
+    """
+    S3 schemes.
+    """
+
+    S3 = "s3"
+    S3A = "s3a"
+    S3N = "s3n"
+    ZIP_S3 = "zip+s3"
+
+
+class LocalSchemes(Enum):
+    """
+    Local schemes.
+    """
+
+    LOCAL = ""
+
+
+class InvalidLocalSchemes(Enum):
+    """
+    Local schemes.
+    """
+
+    FILE = "file"
+    LOCAL = "local"
+
+
+class RemoteSchemes(Enum):
+    """
+    Remote schemes.
+    """
+
+    HTTP = "http"
+    HTTPS = "https"
+
+
+class SqlSchemes(Enum):
+    """
+    Sql schemes.
+    """
+
+    SQL = "sql"
+    POSTGRESQL = "postgresql"
+
+
+class GitSchemes(Enum):
+    """
+    Git schemes.
+    """
+
+    GIT = "git"
+    GIT_HTTP = "git+http"
+    GIT_HTTPS = "git+https"
+
+
+class SchemeCategory(Enum):
+    """
+    Scheme types.
+    """
+
+    S3 = "s3"
+    LOCAL = "local"
+    REMOTE = "remote"
+    SQL = "sql"
+    GIT = "git"
+
 
 def map_uri_scheme(uri: str) -> str:
     """
@@ -23,22 +94,22 @@ def map_uri_scheme(uri: str) -> str:
         If the scheme is unknown.
     """
     scheme = urlparse(uri).scheme
-    if scheme in [""]:
-        return "local"
-    if scheme in ["file", "local"]:
-        raise ValueError("For local path, do not use any scheme")
-    if scheme in ["http", "https"]:
-        return "remote"
-    if scheme in ["s3", "s3a", "s3n", "zip+s3"]:
-        return "s3"
-    if scheme in ["sql", "postgresql"]:
-        return "sql"
-    if scheme in ["git", "git+http", "git+https"]:
-        return "git"
+    if scheme in list_enum(LocalSchemes):
+        return SchemeCategory.LOCAL.value
+    if scheme in list_enum(InvalidLocalSchemes):
+        raise ValueError("For local path, do not use any scheme.")
+    if scheme in list_enum(RemoteSchemes):
+        return SchemeCategory.REMOTE.value
+    if scheme in list_enum(S3Schemes):
+        return SchemeCategory.S3.value
+    if scheme in list_enum(SqlSchemes):
+        return SchemeCategory.SQL.value
+    if scheme in list_enum(GitSchemes):
+        return SchemeCategory.GIT.value
     raise ValueError(f"Unknown scheme '{scheme}'!")
 
 
-def check_local_path(path: str) -> bool:
+def has_local_scheme(path: str) -> bool:
     """
     Check if path is local.
 
@@ -52,5 +123,72 @@ def check_local_path(path: str) -> bool:
     bool
         True if path is local.
     """
-    scheme = map_uri_scheme(path)
-    return scheme == "local"
+    return map_uri_scheme(path) == SchemeCategory.LOCAL.value
+
+
+def has_remote_scheme(path: str) -> bool:
+    """
+    Check if path is remote.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is remote.
+    """
+    return map_uri_scheme(path) == SchemeCategory.REMOTE.value
+
+
+def has_s3_scheme(path: str) -> bool:
+    """
+    Check if path is s3.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is s3.
+    """
+    return map_uri_scheme(path) == SchemeCategory.S3.value
+
+
+def has_sql_scheme(path: str) -> bool:
+    """
+    Check if path is sql.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is sql.
+    """
+    return map_uri_scheme(path) == SchemeCategory.SQL.value
+
+
+def has_git_scheme(path: str) -> bool:
+    """
+    Check if path is git.
+
+    Parameters
+    ----------
+    path : str
+        Path of some source.
+
+    Returns
+    -------
+    bool
+        True if path is git.
+    """
+    return map_uri_scheme(path) == SchemeCategory.GIT.value

{digitalhub-0.8.1.dist-info → digitalhub-0.9.0b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: digitalhub
-Version: 0.8.1
+Version: 0.9.0b0
 Summary: Python SDK for Digitalhub
 Author-email: Fondazione Bruno Kessler <dslab@fbk.eu>, Matteo Martini <mmartini@fbk.eu>
 License:                               Apache License
@@ -230,7 +230,7 @@ Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE.txt
 Requires-Dist: boto3
-Requires-Dist: pydantic<2
+Requires-Dist: pydantic
 Requires-Dist: sqlalchemy<2
 Requires-Dist: pyarrow
 Requires-Dist: numpy<2