digitalhub 0.10.0b1__py3-none-any.whl → 0.10.0b2__py3-none-any.whl

digitalhub/entities/dataitem/_base/entity.py

@@ -1,13 +1,9 @@
 from __future__ import annotations
 
 import typing
-from pathlib import Path
 
 from digitalhub.entities._base.material.entity import MaterialEntity
 from digitalhub.entities._commons.enums import EntityTypes
-from digitalhub.entities.dataitem.utils import DEFAULT_EXTENSION
-from digitalhub.utils.exceptions import EntityError
-from digitalhub.utils.uri_utils import has_sql_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -36,40 +32,3 @@ class Dataitem(MaterialEntity):
         super().__init__(project, name, uuid, kind, metadata, spec, status, user)
         self.spec: DataitemSpec
         self.status: DataitemStatus
-
-    ##############################
-    #  Helper methods
-    ##############################
-
-    @staticmethod
-    def _get_extension(path: str, file_format: str | None = None) -> str:
-        """
-        Get extension of path.
-
-        Parameters
-        ----------
-        path : str
-            Path to get extension from.
-        file_format : str
-            File format.
-
-        Returns
-        -------
-        str
-            File extension.
-
-        Raises
-        ------
-        EntityError
-            If file format is not supported.
-        """
-        if file_format is not None:
-            return file_format
-
-        if has_sql_scheme(path):
-            return DEFAULT_EXTENSION
-
-        ext = Path(path).suffix[1:]
-        if ext is not None:
-            return ext
-        raise EntityError("Unknown file format. Only csv and parquet are supported.")

digitalhub/entities/dataitem/table/entity.py

@@ -7,7 +7,7 @@ from typing import Any
 
 from digitalhub.entities.dataitem._base.entity import Dataitem
 from digitalhub.stores.api import get_store
-from digitalhub.utils.uri_utils import has_local_scheme
+from digitalhub.utils.uri_utils import has_sql_scheme
 
 if typing.TYPE_CHECKING:
     from digitalhub.entities._base.entity.metadata import Metadata
@@ -36,19 +36,39 @@ class DataitemTable(Dataitem):
         self.spec: DataitemSpecTable
         self.status: DataitemStatusTable
 
+        self._query: str | None = None
+
+    def query(self, query: str) -> DataitemTable:
+        """
+        Set query to execute.
+
+        Parameters
+        ----------
+        query : str
+            Query to execute.
+
+        Returns
+        -------
+        DataitemTable
+            Self object.
+        """
+        # to remove in future
+        if not has_sql_scheme(self.spec.path):
+            raise ValueError(
+                f"Dataitem path is not a SQL scheme: {self.spec.path}",
+                " Query can be made only on a SQL scheme.",
+            )
+        self._query = query
+        return self
+
     def as_df(
         self,
         file_format: str | None = None,
-        engine: str | None = None,
-        clean_tmp_path: bool = True,
+        engine: str | None = "pandas",
         **kwargs,
     ) -> Any:
         """
         Read dataitem file (csv or parquet) as a DataFrame from spec.path.
-        If the dataitem is not local, it will be downloaded to a temporary
-        folder named tmp_dir in the project context folder.
-        If clean_tmp_path is True, the temporary folder will be deleted after the
-        method is executed.
         It's possible to pass additional arguments to the this function. These
         keyword arguments will be passed to the DataFrame reader function such as
         pandas's read_csv or read_parquet.
@@ -56,11 +76,10 @@ class DataitemTable(Dataitem):
         Parameters
         ----------
         file_format : str
-            Format of the file. (Supported csv and parquet).
+            Format of the file to read. By default, it will be inferred from
+            the extension of the file.
         engine : str
             Dataframe framework, by default pandas.
-        clean_tmp_path : bool
-            If True, the temporary folder will be deleted.
         **kwargs : dict
             Keyword arguments passed to the read_df function.
 
@@ -69,30 +88,20 @@ class DataitemTable(Dataitem):
         Any
             DataFrame.
         """
-        try:
-            if has_local_scheme(self.spec.path):
-                tmp_dir = None
-                data_path = self.spec.path
-            else:
-                tmp_dir = self._context().root / "tmp_data"
-                tmp_dir.mkdir(parents=True, exist_ok=True)
-                data_path = self.download(destination=str(tmp_dir), overwrite=True)
-
-            if Path(data_path).is_dir():
-                files = [str(i) for i in Path(data_path).rglob("*") if i.is_file()]
-                checker = files[0]
-            else:
-                checker = data_path
-
-            extension = self._get_extension(checker, file_format)
-            return get_store(self.project, "").read_df(data_path, extension, engine, **kwargs)
-
-        except Exception as e:
-            raise e
-
-        finally:
-            # Delete tmp folder
-            self._clean_tmp_path(tmp_dir, clean_tmp_path)
+        if self._query is not None:
+            df = get_store(self.project, self.spec.path).query(
+                self._query,
+                self.spec.path,
+                engine,
+            )
+            self._query = None
+            return df
+        return get_store(self.project, self.spec.path).read_df(
+            self.spec.path,
+            file_format,
+            engine,
+            **kwargs,
+        )
 
     def write_df(
         self,
@@ -119,7 +128,12 @@ class DataitemTable(Dataitem):
         str
             Path to the written dataframe.
         """
-        return get_store(self.project, self.spec.path).write_df(df, self.spec.path, extension=extension, **kwargs)
+        return get_store(self.project, self.spec.path).write_df(
+            df,
+            self.spec.path,
+            extension=extension,
+            **kwargs,
+        )
 
     @staticmethod
     def _clean_tmp_path(pth: Path | None, clean: bool) -> None:

digitalhub/entities/dataitem/utils.py

@@ -8,7 +8,7 @@ from digitalhub.context.api import get_context
 from digitalhub.entities._base.entity._constructors.uuid import build_uuid
 from digitalhub.entities._base.material.utils import build_log_path_from_source, eval_local_source
 from digitalhub.entities._commons.enums import EntityKinds, EntityTypes
-from digitalhub.readers.api import get_reader_by_object
+from digitalhub.readers.data.api import get_reader_by_object
 from digitalhub.utils.generic_utils import slugify_string
 
 if typing.TYPE_CHECKING:

digitalhub/readers/{_base → data/_base}/builder.py

@@ -6,7 +6,7 @@ from abc import abstractmethod
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 class ReaderBuilder:

digitalhub/readers/{_base → data/_base}/reader.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from abc import abstractmethod
-from typing import Any
+from typing import IO, Any
 
 
 class DataframeReader:
@@ -13,11 +13,16 @@ class DataframeReader:
     # Read methods
     ##############################
 
-    @staticmethod
     @abstractmethod
-    def read_df(path: str | list[str], extension: str, **kwargs) -> Any:
+    def read_df(self, path_or_buffer: str | IO, extension: str, **kwargs) -> Any:
+        """
+        Read DataFrame from path or buffer.
+        """
+
+    @abstractmethod
+    def read_table(self, *args, **kwargs) -> Any:
         """
-        Read DataFrame from path.
+        Read table from db.
         """
 
     ##############################
@@ -68,3 +73,10 @@ class DataframeReader:
         """
         Get preview.
         """
+
+    @staticmethod
+    @abstractmethod
+    def concat_dfs(dfs: list[Any]) -> Any:
+        """
+        Concatenate a list of DataFrames.
+        """

digitalhub/readers/{api.py → data/api.py}

@@ -3,11 +3,11 @@ from __future__ import annotations
 import typing
 from typing import Any
 
-from digitalhub.readers.factory import factory
+from digitalhub.readers.data.factory import factory
 from digitalhub.utils.exceptions import ReaderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 def get_reader_by_engine(engine: str | None = None) -> DataframeReader:

digitalhub/readers/{factory.py → data/factory.py}

@@ -5,8 +5,8 @@ import typing
 from digitalhub.utils.exceptions import BuilderError
 
 if typing.TYPE_CHECKING:
-    from digitalhub.readers._base.builder import ReaderBuilder
-    from digitalhub.readers._base.reader import DataframeReader
+    from digitalhub.readers.data._base.builder import ReaderBuilder
+    from digitalhub.readers.data._base.reader import DataframeReader
 
 
 class ReaderFactory:
@@ -126,7 +126,7 @@ class ReaderFactory:
 factory = ReaderFactory()
 
 try:
-    from digitalhub.readers.pandas.builder import ReaderBuilderPandas
+    from digitalhub.readers.data.pandas.builder import ReaderBuilderPandas
 
     factory.add_builder(
         ReaderBuilderPandas.ENGINE,

digitalhub/readers/{pandas → data/pandas}/builder.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
-from digitalhub.readers._base.builder import ReaderBuilder
-from digitalhub.readers.pandas.reader import DataframeReaderPandas
+from digitalhub.readers.data._base.builder import ReaderBuilder
+from digitalhub.readers.data.pandas.reader import DataframeReaderPandas
 
 
 class ReaderBuilderPandas(ReaderBuilder):

digitalhub/readers/{pandas → data/pandas}/reader.py

@@ -2,15 +2,15 @@ from __future__ import annotations
 
 import json
 from io import BytesIO
-from typing import Any
+from typing import IO, Any
 
 import numpy as np
 import pandas as pd
 from pandas.errors import ParserError
 
 from digitalhub.entities.dataitem.table.utils import check_preview_size, finalize_preview, prepare_data, prepare_preview
-from digitalhub.readers._base.reader import DataframeReader
-from digitalhub.readers.pandas.enums import Extensions
+from digitalhub.readers.data._base.reader import DataframeReader
+from digitalhub.readers.data.pandas.enums import Extensions
 from digitalhub.utils.exceptions import ReaderError
 from digitalhub.utils.generic_utils import CustomJsonEncoder
 
@@ -24,14 +24,14 @@ class DataframeReaderPandas(DataframeReader):
     # Read methods
     ##############################
 
-    def read_df(self, path: str | list[str], extension: str, **kwargs) -> pd.DataFrame:
+    def read_df(self, path_or_buffer: str | IO, extension: str, **kwargs) -> pd.DataFrame:
         """
-        Read DataFrame from path.
+        Read DataFrame from path or buffer.
 
         Parameters
         ----------
-        path : str | list[str]
-            Path(s) to read DataFrame from.
+        path_or_buffer : str | IO
+            Path or buffer to read DataFrame from.
         extension : str
             Extension of the file.
         **kwargs : dict
@@ -43,25 +43,40 @@ class DataframeReaderPandas(DataframeReader):
             Pandas DataFrame.
         """
         if extension == Extensions.CSV.value:
-            method = pd.read_csv
-        elif extension == Extensions.PARQUET.value:
-            method = pd.read_parquet
-        elif extension == Extensions.JSON.value:
-            method = pd.read_json
-        elif extension in (Extensions.EXCEL.value, Extensions.EXCEL_OLD.value):
-            method = pd.read_excel
-        elif extension in (Extensions.TXT.value, Extensions.FILE.value):
+            return pd.read_csv(path_or_buffer, **kwargs)
+        if extension == Extensions.PARQUET.value:
+            return pd.read_parquet(path_or_buffer, **kwargs)
+        if extension == Extensions.JSON.value:
+            return pd.read_json(path_or_buffer, **kwargs)
+        if extension in (Extensions.EXCEL.value, Extensions.EXCEL_OLD.value):
+            return pd.read_excel(path_or_buffer, **kwargs)
+        if extension in (Extensions.TXT.value, Extensions.FILE.value):
             try:
-                return self.read_df(path, Extensions.CSV.value, **kwargs)
+                return self.read_df(path_or_buffer, Extensions.CSV.value, **kwargs)
             except ParserError:
-                raise ReaderError(f"Unable to read from {path}.")
+                raise ReaderError(f"Unable to read from {path_or_buffer}.")
         else:
             raise ReaderError(f"Unsupported extension '{extension}' for reading.")
 
-        if isinstance(path, list):
-            dfs = [method(p, **kwargs) for p in path]
-            return pd.concat(dfs)
-        return method(path, **kwargs)
+    def read_table(self, sql: str, engine: Any, **kwargs) -> pd.DataFrame:
+        """
+        Read table from db.
+
+        Parameters
+        ----------
+        sql : str
+            SQL query.
+        engine : Any
+            SQL Engine.
+        **kwargs
+            Keyword arguments.
+
+        Returns
+        -------
+        pd.DataFrame
+            Pandas DataFrame.
+        """
+        return pd.read_sql(sql=sql, con=engine, **kwargs)
 
     ##############################
     # Write methods
@@ -92,7 +107,7 @@ class DataframeReaderPandas(DataframeReader):
         """
         if extension == Extensions.CSV.value:
             return self.write_csv(df, dst, **kwargs)
-        elif extension == Extensions.PARQUET.value:
+        if extension == Extensions.PARQUET.value:
             return self.write_parquet(df, dst, **kwargs)
         raise ReaderError(f"Unsupported extension '{extension}' for writing.")
 
@@ -137,7 +152,7 @@ class DataframeReaderPandas(DataframeReader):
         df.to_parquet(dst, index=False, **kwargs)
 
     @staticmethod
-    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str, **kwargs) -> None:
+    def write_table(df: pd.DataFrame, table: str, engine: Any, schema: str | None = None, **kwargs) -> None:
         """
         Write DataFrame as table.
 
@@ -148,7 +163,7 @@ class DataframeReaderPandas(DataframeReader):
         table : str
             The destination table.
         engine : Any
-            The SQLAlchemy engine.
+            SQL Engine.
         schema : str
             The destination schema.
         **kwargs : dict
@@ -164,6 +179,23 @@ class DataframeReaderPandas(DataframeReader):
     # Utils
     ##############################
 
+    @staticmethod
+    def concat_dfs(dfs: list[pd.DataFrame]) -> pd.DataFrame:
+        """
+        Concatenate a list of DataFrames.
+
+        Parameters
+        ----------
+        dfs : list[pd.DataFrame]
+            The DataFrames to concatenate.
+
+        Returns
+        -------
+        pd.DataFrame
+            The concatenated DataFrame.
+        """
+        return pd.concat(dfs, ignore_index=True)
+
     @staticmethod
     def get_schema(df: pd.DataFrame) -> Any:
         """

digitalhub/stores/_base/store.py

@@ -1,14 +1,18 @@
 from __future__ import annotations
 
+import typing
 from abc import abstractmethod
 from pathlib import Path
 from tempfile import mkdtemp
 from typing import Any
 
-from digitalhub.readers.api import get_reader_by_engine
+from digitalhub.readers.data.api import get_reader_by_engine
 from digitalhub.utils.exceptions import StoreError
 from digitalhub.utils.uri_utils import has_local_scheme
 
+if typing.TYPE_CHECKING:
+    from digitalhub.readers.data._base.reader import DataframeReader
+
 
 class Store:
     """
@@ -52,11 +56,6 @@ class Store:
     ##############################
 
     @abstractmethod
-    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
-        """
-        Write DataFrame as parquet or csv.
-        """
-
     def read_df(
         self,
         path: str | list[str],
@@ -66,25 +65,23 @@ class Store:
     ) -> Any:
         """
         Read DataFrame from path.
+        """
 
-        Parameters
-        ----------
-        path : str | list[str]
-            Path(s) to read DataFrame from.
-        extension : str
-            Extension of the file.
-        engine : str
-            Dataframe engine (pandas, polars, etc.).
-        **kwargs : dict
-            Keyword arguments.
+    @abstractmethod
+    def query(
+        self,
+        query: str,
+        engine: str | None = None,
+    ) -> Any:
+        """
+        Query data from database.
+        """
 
-        Returns
-        -------
-        Any
-            DataFrame.
+    @abstractmethod
+    def write_df(self, df: Any, dst: str, extension: str | None = None, **kwargs) -> str:
+        """
+        Write DataFrame as parquet or csv.
         """
-        reader = get_reader_by_engine(engine)
-        return reader.read_df(path, extension, **kwargs)
 
     ##############################
     # Helpers methods
@@ -187,3 +184,43 @@ class Store:
         """
         tmpdir = mkdtemp()
         return Path(tmpdir)
+
+    @staticmethod
+    def _get_reader(engine: str | None = None) -> DataframeReader:
+        """
+        Get Dataframe reader.
+
+        Parameters
+        ----------
+        engine : str
+            Dataframe engine (pandas, polars, etc.).
+
+        Returns
+        -------
+        Any
+            Reader object.
+        """
+        return get_reader_by_engine(engine)
+
+    @staticmethod
+    def _get_extension(extension: str | None = None, path: str | None = None) -> str:
+        """
+        Get extension from path.
+
+        Parameters
+        ----------
+        extension : str
+            The extension to get.
+        path : str
+            The path to get the extension from.
+
+        Returns
+        -------
+        str
+            The extension.
+        """
+        if extension is not None:
+            return extension
+        if path is not None:
+            return Path(path).suffix.removeprefix(".")
+        raise ValueError("Extension or path must be provided.")