hopeit.dataframes 0.25.3__tar.gz → 0.25.4__tar.gz

{hopeit_dataframes-0.25.3 → hopeit_dataframes-0.25.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.3
+Version: 0.25.4
 Summary: Hopeit Engine Dataframes Toolkit
 Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
@@ -24,7 +24,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.9
 Description-Content-Type: text/plain
-Requires-Dist: hopeit.engine[fs-storage]==0.25.3
+Requires-Dist: hopeit.engine[fs-storage]==0.25.4
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit_dataframes-0.25.3 → hopeit_dataframes-0.25.4}/src/hopeit/dataframes/__init__.py

@@ -77,12 +77,13 @@ from typing import Dict, Generic, Iterator, List, Type
 import numpy as np
 import pandas as pd
 from hopeit.dataframes.dataframe import DataFrameT, dataframe
+from hopeit.dataframes.datablocks import DataBlocks
 from hopeit.dataframes.serialization.dataset import Dataset
 from hopeit.dataframes.serialization.settings import DatasetSerialization
 from hopeit.dataframes.setup.dataframes import register_serialization
 from hopeit.dataobjects import DataObject
 
-__all__ = ["DataFrames", "Dataset", "dataframe"]
+__all__ = ["DataBlocks", "DataFrames", "Dataset", "dataframe"]
 
 
 class DataFrames(Generic[DataFrameT, DataObject]):

hopeit_dataframes-0.25.4/src/hopeit/dataframes/datablocks.py

@@ -0,0 +1,140 @@
+from typing import Generic, Optional, Type, TypeVar, get_args, get_origin
+
+import pandas as pd
+from hopeit.dataobjects import fields
+
+from hopeit.dataframes.serialization.dataset import Dataset, DatasetLoadError, find_dataframe_type
+
+DataBlockType = TypeVar("DataBlockType")
+DataBlockItemType = TypeVar("DataBlockItemType")
+DataFrameType = TypeVar("DataFrameType")
+
+
+class TempDataBlock(Generic[DataBlockType, DataBlockItemType]):
+    def __init__(self, datatype: Type[DataBlockType], df: pd.DataFrame):
+        self.datatype = datatype
+        self.df = df
+
+    @classmethod
+    def from_dataobjects(
+        cls, datatype: Type[DataBlockType], items: list[DataBlockItemType]
+    ) -> "TempDataBlock[DataBlockType, DataBlockItemType]":
+        result_df: Optional[pd.DataFrame] = None
+        for field_name, field_info in fields(datatype).items():  # type: ignore[type-var]
+            if get_origin(field_info.annotation) is Dataset:
+                block_items = (getattr(item, field_name) for item in items)
+                block_type = get_args(field_info.annotation)[0]
+                block = block_type._from_dataobjects(block_items)
+                block_df = block._df
+            else:
+                block_df = pd.DataFrame({field_name: [getattr(item, field_name) for item in items]})
+
+            if result_df is None:
+                result_df = block_df
+            else:
+                # Skips duplicated column names to they are included only once
+                result_df = result_df.join(
+                    block_df[[col for col in block_df.columns if col not in result_df.columns]]
+                )
+        assert result_df is not None
+        return cls(datatype, result_df)
+
+    def to_dataobjects(
+        self, item_type: Type[DataBlockItemType], *, normalize_null_values: bool = False
+    ) -> list[DataBlockItemType]:
+        keys: list[str] = []
+        entries: list[list] = []
+        for field_name, field_info in fields(self.datatype).items():  # type: ignore[type-var]
+            if get_origin(field_info.annotation) is Dataset:
+                block_type = get_args(field_info.annotation)[0]
+                keys.append(field_name)
+                dataframe = block_type._from_df(self.df)
+                entries.append(
+                    dataframe._to_dataobjects(normalize_null_values=normalize_null_values)
+                )
+            else:
+                keys.append(field_name)
+                entries.append(self.df[field_name].to_list())
+
+        return [
+            item_type(**{field_name: entry[i] for i, field_name in enumerate(keys)})
+            for entry in zip(*entries)
+        ]
+
+
+class DataBlocks(Generic[DataBlockType, DataFrameType]):
+    @classmethod
+    async def df(cls, datablock: DataBlockType, select: Optional[list[str]] = None) -> pd.DataFrame:
+        keys = [
+            field_name
+            for field_name, field_info in fields(datablock).items()  # type: ignore[arg-type]
+            if get_origin(field_info.annotation) is Dataset
+            and (select is None or field_name in select)
+        ]
+
+        # Filter/validate selected field names using saved schema,
+        # generates a single field for every common/duplicated field in the datasets
+        field_names = list(
+            dict.fromkeys(
+                [
+                    field_name
+                    for key in keys
+                    for field_name in getattr(datablock, key).schema["properties"].keys()
+                ]
+            )
+        )
+
+        # Load data from first dataset (datablock uses a single file for all datasets)
+        dataset: Dataset = getattr(datablock, keys[0])
+        result_df = await DataBlocks._load_datablock_df(dataset, field_names)
+
+        # Add missing optional fields using class schema (allows schema evolution)
+        cls._adapt_to_schema(datablock, keys, result_df)
+
+        # Adding constant value fields
+        for field_name, field_info in fields(datablock).items():  # type: ignore[arg-type]
+            if get_origin(field_info.annotation) is not Dataset:
+                result_df[field_name] = getattr(datablock, field_name)  # type: ignore[index]
+
+        return result_df
+
+    @staticmethod
+    async def from_df(
+        datatype: Type[DataBlockType],
+        df: pd.DataFrame,
+        **kwargs,  # Non-Dataset field values for DataBlockType
+    ) -> DataBlockType:
+        blocks = {}
+        block_dataset = await Dataset._save_df(df, datatype)
+        for field_name, field_info in fields(datatype).items():  # type: ignore[type-var]
+            if get_origin(field_info.annotation) is Dataset:
+                block_type = get_args(field_info.annotation)[0]
+                blocks[field_name] = block_dataset._adapt(block_type)
+            else:
+                blocks[field_name] = kwargs[field_name]
+
+        return datatype(**blocks)
+
+    @staticmethod
+    def default(datatype: Type[DataBlockType]) -> DataBlockType:
+        return datatype(**{field_name: [] for field_name in list(fields(datatype))})  # type: ignore[type-var]
+
+    @staticmethod
+    async def _load_datablock_df(
+        dataset: Dataset, columns: Optional[list[str]] = None
+    ) -> pd.DataFrame:
+        try:
+            return await dataset._load_df(columns)
+        except (RuntimeError, IOError, KeyError) as e:
+            raise DatasetLoadError(
+                f"Error {type(e).__name__}: {e} loading datablock of type {dataset.datatype} "
+                f"at location {dataset.partition_key}/{dataset.key}"
+            ) from e
+
+    @classmethod
+    def _adapt_to_schema(cls, datablock: DataBlockType, keys: list[str], df: pd.DataFrame):
+        for key in keys:
+            datatype = find_dataframe_type(getattr(datablock, key).datatype)  # type: ignore[var-annotated]
+            valid_df = datatype._from_df(df)._df
+            for col in valid_df.columns:
+                df[col] = valid_df[col]

hopeit_dataframes-0.25.4/src/hopeit/dataframes/serialization/dataset.py

@@ -0,0 +1,93 @@
+"""Dataset objects definition, used as a result of serialized dataframes"""
+
+from importlib import import_module
+from typing import Any, Dict, Generic, Optional, Type, TypeVar
+
+from hopeit.dataobjects import dataclass, dataobject, field
+import pandas as pd
+from pydantic import TypeAdapter
+
+DataFrameT = TypeVar("DataFrameT")
+GenericDataFrameT = TypeVar("GenericDataFrameT")
+
+
+class DatasetLoadError(Exception):
+    pass
+
+
+class DatasetConvertError(Exception):
+    pass
+
+
+@dataobject
+@dataclass
+class Dataset(Generic[DataFrameT]):
+    """Persisted representation of a @dataframe object"""
+
+    protocol: str
+    partition_key: str
+    key: str
+    datatype: str
+    schema: Dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    async def save(cls, dataframe: DataFrameT) -> "Dataset[DataFrameT]":
+        return await cls.__storage.save(dataframe)  # type: ignore[attr-defined]
+
+    async def load(self) -> DataFrameT:
+        try:
+            df = await self._load_df()
+            return self._convert(df)
+        except (RuntimeError, IOError, KeyError) as e:
+            raise DatasetLoadError(
+                f"Error {type(e).__name__}: {e} loading dataset of type {self.datatype} "
+                f"at location {self.partition_key}/{self.key}"
+            ) from e
+
+    async def _load_df(self, columns: Optional[list[str]] = None) -> pd.DataFrame:
+        return await self.__storage.load_df(self, columns)  # type: ignore[attr-defined]
+
+    def _convert(self, df: pd.DataFrame) -> DataFrameT:
+        """Converts loaded pandas Dataframe to @dataframe annotated object using Dataset metadata"""
+        datatype: Type[DataFrameT] = find_dataframe_type(self.datatype)
+        return datatype._from_df(df)  # type: ignore[attr-defined]
+
+    def _adapt(self, datatype: DataFrameT) -> "Dataset[DataFrameT]":
+        """Adapts a more generic dataset that contains combined fields to be type specific"""
+        return Dataset(
+            protocol=self.protocol,
+            partition_key=self.partition_key,
+            key=self.key,
+            datatype=f"{datatype.__module__}.{datatype.__qualname__}",  # type: ignore[attr-defined]
+            schema=TypeAdapter(datatype).json_schema(),
+        )
+
+    @classmethod
+    async def _save_df(
+        cls, df: pd.DataFrame, datatype: Type[GenericDataFrameT]
+    ) -> "Dataset[GenericDataFrameT]":
+        return await cls.__storage.save_df(df, datatype)  # type: ignore[attr-defined]
+
+
+def find_protocol_impl(qual_type_name: str) -> Type:
+    mod_name, type_name = (
+        ".".join(qual_type_name.split(".")[:-1]),
+        qual_type_name.split(".")[-1],
+    )
+    module = import_module(mod_name)
+    datatype = getattr(module, type_name)
+    return datatype
+
+
+def find_dataframe_type(qual_type_name: str) -> Type[DataFrameT]:
+    """Returns dataframe class based on type name used during serialization"""
+    mod_name, type_name = (
+        ".".join(qual_type_name.split(".")[:-1]),
+        qual_type_name.split(".")[-1],
+    )
+    module = import_module(mod_name)
+    datatype = getattr(module, type_name)
+    assert hasattr(
+        datatype, "__dataframe__"
+    ), f"Type {qual_type_name} must be annotated with `@dataframe`."
+    return datatype

{hopeit_dataframes-0.25.3 → hopeit_dataframes-0.25.4}/src/hopeit/dataframes/serialization/files.py

@@ -1,7 +1,6 @@
 """Support for `@dataframes` serialization to files"""
 
 import io
-from importlib import import_module
 from typing import Generic, Optional, Type, TypeVar
 from uuid import uuid4
 
@@ -18,7 +17,7 @@ except ImportError as e:
 
 from hopeit.dataframes.dataframe import DataFrameMixin
 from hopeit.dataframes.serialization.dataset import Dataset
-from hopeit.dataobjects import EventPayloadType
+from hopeit.dataobjects import DataObject
 from hopeit.fs_storage import FileStorage
 
 DataFrameT = TypeVar("DataFrameT", bound=DataFrameMixin)
@@ -57,25 +56,30 @@ class DatasetFileStorage(Generic[DataFrameT]):
             schema=TypeAdapter(datatype).json_schema(),
         )
 
-    async def load(self, dataset: Dataset) -> EventPayloadType:
-        """Loads @dataframe annotated object using Dataset metadata"""
-        datatype: Type[DataFrameT] = find_dataframe_type(dataset.datatype)
+    async def save_df(self, df: pd.DataFrame, datatype: Type[DataObject]) -> Dataset:
+        """Saves pandas df object as parquet to file system
+        and returns Dataset metadata to be used when retrieval
+        is handled externally
+        """
+        key = f"{datatype.__qualname__.lower()}_{uuid4()}.parquet"
+        data = io.BytesIO(
+            df.to_parquet(  # pylint: disable=protected-access
+                engine="pyarrow"
+            )
+        )
+        location = await self.storage.store_file(file_name=key, value=data)
+        partition_key = self.storage.partition_key(location)
+
+        return Dataset(
+            protocol=f"{__name__}.{type(self).__name__}",
+            partition_key=partition_key,
+            key=key,
+            datatype=f"{datatype.__module__}.{datatype.__qualname__}",
+            schema=TypeAdapter(datatype).json_schema(),
+        )
+
+    async def load_df(self, dataset: Dataset, columns: Optional[list[str]] = None) -> pd.DataFrame:
         data = await self.storage.get_file(dataset.key, partition_key=dataset.partition_key)
         if data is None:
             raise FileNotFoundError(dataset.key)
-        df = pd.read_parquet(io.BytesIO(data), engine="pyarrow")
-        return datatype._from_df(df)  # pylint: disable=protected-access
-
-
-def find_dataframe_type(qual_type_name: str) -> Type[DataFrameT]:
-    """Returns dataframe class based on type name used during serialization"""
-    mod_name, type_name = (
-        ".".join(qual_type_name.split(".")[:-1]),
-        qual_type_name.split(".")[-1],
-    )
-    module = import_module(mod_name)
-    datatype = getattr(module, type_name)
-    assert hasattr(
-        datatype, "__dataframe__"
-    ), f"Type {qual_type_name} must be annotated with `@dataframe`."
-    return datatype
+        return pd.read_parquet(io.BytesIO(data), engine="pyarrow", columns=columns)

{hopeit_dataframes-0.25.3 → hopeit_dataframes-0.25.4}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.3
+Version: 0.25.4
 Summary: Hopeit Engine Dataframes Toolkit
 Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
@@ -24,7 +24,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.9
 Description-Content-Type: text/plain
-Requires-Dist: hopeit.engine[fs-storage]==0.25.3
+Requires-Dist: hopeit.engine[fs-storage]==0.25.4
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit_dataframes-0.25.3 → hopeit_dataframes-0.25.4}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -7,6 +7,7 @@ src/hopeit.dataframes.egg-info/dependency_links.txt
 src/hopeit.dataframes.egg-info/requires.txt
 src/hopeit.dataframes.egg-info/top_level.txt
 src/hopeit/dataframes/__init__.py
+src/hopeit/dataframes/datablocks.py
 src/hopeit/dataframes/dataframe.py
 src/hopeit/dataframes/py.typed
 src/hopeit/dataframes/serialization/__init__.py

hopeit_dataframes-0.25.4/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.25.4
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit_dataframes-0.25.3/src/hopeit/dataframes/serialization/dataset.py

@@ -1,48 +0,0 @@
-"""Dataset objects definition, used as a result of serialized dataframes"""
-
-from importlib import import_module
-from typing import Any, Dict, Generic, Type, TypeVar
-
-from hopeit.dataobjects import dataclass, dataobject, field
-
-DataFrameT = TypeVar("DataFrameT")
-
-
-class DatasetLoadError(Exception):
-    pass
-
-
-@dataobject
-@dataclass
-class Dataset(Generic[DataFrameT]):
-    """Persisted representation of a @dataframe object"""
-
-    protocol: str
-    partition_key: str
-    key: str
-    datatype: str
-    schema: Dict[str, Any] = field(default_factory=dict)
-
-    async def load(self) -> DataFrameT:
-        try:
-            dataframe = await self.__storage.load(self)  # type: ignore[attr-defined]
-            return dataframe
-        except (RuntimeError, IOError, KeyError) as e:
-            raise DatasetLoadError(
-                f"Error {type(e).__name__}: {e} loading dataset of type {self.datatype} "
-                f"at location {self.partition_key}/{self.key}"
-            ) from e
-
-    @classmethod
-    async def save(cls, dataframe: DataFrameT) -> "Dataset[DataFrameT]":
-        return await cls.__storage.save(dataframe)  # type: ignore[attr-defined]
-
-
-def find_protocol_impl(qual_type_name: str) -> Type:
-    mod_name, type_name = (
-        ".".join(qual_type_name.split(".")[:-1]),
-        qual_type_name.split(".")[-1],
-    )
-    module = import_module(mod_name)
-    datatype = getattr(module, type_name)
-    return datatype

hopeit_dataframes-0.25.3/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.25.3
-pandas
-numpy
-
-[pyarrow]
-pyarrow