hopeit.dataframes 0.24.0__tar.gz → 0.24.2__tar.gz

{hopeit.dataframes-0.24.0 → hopeit.dataframes-0.24.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.0
+Version: 0.24.2
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.0
+Requires-Dist: hopeit.engine[fs-storage]==0.24.2
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit.dataframes-0.24.0 → hopeit.dataframes-0.24.2}/setup.py

@@ -39,6 +39,8 @@ setuptools.setup(
     package_dir={"": "src"},
     packages=[
         "hopeit.dataframes",
+        "hopeit.dataframes.serialization",
+        "hopeit.dataframes.setup",
     ],
     include_package_data=True,
     package_data={

{hopeit.dataframes-0.24.0 → hopeit.dataframes-0.24.2}/src/hopeit/dataframes/dataframe.py

@@ -81,9 +81,10 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
     def __init_from_series__(
         self, **series: pd.Series
     ):  # pylint: disable=bad-staticmethod-argument
-        if self.__data_object__["validate"]:
-            series = self._coerce_datatypes(series)
         df = pd.DataFrame(series)
+        df.index.name = None  # Removes index name to avoid colisions with series name
+        if self.__data_object__["validate"]:
+            df = pd.DataFrame(self._coerce_datatypes(df))
         setattr(self, "__df", df[self.__dataframe__.columns])
 
     @classmethod
@@ -171,9 +172,9 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         else:
             object.__setattr__(self, name, value)
 
-    def _coerce_datatypes(self, series: Dict[str, pd.Series]) -> Dict[str, pd.Series]:
+    def _coerce_datatypes(self, df: pd.DataFrame) -> Dict[str, pd.Series]:
         return {
-            name: self.DATATYPE_MAPPING[field.type](series[name])  # type: ignore
+            name: self.DATATYPE_MAPPING[field.type](df[name])  # type: ignore
             for name, field in self.__dataframe__.fields.items()
         }
 

hopeit.dataframes-0.24.2/src/hopeit/dataframes/serialization/dataset.py

@@ -0,0 +1,28 @@
+"""Dataset objects definition, used as a result of serialized dataframes
+"""
+
+from importlib import import_module
+from typing import Type, TypeVar
+
+from hopeit.dataobjects import dataclass, dataobject
+
+DataFrameT = TypeVar("DataFrameT")
+
+
+@dataobject
+@dataclass
+class Dataset:
+    protocol: str
+    partition_key: str
+    key: str
+    datatype: str
+
+
+def find_protocol_impl(qual_type_name: str) -> Type:
+    mod_name, type_name = (
+        ".".join(qual_type_name.split(".")[:-1]),
+        qual_type_name.split(".")[-1],
+    )
+    module = import_module(mod_name)
+    datatype = getattr(module, type_name)
+    return datatype

hopeit.dataframes-0.24.2/src/hopeit/dataframes/serialization/files.py

@@ -0,0 +1,116 @@
+"""Support for `@dataframes` serialization to files
+"""
+
+import io
+from importlib import import_module
+from typing import Callable, Generic, Optional, Type, TypeVar, Union
+from uuid import uuid4
+
+import pandas as pd
+
+try:
+    import pyarrow  # type: ignore  # noqa  # pylint: disable=unused-import
+except ImportError as e:
+    raise ImportError(
+        "`pyarrow` needs to be installed to use `DatasetFileStorage`",
+        "Run `pip install hopeit.dataframes[pyarrow]`",
+    ) from e
+
+from hopeit.dataframes.dataframe import DataFrameMixin
+from hopeit.dataframes.serialization.dataset import Dataset
+from hopeit.dataobjects import EventPayloadType
+from hopeit.fs_storage import FileStorage
+
+DataFrameT = TypeVar("DataFrameT", bound=DataFrameMixin)
+
+
+class DatasetFileStorage(Generic[DataFrameT]):
+    """Support to store dataframes as files,
+    using pandas parquet format support in combination
+    with `hopeit.engine` file storage plugins
+    """
+
+    def __init__(self, *, location: str, partition_dateformat: Optional[str], **kwargs):
+        self.storage: FileStorage = FileStorage(
+            path=location, partition_dateformat=partition_dateformat
+        )
+
+    async def save(self, dataframe: DataFrameT) -> Dataset:
+        """Saves @dataframe annotated object as parquet to file system
+        and returns Dataset metadata to be used for retrieval
+        """
+        datatype = type(dataframe)
+        key = f"{datatype.__qualname__.lower()}_{uuid4()}.parquet"
+        data = io.BytesIO(
+            dataframe._df.to_parquet(  # pylint: disable=protected-access
+                engine="pyarrow"
+            )
+        )
+        location = await self.storage.store_file(file_name=key, value=data)
+        partition_key = self.storage.partition_key(location)
+
+        return Dataset(
+            protocol=f"{__name__}.{type(self).__name__}",
+            partition_key=partition_key,
+            key=key,
+            datatype=f"{datatype.__module__}.{datatype.__qualname__}",
+        )
+
+    async def load(self, dataset: Dataset) -> EventPayloadType:
+        """Loads @dataframe annotated object using Dataset metadata"""
+        datatype: Type[DataFrameT] = find_dataframe_type(dataset.datatype)
+        data = await self.storage.get_file(
+            dataset.key, partition_key=dataset.partition_key
+        )
+        if data is None:
+            raise FileNotFoundError(dataset.key)
+        df = pd.read_parquet(io.BytesIO(data), engine="pyarrow")
+        return datatype._from_df(df)  # pylint: disable=protected-access
+
+    async def ser_wrapper(
+        self,
+        base_serialization: Callable,
+        data: Union[EventPayloadType, DataFrameT],
+        level: int,
+    ) -> bytes:
+        """Serialization wrapper that plugins-in into hopeit.engine
+        serialization when dataframes plugin is initialized
+        """
+        if hasattr(data, "__dataframeobject__"):
+            data = await data._serialize()  # type: ignore  # pylint: disable=protected-access
+        if hasattr(data, "__dataframe__"):
+            data = await self.save(data)  # type: ignore
+        return await base_serialization(data, level)
+
+    async def deser_wrapper(
+        self,
+        base_deserialization: Callable,
+        data: bytes,
+        datatype: Union[Type[EventPayloadType], Type[DataFrameT]],
+    ) -> Union[EventPayloadType, DataFrameT]:
+        """Deerialization wrapper that plugins-in into hopeit.engine
+        deserialization when dataframes plugin is initialized
+        """
+        if hasattr(datatype, "__dataframeobject__"):
+            dataset = await base_deserialization(
+                data, datatype.__dataframeobject__.serialized_type  # type: ignore
+            )
+            return await datatype._deserialize(dataset)  # type: ignore  # pylint: disable=protected-access
+        if hasattr(datatype, "__dataframe__"):
+            dataset = await base_deserialization(data, Dataset)
+            return await self.load(dataset)
+        return await base_deserialization(data, datatype)
+
+
+def find_dataframe_type(qual_type_name: str) -> Type[DataFrameT]:
+    """Returns dataframe class based on type name used during serialization"""
+    mod_name, type_name = (
+        ".".join(qual_type_name.split(".")[:-1]),
+        qual_type_name.split(".")[-1],
+    )
+    module = import_module(mod_name)
+    datatype = getattr(module, type_name)
+    assert hasattr(
+        datatype, "__dataframe__"
+    ), f"Type {qual_type_name} must be annotated with `@dataframe`."
+    return datatype

hopeit.dataframes-0.24.2/src/hopeit/dataframes/serialization/settings.py

@@ -0,0 +1,14 @@
+"""Support for plugin configuration
+"""
+
+from typing import Optional
+
+from hopeit.dataobjects import dataclass, dataobject
+
+
+@dataobject
+@dataclass
+class DatasetSerialization:
+    protocol: str
+    location: str
+    partition_dateformat: Optional[str] = None

hopeit.dataframes-0.24.2/src/hopeit/dataframes/setup/dataframes.py

@@ -0,0 +1,52 @@
+"""hopeit.engine dataframes plugin SETUP event.
+
+This event executes when engine starts with dataframes plugin configuration file loaded,
+and ensures that the engine will support serialization of `@dataframe` and `@dataframeobject`
+types
+"""
+
+from functools import partial
+
+from hopeit.app.context import EventContext
+from hopeit.app.logger import app_logger
+from hopeit.dataframes.dataframeobject import DataFrameObjectMixin
+from hopeit.dataframes.serialization.dataset import find_protocol_impl
+from hopeit.dataframes.serialization.settings import DatasetSerialization
+from hopeit.server import serialization
+
+logger = app_logger()
+
+__steps__ = ["register_serialization"]
+
+
+def register_serialization(payload: None, context: EventContext) -> None:
+    """Setups serizaltion wrappers in hopeit.engine based on
+    `DataSerialization` settings configured in plugin configuration file
+    """
+    logger.info(context, "Registering serialization methods...")
+
+    settings: DatasetSerialization = context.settings(
+        key="dataset_serialization", datatype=DatasetSerialization
+    )
+    impl = find_protocol_impl(settings.protocol)
+
+    storage = impl(
+        protocol=settings.protocol,
+        location=settings.location,
+        partition_dateformat=settings.partition_dateformat,
+    )
+    setattr(DataFrameObjectMixin, "_DataFrameObjectMixin__storage", storage)
+
+    serdeser_wrappers = {}
+    for (
+        serdeser,
+        methods,
+    ) in serialization._SERDESER.items():  # pylint: disable=protected-access
+        serdeser_wrappers[serdeser] = (
+            partial(storage.ser_wrapper, methods[0]),
+            methods[1],
+            partial(storage.deser_wrapper, methods[2]),
+        )
+
+    for serdeser, methods in serdeser_wrappers.items():
+        serialization._SERDESER[serdeser] = methods  # pylint: disable=protected-access

{hopeit.dataframes-0.24.0 → hopeit.dataframes-0.24.2}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.0
+Version: 0.24.2
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.0
+Requires-Dist: hopeit.engine[fs-storage]==0.24.2
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit.dataframes-0.24.0 → hopeit.dataframes-0.24.2}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -8,4 +8,10 @@ src/hopeit.dataframes.egg-info/top_level.txt
 src/hopeit/dataframes/__init__.py
 src/hopeit/dataframes/dataframe.py
 src/hopeit/dataframes/dataframeobject.py
-src/hopeit/dataframes/py.typed
+src/hopeit/dataframes/py.typed
+src/hopeit/dataframes/serialization/__init__.py
+src/hopeit/dataframes/serialization/dataset.py
+src/hopeit/dataframes/serialization/files.py
+src/hopeit/dataframes/serialization/settings.py
+src/hopeit/dataframes/setup/__init__.py
+src/hopeit/dataframes/setup/dataframes.py

hopeit.dataframes-0.24.2/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.24.2
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit.dataframes-0.24.0/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.24.0
-pandas
-numpy
-
-[pyarrow]
-pyarrow