hopeit.dataframes 0.24.2__tar.gz → 0.25.0__tar.gz

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.2
+Version: 0.25.0
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -13,7 +13,6 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -24,9 +23,9 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.2
+Requires-Dist: hopeit.engine[fs-storage]==0.25.0
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/setup.py

@@ -18,7 +18,6 @@ setuptools.setup(
         "License :: OSI Approved :: Apache Software License",
         "Intended Audience :: Developers",
         "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
@@ -46,7 +45,7 @@ setuptools.setup(
     package_data={
         "hopeit.dataframes": ["py.typed"],
     },
-    python_requires=">=3.8",
+    python_requires=">=3.9",
     install_requires=[
         f"hopeit.engine[fs-storage]=={version['ENGINE_VERSION']}",
         "pandas",

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit/dataframes/__init__.py

@@ -1,37 +1,74 @@
 """
 hopeit.engine dataframes plugin entry point
 
-This module exposes the 3 main constructions to be used inside apps:
+This module exposes the 2 main constructions to be used inside apps,
+to extend @dataobject functionallity supporting working with `pandas DataFrames`
 `@dataframe` dataclass annotation
-`@dataframeobject` dataclass annotation
 `DataFrames` class to handle manipulation of dataframe/dataframeobjects
 
 Usage:
 ```
-from hopeit.dataframes import DataFrames, dataframe, dataframeobject
+from typing import List
+
+import pandas as pd
+
+from hopeit.dataframes.serialization.settings import DatasetSerialization
+from hopeit.dataframes import DataFrames, Dataset, dataframe
+from hopeit.dataobjects import dataobject, dataclass
+from hopeit.dataobjects.payload import Payload
 
 @dataframe
 @dataclass
-class MyDataFrame:
+class MyData:
     field1: int
     field2: str
     ...
 
-@dataframeobject
+@dataobject
 @dataclass
 class MyDataset:
     dataset_name: str
-    example_data: MyDataFrame
+    example_data: Dataset[MyData]
+
+@dataobject
+@dataclass
+class MyWebResponse:
+    dataset_name: str
+    example_data: List[MyData.DataObject]
+
+# This step is not needed if SETUP event is configured in app
+DataFrames.setup(DatasetSerialization(
+      protocol="hopeit.dataframes.serialization.files.DatasetFileStorage",
+      location="/tmp/data",
+      partition_dateformat="%Y/%m/%d/%H/",
+))
+
+df = pd.DataFrame([  # Create or load a pandas DataFrame
+    {"field1": 1, "field2": "text1"},
+    {"field1": 2, "field2": "text2"},
+])
+
+my_data: MyData = DataFrames.from_df(MyData, df)
 
+# return dataset after saving data to disk
+my_dataset = MyDataset(
+    dataset_name="example",
+    example_data=await Dataset.save(my_data)
+)
+
+print(Payload.to_json(my_dataset))
 
-df = pd.DataFrame(...)  # create or load your pandas dataframe
+my_data_again: MyData = await my_dataset.example_data.load()
 
-my_data = DataFrames.from_df(pd.DataFrame(..))
+print(DataFrames.df(my_data_again))
 
-return MyDataSet(
+# return dataframe converted to list of dataobjects that can be directly converted to json
+my_json_response = MyWebResponse(
     dataset_name="example",
-    example_data=my_data
+    example_data=DataFrames.to_dataobjects(my_data)
 )
+
+print(Payload.to_json(my_json_response))
 ```
 """
 
@@ -40,35 +77,22 @@ from typing import Dict, Generic, Iterator, List, Type
 import numpy as np
 import pandas as pd
 from hopeit.dataframes.dataframe import DataFrameT, dataframe
-from hopeit.dataframes.dataframeobject import DataFrameObjectT, dataframeobject
+from hopeit.dataframes.serialization.dataset import Dataset
+from hopeit.dataframes.serialization.settings import DatasetSerialization
+from hopeit.dataframes.setup.dataframes import register_serialization
 from hopeit.dataobjects import DataObject
 
-__all__ = ["DataFrames", "dataframe", "dataframeobject"]
+__all__ = ["DataFrames", "Dataset", "dataframe"]
 
 
-class DataFrames(Generic[DataFrameT, DataFrameObjectT, DataObject]):
+class DataFrames(Generic[DataFrameT, DataObject]):
     """
     Dataframes manipulation utilities methods
     """
 
     @staticmethod
-    async def serialize(obj: DataFrameObjectT) -> DataObject:
-        """Serialize/saves contents of dataframe fields of a `@dataframeobject`
-        and converts to a `DataObject` json-compatible with pointers to saved
-        locations.
-
-        This method can be used to i.e. return `@dataframeobject`s as a JSON response
-        """
-        return await obj._serialize()  # type: ignore  # pylint: disable=protected-access
-
-    @staticmethod
-    async def deserialize(
-        datatype: Type[DataFrameObjectT], dataobject: DataObject
-    ) -> DataFrameObjectT:
-        """Deserialize/load contents of serialized dataobject fields of a `@dataframeobject`
-        loading saved Dataset information for @dataframe fields
-        """
-        return await datatype._deserialize(dataobject)  # type: ignore  # pylint: disable=protected-access
+    def setup(settings: DatasetSerialization):
+        register_serialization(settings)
 
     @staticmethod
     def from_df(
@@ -88,7 +112,7 @@ class DataFrames(Generic[DataFrameT, DataFrameObjectT, DataObject]):
 
     @staticmethod
     def from_dataobjects(
-        datatype: Type[DataFrameT], dataobjects: Iterator[DataFrameObjectT]
+        datatype: Type[DataFrameT], dataobjects: Iterator[DataObject]
     ) -> DataFrameT:
         """Converts standard json serializable `@dataobject`s to a single `@dataframe`"""
         return datatype._from_dataobjects(dataobjects)  # type: ignore  # pylint: disable=protected-access

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit/dataframes/dataframe.py

@@ -1,58 +1,49 @@
 """
 DataFrames type abstractions.
-
-Example:
-
-    from hopeit.dataobjects import dataclass # equivalent to `dataclasses.dataclass`
-    from hopeit.dataframes import dataframe
-
-    @dataframe
-    @dataclass
-    class MyObject:
-        name: str
-        number: int
 """
 
-from dataclasses import Field, asdict, dataclass, fields, make_dataclass
+import dataclasses
 from datetime import date, datetime, timezone
-from typing import Any, Callable, Dict, Generic, Iterator, List, Optional, Type, TypeVar
+from functools import partial
+from typing import Any, Callable, Dict, Generic, Iterator, List, Type, TypeVar
 
 import numpy as np
 import pandas as pd
-from dataclasses_jsonschema import JsonSchemaMixin
+from pydantic import create_model
+from pydantic.fields import FieldInfo
+
 from hopeit.dataobjects import (
     DataObject,
     StreamEventMixin,
     StreamEventParams,
     dataobject,
+    fields,
 )
+from hopeit.dataobjects.payload import Payload
 
 DataFrameT = TypeVar("DataFrameT")
 
 
-@dataclass
-class DataFrameMetadata(Generic[DataObject]):
+@dataclasses.dataclass
+class DataFrameMetadata:
     columns: List[str]
-    fields: Dict[str, Field]
-    serialized_type: Type[DataObject]
+    fields: Dict[str, FieldInfo]
 
 
-@dataclass
-class DataFrameParams:
-    """
-    Helper class used to access attributes in @dataframe
-    decorated objects, based on dot notation expressions
-    """
+# Functions to do type coercion
+def _series_to_int(x: pd.Series) -> pd.Series:
+    return x.astype(np.int64)
 
-    datatypes: Optional[str]
 
-    @staticmethod
-    def extract_attr(obj, expr):
-        value = obj
-        for attr_name in expr.split("."):
-            if value:
-                value = getattr(value, attr_name)
-        return value
+def _series_to_float(x: pd.Series) -> pd.Series:
+    return x.astype(np.float64)
+
+
+def _series_to_str(x: pd.Series) -> pd.Series:
+    return x.astype(str)
+
+
+_series_to_utc_datetime = partial(pd.to_datetime, utc=True)
 
 
 class DataFrameMixin(Generic[DataFrameT, DataObject]):
@@ -63,11 +54,11 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
     """
 
     DATATYPE_MAPPING = {
-        int: lambda x: x.astype(np.int64),
-        float: lambda x: x.astype(np.float64),
-        str: lambda x: x.astype(object),
+        int: _series_to_int,
+        float: _series_to_float,
+        str: _series_to_str,
         date: pd.to_datetime,
-        datetime: pd.to_datetime,
+        datetime: _series_to_utc_datetime,
     }
 
     def __init__(self) -> None:
@@ -78,9 +69,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         raise NotImplementedError  # must use @dataframe decorator  # pragma: no cover
 
     @staticmethod
-    def __init_from_series__(
-        self, **series: pd.Series
-    ):  # pylint: disable=bad-staticmethod-argument
+    def __init_from_series__(self, **series: pd.Series):  # pylint: disable=bad-staticmethod-argument
         df = pd.DataFrame(series)
         df.index.name = None  # Removes index name to avoid colisions with series name
         if self.__data_object__["validate"]:
@@ -99,7 +88,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
 
     @classmethod
     def _from_dataobjects(cls, items: Iterator[DataObject]) -> DataFrameT:
-        return cls._from_df(pd.DataFrame(asdict(item) for item in items))  # type: ignore
+        return cls._from_df(pd.DataFrame(Payload.to_obj(item) for item in items))  # type: ignore[misc]
 
     @classmethod
     def _from_df_unsafe(cls, df: pd.DataFrame, **series: pd.Series) -> DataFrameT:
@@ -116,40 +105,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         return self._from_df(self.__df[key])
 
     def _to_dataobjects(self) -> List[DataObject]:
-        return [
-            self.__dataframe__.serialized_type(**fields)
-            for fields in self.__df.to_dict(orient="records")
-        ]
-
-    def to_json(self, *args, **kwargs) -> str:
-        raise NotImplementedError(
-            "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
-        )
-
-    def to_dict(self, *args, **kwargs) -> Dict[str, Any]:
-        raise NotImplementedError(
-            "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
-        )
-
-    @classmethod
-    def from_json(cls, *args, **kwargs) -> DataObject:
-        return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
-
-    @classmethod
-    def from_dict(
-        cls,
-        *args,
-        **kwargs,
-    ) -> DataObject:
-        return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
-
-    @classmethod
-    def json_schema(cls, *args, **kwargs) -> Dict[str, Any]:
-        if cls.__data_object__["schema"]:
-            schema = cls.__dataframe__.serialized_type.json_schema(*args, **kwargs)
-            schema[cls.__name__] = schema[cls.__dataframe__.serialized_type.__name__]
-            return schema
-        return {}
+        return [self.DataObject(**fields) for fields in self.__df.to_dict(orient="records")]
 
     def event_id(self, *args, **kwargs) -> str:
         return ""
@@ -174,7 +130,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
 
     def _coerce_datatypes(self, df: pd.DataFrame) -> Dict[str, pd.Series]:
         return {
-            name: self.DATATYPE_MAPPING[field.type](df[name])  # type: ignore
+            name: self.DATATYPE_MAPPING[field.annotation](df[name])  # type: ignore
             for name, field in self.__dataframe__.fields.items()
         }
 
@@ -193,7 +149,7 @@ def dataframe(
         if hasattr(cls, "__annotations__") and hasattr(cls, "__dataclass_fields__"):
             amended_class = type(
                 cls.__name__,
-                (DataFrameMixin, JsonSchemaMixin) + cls.__mro__,
+                (DataFrameMixin,) + cls.__mro__,
                 dict(cls.__dict__),
             )
             setattr(amended_class, "__init__", DataFrameMixin.__init_from_series__)
@@ -201,17 +157,17 @@ def dataframe(
         return cls
 
     def add_dataframe_metadata(cls):
-        serialized_fiels = [(field.name, field.type) for field in fields(cls)]
-        serialized_type = make_dataclass(cls.__name__ + "_", serialized_fiels)
-        serialized_type = dataobject(serialized_type, unsafe=True)
+        serialized_fields = {k: (v.annotation, v) for k, v in fields(cls).items()}
+        dataobject_type = create_model(cls.__name__ + "DataObject", **serialized_fields)
+        dataobject_type = dataobject(dataobject_type, unsafe=True)
 
+        setattr(cls, "DataObject", dataobject_type)
         setattr(
             cls,
             "__dataframe__",
             DataFrameMetadata(
-                columns=[field.name for field in fields(cls)],
-                fields={field.name: field for field in fields(cls)},
-                serialized_type=serialized_type,
+                columns=list(fields(cls).keys()),
+                fields=dict(fields(cls).items()),
             ),
         )
 
@@ -226,14 +182,14 @@ def dataframe(
         setattr(cls, "event_ts", StreamEventMixin.event_ts)
 
     def set_fields_optional(cls):
-        for field in fields(cls):
+        for _, field in fields(cls).items():
             field.default = None
 
     def wrap(cls) -> Type[DataFrameMixin]:
         if hasattr(cls, "__dataframe__"):
             return cls
+        add_dataframe_metadata(cls)
         amended_class = add_dataframe_mixin(cls)
-        add_dataframe_metadata(amended_class)
         add_dataobject_annotations(amended_class, unsafe, validate, schema)
         set_fields_optional(amended_class)
         return amended_class

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit/dataframes/serialization/dataset.py

@@ -1,8 +1,7 @@
-"""Dataset objects definition, used as a result of serialized dataframes
-"""
+"""Dataset objects definition, used as a result of serialized dataframes"""
 
 from importlib import import_module
-from typing import Type, TypeVar
+from typing import Generic, Type, TypeVar
 
 from hopeit.dataobjects import dataclass, dataobject
 
@@ -11,12 +10,21 @@ DataFrameT = TypeVar("DataFrameT")
 
 @dataobject
 @dataclass
-class Dataset:
+class Dataset(Generic[DataFrameT]):
+    """Persisted representation of a @dataframe object"""
+
     protocol: str
     partition_key: str
     key: str
     datatype: str
 
+    async def load(self) -> DataFrameT:
+        return await self.__storage.load(self)  # type: ignore[attr-defined]
+
+    @classmethod
+    async def save(cls, dataframe: DataFrameT) -> "Dataset[DataFrameT]":
+        return await cls.__storage.save(dataframe)  # type: ignore[attr-defined]
+
 
 def find_protocol_impl(qual_type_name: str) -> Type:
     mod_name, type_name = (

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit/dataframes/serialization/files.py

@@ -1,9 +1,8 @@
-"""Support for `@dataframes` serialization to files
-"""
+"""Support for `@dataframes` serialization to files"""
 
 import io
 from importlib import import_module
-from typing import Callable, Generic, Optional, Type, TypeVar, Union
+from typing import Generic, Optional, Type, TypeVar
 from uuid import uuid4
 
 import pandas as pd
@@ -59,48 +58,12 @@ class DatasetFileStorage(Generic[DataFrameT]):
     async def load(self, dataset: Dataset) -> EventPayloadType:
         """Loads @dataframe annotated object using Dataset metadata"""
         datatype: Type[DataFrameT] = find_dataframe_type(dataset.datatype)
-        data = await self.storage.get_file(
-            dataset.key, partition_key=dataset.partition_key
-        )
+        data = await self.storage.get_file(dataset.key, partition_key=dataset.partition_key)
         if data is None:
             raise FileNotFoundError(dataset.key)
         df = pd.read_parquet(io.BytesIO(data), engine="pyarrow")
         return datatype._from_df(df)  # pylint: disable=protected-access
 
-    async def ser_wrapper(
-        self,
-        base_serialization: Callable,
-        data: Union[EventPayloadType, DataFrameT],
-        level: int,
-    ) -> bytes:
-        """Serialization wrapper that plugins-in into hopeit.engine
-        serialization when dataframes plugin is initialized
-        """
-        if hasattr(data, "__dataframeobject__"):
-            data = await data._serialize()  # type: ignore  # pylint: disable=protected-access
-        if hasattr(data, "__dataframe__"):
-            data = await self.save(data)  # type: ignore
-        return await base_serialization(data, level)
-
-    async def deser_wrapper(
-        self,
-        base_deserialization: Callable,
-        data: bytes,
-        datatype: Union[Type[EventPayloadType], Type[DataFrameT]],
-    ) -> Union[EventPayloadType, DataFrameT]:
-        """Deerialization wrapper that plugins-in into hopeit.engine
-        deserialization when dataframes plugin is initialized
-        """
-        if hasattr(datatype, "__dataframeobject__"):
-            dataset = await base_deserialization(
-                data, datatype.__dataframeobject__.serialized_type  # type: ignore
-            )
-            return await datatype._deserialize(dataset)  # type: ignore  # pylint: disable=protected-access
-        if hasattr(datatype, "__dataframe__"):
-            dataset = await base_deserialization(data, Dataset)
-            return await self.load(dataset)
-        return await base_deserialization(data, datatype)
-
 
 def find_dataframe_type(qual_type_name: str) -> Type[DataFrameT]:
     """Returns dataframe class based on type name used during serialization"""

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit/dataframes/serialization/settings.py

@@ -1,5 +1,4 @@
-"""Support for plugin configuration
-"""
+"""Support for plugin configuration"""
 
 from typing import Optional
 

hopeit_dataframes-0.25.0/src/hopeit/dataframes/setup/dataframes.py

@@ -0,0 +1,36 @@
+"""hopeit.engine dataframes plugin SETUP event.
+
+This event executes when engine starts with dataframes plugin configuration file loaded,
+and ensures that the engine will support serialization of `@dataframe` and `@dataframeobject`
+types
+"""
+
+from hopeit.app.context import EventContext
+from hopeit.app.logger import app_logger
+from hopeit.dataframes.serialization.dataset import Dataset, find_protocol_impl
+from hopeit.dataframes.serialization.settings import DatasetSerialization
+
+logger = app_logger()
+
+__steps__ = ["setup"]
+
+
+def setup(payload: None, context: EventContext) -> None:
+    """Setups serizaltion wrappers in hopeit.engine based on
+    `DataSerialization` settings configured in plugin configuration file
+    """
+    logger.info(context, "Configuring Dataset serialization...")
+    settings: DatasetSerialization = context.settings(
+        key="dataset_serialization", datatype=DatasetSerialization
+    )
+    register_serialization(settings)
+
+
+def register_serialization(settings: DatasetSerialization):
+    impl = find_protocol_impl(settings.protocol)
+    storage = impl(
+        protocol=settings.protocol,
+        location=settings.location,
+        partition_dateformat=settings.partition_dateformat,
+    )
+    setattr(Dataset, "_Dataset__storage", storage)

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.2
+Version: 0.25.0
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -13,7 +13,6 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -24,9 +23,9 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.8
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.2
+Requires-Dist: hopeit.engine[fs-storage]==0.25.0
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit.dataframes-0.24.2 → hopeit_dataframes-0.25.0}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -7,7 +7,6 @@ src/hopeit.dataframes.egg-info/requires.txt
 src/hopeit.dataframes.egg-info/top_level.txt
 src/hopeit/dataframes/__init__.py
 src/hopeit/dataframes/dataframe.py
-src/hopeit/dataframes/dataframeobject.py
 src/hopeit/dataframes/py.typed
 src/hopeit/dataframes/serialization/__init__.py
 src/hopeit/dataframes/serialization/dataset.py

hopeit_dataframes-0.25.0/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.25.0
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit.dataframes-0.24.2/src/hopeit/dataframes/dataframeobject.py

@@ -1,184 +0,0 @@
-"""
-`@dataframeobject` annonation mixin to serialize a group of `@dataframe`s.
-
-Datasets behaves as DataObject so they can be used as payload
-for endpoints and streams.
-"""
-
-from dataclasses import Field, dataclass, fields, make_dataclass
-from typing import (
-    Any,
-    Callable,
-    ClassVar,
-    Dict,
-    Generic,
-    Optional,
-    Type,
-    TypeVar,
-    Union,
-    get_args,
-    get_origin,
-)
-
-from hopeit.dataframes.serialization.dataset import Dataset
-from hopeit.dataobjects import (
-    DataObject,
-    StreamEventMixin,
-    StreamEventParams,
-    dataobject,
-)
-
-DataFrameObjectT = TypeVar("DataFrameObjectT")
-NoneType = type(None)
-
-
-@dataclass
-class DataFrameObjectMetadata(Generic[DataObject]):
-    serialized_type: Type[DataObject]
-
-
-class DataFrameObjectMixin(Generic[DataFrameObjectT]):
-    """
-    MixIn class to add functionality for `@dataframeobject`s
-
-    Do not use this class directly, instead use `@dataframeobject` class decorator.
-    """
-
-    __storage: ClassVar[Any] = None  # pylint: disable=invalid-name
-
-    def __init__(self) -> None:
-        self.__dataframeobject__: DataFrameObjectMetadata = None  # type: ignore
-        raise NotImplementedError(
-            "DataFrameObjectMixin() should not be called directly. Use `@dataframeobject` annotation"
-        )
-
-    async def _serialize(self) -> Optional[DataObject]:
-        """Saves internal `@dataframe`s using configured serialization protocol
-        and returns json-serialiable dataobject
-        """
-        datasets = {}
-        for field in fields(self):  # type: ignore
-            if _is_dataframe_field(field):
-                dataframe = getattr(self, field.name)
-                dataset = (
-                    None if dataframe is None else await self.__storage.save(dataframe)
-                )
-                datasets[field.name] = dataset
-            else:
-                datasets[field.name] = getattr(self, field.name)
-        return self.__dataframeobject__.serialized_type(**datasets)
-
-    @classmethod
-    async def _deserialize(
-        cls, serialized: DataObject
-    ) -> "DataFrameObjectMixin[DataFrameObjectT]":
-        """From a serialized datframeobject, load inner `@dataframe` objects
-        and returns a `@dataframeobject` instance"""
-        dataframes = {}
-        for field in fields(cls):  # type: ignore
-            if _is_dataframe_field(field):
-                dataset = getattr(serialized, field.name)
-                dataframe = (
-                    None if dataset is None else await cls.__storage.load(dataset)
-                )
-                dataframes[field.name] = dataframe
-            else:
-                dataframes[field.name] = getattr(serialized, field.name)
-        return cls(**dataframes)
-
-    @classmethod
-    def json_schema(cls, *args, **kwargs) -> Dict[str, Any]:
-        schema = cls.__dataframeobject__.serialized_type.json_schema(*args, **kwargs)
-        schema[cls.__name__] = schema[cls.__dataframeobject__.serialized_type.__name__]
-        return schema
-
-    def to_json(self, *args, **kwargs) -> Dict[str, Any]:
-        raise RuntimeError(
-            f"`{type(self).__name__}` `@dataframeobject` cannot be converted to json directly. "
-            "i.e. use `return await DataFrames.serialize(obj)` to return it as a reponse."
-        )
-
-
-def _is_dataframe_field(field: Field) -> bool:
-    return any(
-        hasattr(field_type, "__dataframe__")
-        for field_type in [field.type, *get_args(field.type)]
-    )
-
-
-def _serialized_field_type(field: Field) -> Type[Any]:
-    """Computes the `@dataobject` datatype used as a result
-    of serialized `@dataframeobject`
-    """
-    if hasattr(field.type, "__dataframe__"):
-        return Dataset
-    if get_origin(field.type) is Union:
-        args = get_args(field.type)
-        if (
-            len(args) == 2
-            and any(hasattr(field_type, "__dataframe__") for field_type in args)
-            and any(field_type is NoneType for field_type in args)
-        ):
-            return Optional[Dataset]  # type: ignore
-    if _is_dataframe_field(field):
-        raise TypeError(
-            f"field {field.name}: only `DataFrameT` or `Optional[DataFrameT]` are supported"
-        )
-    return field.type
-
-
-def dataframeobject(
-    decorated_class=None,
-) -> Callable[[Type], Type[DataFrameObjectMixin]]:
-    """
-    Decorator for dataclasses intended to be used as dataframes.
-    """
-
-    def add_dataframe_mixin(cls) -> Type[DataFrameObjectMixin]:
-        if hasattr(cls, "__annotations__") and hasattr(cls, "__dataclass_fields__"):
-            amended_class = type(
-                cls.__name__,
-                (DataFrameObjectMixin,) + cls.__mro__,
-                dict(cls.__dict__),
-            )
-            return amended_class
-        return cls
-
-    def add_dataframeobject_metadata(cls):
-        serialized_fiels = [
-            (field.name, _serialized_field_type(field)) for field in fields(cls)
-        ]
-        serialized_type = make_dataclass(cls.__name__ + "_", serialized_fiels)
-        serialized_type = dataobject(serialized_type, unsafe=True)
-
-        setattr(
-            cls,
-            "__dataframeobject__",
-            DataFrameObjectMetadata(
-                serialized_type=serialized_type,
-            ),
-        )
-
-    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool):
-        setattr(
-            cls,
-            "__data_object__",
-            {"unsafe": unsafe, "validate": validate, "schema": schema},
-        )
-        setattr(cls, "__stream_event__", StreamEventParams(None, None))
-        setattr(cls, "event_id", StreamEventMixin.event_id)
-        setattr(cls, "event_ts", StreamEventMixin.event_ts)
-
-    def wrap(cls) -> Type[DataFrameObjectMixin]:
-        if hasattr(cls, "__dataframeobject__"):
-            return cls
-        amended_class = add_dataframe_mixin(cls)
-        add_dataframeobject_metadata(amended_class)
-        add_dataobject_annotations(
-            amended_class, unsafe=False, validate=True, schema=True
-        )
-        return amended_class
-
-    if decorated_class is None:
-        return wrap
-    return wrap(decorated_class)  # type: ignore

hopeit.dataframes-0.24.2/src/hopeit/dataframes/setup/dataframes.py

@@ -1,52 +0,0 @@
-"""hopeit.engine dataframes plugin SETUP event.
-
-This event executes when engine starts with dataframes plugin configuration file loaded,
-and ensures that the engine will support serialization of `@dataframe` and `@dataframeobject`
-types
-"""
-
-from functools import partial
-
-from hopeit.app.context import EventContext
-from hopeit.app.logger import app_logger
-from hopeit.dataframes.dataframeobject import DataFrameObjectMixin
-from hopeit.dataframes.serialization.dataset import find_protocol_impl
-from hopeit.dataframes.serialization.settings import DatasetSerialization
-from hopeit.server import serialization
-
-logger = app_logger()
-
-__steps__ = ["register_serialization"]
-
-
-def register_serialization(payload: None, context: EventContext) -> None:
-    """Setups serizaltion wrappers in hopeit.engine based on
-    `DataSerialization` settings configured in plugin configuration file
-    """
-    logger.info(context, "Registering serialization methods...")
-
-    settings: DatasetSerialization = context.settings(
-        key="dataset_serialization", datatype=DatasetSerialization
-    )
-    impl = find_protocol_impl(settings.protocol)
-
-    storage = impl(
-        protocol=settings.protocol,
-        location=settings.location,
-        partition_dateformat=settings.partition_dateformat,
-    )
-    setattr(DataFrameObjectMixin, "_DataFrameObjectMixin__storage", storage)
-
-    serdeser_wrappers = {}
-    for (
-        serdeser,
-        methods,
-    ) in serialization._SERDESER.items():  # pylint: disable=protected-access
-        serdeser_wrappers[serdeser] = (
-            partial(storage.ser_wrapper, methods[0]),
-            methods[1],
-            partial(storage.deser_wrapper, methods[2]),
-        )
-
-    for serdeser, methods in serdeser_wrappers.items():
-        serialization._SERDESER[serdeser] = methods  # pylint: disable=protected-access

hopeit.dataframes-0.24.2/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.24.2
-pandas
-numpy
-
-[pyarrow]
-pyarrow