hopeit.dataframes 0.24.1__tar.gz → 0.25.0b1__tar.gz

{hopeit.dataframes-0.24.1 → hopeit_dataframes-0.25.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.1
+Version: 0.25.0b1
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.1
+Requires-Dist: hopeit.engine[fs-storage]==0.25.0b1
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit.dataframes-0.24.1 → hopeit_dataframes-0.25.0b1}/src/hopeit/dataframes/dataframe.py

@@ -12,32 +12,35 @@ Example:
         name: str
         number: int
 """
-
-from dataclasses import Field, asdict, dataclass, fields, make_dataclass
+import dataclasses
 from datetime import date, datetime, timezone
 from typing import Any, Callable, Dict, Generic, Iterator, List, Optional, Type, TypeVar
 
 import numpy as np
 import pandas as pd
-from dataclasses_jsonschema import JsonSchemaMixin
+from pydantic import create_model
+from pydantic.fields import FieldInfo
+
 from hopeit.dataobjects import (
     DataObject,
     StreamEventMixin,
     StreamEventParams,
     dataobject,
+    fields,
 )
+from hopeit.dataobjects.payload import Payload
 
 DataFrameT = TypeVar("DataFrameT")
 
 
-@dataclass
+@dataclasses.dataclass
 class DataFrameMetadata(Generic[DataObject]):
     columns: List[str]
-    fields: Dict[str, Field]
+    fields: Dict[str, FieldInfo]
     serialized_type: Type[DataObject]
 
 
-@dataclass
+@dataclasses.dataclass
 class DataFrameParams:
     """
     Helper class used to access attributes in @dataframe
@@ -81,9 +84,10 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
     def __init_from_series__(
         self, **series: pd.Series
     ):  # pylint: disable=bad-staticmethod-argument
-        if self.__data_object__["validate"]:
-            series = self._coerce_datatypes(series)
         df = pd.DataFrame(series)
+        df.index.name = None  # Removes index name to avoid colisions with series name
+        if self.__data_object__["validate"]:
+            df = pd.DataFrame(self._coerce_datatypes(df))
         setattr(self, "__df", df[self.__dataframe__.columns])
 
     @classmethod
@@ -98,7 +102,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
 
     @classmethod
     def _from_dataobjects(cls, items: Iterator[DataObject]) -> DataFrameT:
-        return cls._from_df(pd.DataFrame(asdict(item) for item in items))  # type: ignore
+        return cls._from_df(pd.DataFrame(Payload.to_obj(item) for item in items))  # type: ignore[misc]
 
     @classmethod
     def _from_df_unsafe(cls, df: pd.DataFrame, **series: pd.Series) -> DataFrameT:
@@ -120,35 +124,35 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
             for fields in self.__df.to_dict(orient="records")
         ]
 
-    def to_json(self, *args, **kwargs) -> str:
-        raise NotImplementedError(
-            "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
-        )
-
-    def to_dict(self, *args, **kwargs) -> Dict[str, Any]:
-        raise NotImplementedError(
-            "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
-        )
-
-    @classmethod
-    def from_json(cls, *args, **kwargs) -> DataObject:
-        return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
-
-    @classmethod
-    def from_dict(
-        cls,
-        *args,
-        **kwargs,
-    ) -> DataObject:
-        return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
-
-    @classmethod
-    def json_schema(cls, *args, **kwargs) -> Dict[str, Any]:
-        if cls.__data_object__["schema"]:
-            schema = cls.__dataframe__.serialized_type.json_schema(*args, **kwargs)
-            schema[cls.__name__] = schema[cls.__dataframe__.serialized_type.__name__]
-            return schema
-        return {}
+    # def to_json(self, *args, **kwargs) -> str:
+    #     raise NotImplementedError(
+    #         "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
+    #     )
+
+    # def to_dict(self, *args, **kwargs) -> Dict[str, Any]:
+    #     raise NotImplementedError(
+    #         "Dataframe must be used inside `@dataobject(unsafe=True)` to be used as an output"
+    #     )
+
+    # @classmethod
+    # def from_json(cls, *args, **kwargs) -> DataObject:
+    #     return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
+
+    # @classmethod
+    # def from_dict(
+    #     cls,
+    #     *args,
+    #     **kwargs,
+    # ) -> DataObject:
+    #     return cls.__dataframe__.serialized_type.from_dict(*args, **kwargs)
+
+    # @classmethod
+    # def json_schema(cls, *args, **kwargs) -> Dict[str, Any]:
+    #     if cls.__data_object__["schema"]:
+    #         schema = cls.__dataframe__.serialized_type.json_schema(*args, **kwargs)
+    #         schema[cls.__name__] = schema[cls.__dataframe__.serialized_type.__name__]
+    #         return schema
+    #     return {}
 
     def event_id(self, *args, **kwargs) -> str:
         return ""
@@ -171,9 +175,9 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         else:
             object.__setattr__(self, name, value)
 
-    def _coerce_datatypes(self, series: Dict[str, pd.Series]) -> Dict[str, pd.Series]:
+    def _coerce_datatypes(self, df: pd.DataFrame) -> Dict[str, pd.Series]:
         return {
-            name: self.DATATYPE_MAPPING[field.type](series[name])  # type: ignore
+            name: self.DATATYPE_MAPPING[field.annotation](df[name])  # type: ignore
             for name, field in self.__dataframe__.fields.items()
         }
 
@@ -192,7 +196,7 @@ def dataframe(
         if hasattr(cls, "__annotations__") and hasattr(cls, "__dataclass_fields__"):
             amended_class = type(
                 cls.__name__,
-                (DataFrameMixin, JsonSchemaMixin) + cls.__mro__,
+                (DataFrameMixin, ) + cls.__mro__,
                 dict(cls.__dict__),
             )
             setattr(amended_class, "__init__", DataFrameMixin.__init_from_series__)
@@ -200,16 +204,16 @@ def dataframe(
         return cls
 
     def add_dataframe_metadata(cls):
-        serialized_fiels = [(field.name, field.type) for field in fields(cls)]
-        serialized_type = make_dataclass(cls.__name__ + "_", serialized_fiels)
+        serialized_fields = {k: (v.annotation, v) for k, v in fields(cls).items()}
+        serialized_type = create_model(cls.__name__+"_", **serialized_fields)
         serialized_type = dataobject(serialized_type, unsafe=True)
 
         setattr(
             cls,
             "__dataframe__",
             DataFrameMetadata(
-                columns=[field.name for field in fields(cls)],
-                fields={field.name: field for field in fields(cls)},
+                columns=list(fields(cls).keys()),
+                fields=dict(fields(cls).items()),
                 serialized_type=serialized_type,
             ),
         )
@@ -225,14 +229,14 @@ def dataframe(
         setattr(cls, "event_ts", StreamEventMixin.event_ts)
 
     def set_fields_optional(cls):
-        for field in fields(cls):
+        for _, field in fields(cls).items():
             field.default = None
 
     def wrap(cls) -> Type[DataFrameMixin]:
         if hasattr(cls, "__dataframe__"):
             return cls
+        add_dataframe_metadata(cls)
         amended_class = add_dataframe_mixin(cls)
-        add_dataframe_metadata(amended_class)
         add_dataobject_annotations(amended_class, unsafe, validate, schema)
         set_fields_optional(amended_class)
         return amended_class

{hopeit.dataframes-0.24.1 → hopeit_dataframes-0.25.0b1}/src/hopeit/dataframes/dataframeobject.py

@@ -5,7 +5,7 @@ Datasets behaves as DataObject so they can be used as payload
 for endpoints and streams.
 """
 
-from dataclasses import Field, dataclass, fields, make_dataclass
+import dataclasses
 from typing import (
     Any,
     Callable,
@@ -20,19 +20,23 @@ from typing import (
     get_origin,
 )
 
+from pydantic import TypeAdapter, create_model
+from pydantic.fields import FieldInfo
+
 from hopeit.dataframes.serialization.dataset import Dataset
 from hopeit.dataobjects import (
     DataObject,
     StreamEventMixin,
     StreamEventParams,
     dataobject,
+    fields,
 )
 
 DataFrameObjectT = TypeVar("DataFrameObjectT")
 NoneType = type(None)
 
 
-@dataclass
+@dataclasses.dataclass
 class DataFrameObjectMetadata(Generic[DataObject]):
     serialized_type: Type[DataObject]
 
@@ -57,15 +61,15 @@ class DataFrameObjectMixin(Generic[DataFrameObjectT]):
         and returns json-serialiable dataobject
         """
         datasets = {}
-        for field in fields(self):  # type: ignore
-            if _is_dataframe_field(field):
-                dataframe = getattr(self, field.name)
+        for field_name, field in fields(self).items():  # type: ignore[arg-type]
+            if Dataset in {field.annotation, *get_args(field.annotation)}:
+                dataframe = getattr(self, field_name)
                 dataset = (
                     None if dataframe is None else await self.__storage.save(dataframe)
                 )
-                datasets[field.name] = dataset
+                datasets[field_name] = dataset
             else:
-                datasets[field.name] = getattr(self, field.name)
+                datasets[field_name] = getattr(self, field_name)
         return self.__dataframeobject__.serialized_type(**datasets)
 
     @classmethod
@@ -75,45 +79,44 @@ class DataFrameObjectMixin(Generic[DataFrameObjectT]):
         """From a serialized datframeobject, load inner `@dataframe` objects
         and returns a `@dataframeobject` instance"""
         dataframes = {}
-        for field in fields(cls):  # type: ignore
-            if _is_dataframe_field(field):
-                dataset = getattr(serialized, field.name)
+        for field_name, field in fields(cls).items():  # type: ignore[type-var]
+            if Dataset in {field.annotation, *get_args(field.annotation)}:
+                dataset = getattr(serialized, field_name)
                 dataframe = (
                     None if dataset is None else await cls.__storage.load(dataset)
                 )
-                dataframes[field.name] = dataframe
+                dataframes[field_name] = dataframe
             else:
-                dataframes[field.name] = getattr(serialized, field.name)
+                dataframes[field_name] = getattr(serialized, field_name)
         return cls(**dataframes)
 
     @classmethod
     def json_schema(cls, *args, **kwargs) -> Dict[str, Any]:
-        schema = cls.__dataframeobject__.serialized_type.json_schema(*args, **kwargs)
-        schema[cls.__name__] = schema[cls.__dataframeobject__.serialized_type.__name__]
+        schema = TypeAdapter(cls.__dataframeobject__.serialized_type).json_schema(*args, **kwargs)
         return schema
 
-    def to_json(self, *args, **kwargs) -> Dict[str, Any]:
-        raise RuntimeError(
-            f"`{type(self).__name__}` `@dataframeobject` cannot be converted to json directly. "
-            "i.e. use `return await DataFrames.serialize(obj)` to return it as a reponse."
-        )
+    # def to_json(self, *args, **kwargs) -> Dict[str, Any]:
+    #     raise RuntimeError(
+    #         f"`{type(self).__name__}` `@dataframeobject` cannot be converted to json directly. "
+    #         "i.e. use `return await DataFrames.serialize(obj)` to return it as a response."
+    #     )
 
 
-def _is_dataframe_field(field: Field) -> bool:
+def _is_dataframe_field(field: FieldInfo) -> bool:
     return any(
         hasattr(field_type, "__dataframe__")
-        for field_type in [field.type, *get_args(field.type)]
+        for field_type in [field.annotation, *get_args(field.annotation)]
     )
 
 
-def _serialized_field_type(field: Field) -> Type[Any]:
+def _serialized_field_type(field_name: str, field: FieldInfo) -> Optional[Type[Any]]:
     """Computes the `@dataobject` datatype used as a result
     of serialized `@dataframeobject`
     """
-    if hasattr(field.type, "__dataframe__"):
+    if hasattr(field.annotation, "__dataframe__"):
         return Dataset
-    if get_origin(field.type) is Union:
-        args = get_args(field.type)
+    if get_origin(field.annotation) is Union:
+        args = get_args(field.annotation)
         if (
             len(args) == 2
             and any(hasattr(field_type, "__dataframe__") for field_type in args)
@@ -122,9 +125,9 @@ def _serialized_field_type(field: Field) -> Type[Any]:
             return Optional[Dataset]  # type: ignore
     if _is_dataframe_field(field):
         raise TypeError(
-            f"field {field.name}: only `DataFrameT` or `Optional[DataFrameT]` are supported"
+            f"field {field_name}: only `DataFrameT` or `Optional[DataFrameT]` are supported"
         )
-    return field.type
+    return field.annotation
 
 
 def dataframeobject(
@@ -145,12 +148,12 @@ def dataframeobject(
         return cls
 
     def add_dataframeobject_metadata(cls):
-        serialized_fiels = [
-            (field.name, _serialized_field_type(field)) for field in fields(cls)
-        ]
-        serialized_type = make_dataclass(cls.__name__ + "_", serialized_fiels)
+        serialized_fields = {
+            field_name: (_serialized_field_type(field_name, field_info), field_info)
+            for field_name, field_info in fields(cls).items()
+        }
+        serialized_type = create_model(cls.__name__+"_", **serialized_fields)
         serialized_type = dataobject(serialized_type, unsafe=True)
-
         setattr(
             cls,
             "__dataframeobject__",
@@ -159,11 +162,11 @@ def dataframeobject(
             ),
         )
 
-    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool):
+    def add_dataobject_annotations(cls, unsafe: bool, schema: bool):
         setattr(
             cls,
             "__data_object__",
-            {"unsafe": unsafe, "validate": validate, "schema": schema},
+            {"unsafe": unsafe, "schema": schema},
         )
         setattr(cls, "__stream_event__", StreamEventParams(None, None))
         setattr(cls, "event_id", StreamEventMixin.event_id)
@@ -172,10 +175,10 @@ def dataframeobject(
     def wrap(cls) -> Type[DataFrameObjectMixin]:
         if hasattr(cls, "__dataframeobject__"):
             return cls
+        add_dataframeobject_metadata(cls)
         amended_class = add_dataframe_mixin(cls)
-        add_dataframeobject_metadata(amended_class)
         add_dataobject_annotations(
-            amended_class, unsafe=False, validate=True, schema=True
+            amended_class, unsafe=False, schema=True
         )
         return amended_class
 

{hopeit.dataframes-0.24.1 → hopeit_dataframes-0.25.0b1}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.24.1
+Version: 0.25.0b1
 Summary: Hopeit Engine Dataframes Toolkit
 Home-page: https://github.com/hopeit-git/hopeit.engine
 Author: Leo Smerling and Pablo Canto
@@ -26,7 +26,7 @@ Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.24.1
+Requires-Dist: hopeit.engine[fs-storage]==0.25.0b1
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

hopeit_dataframes-0.25.0b1/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.25.0b1
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit.dataframes-0.24.1/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.24.1
-pandas
-numpy
-
-[pyarrow]
-pyarrow