hopeit.dataframes 0.26.0rc0__tar.gz → 0.26.0rc1__tar.gz

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: hopeit.dataframes
-Version: 0.26.0rc0
+Version: 0.26.0rc1
 Summary: Hopeit Engine Dataframes for Pandas
 Author-email: Leo Smerling & Pablo Canto <contact@hopeit.com.ar>, Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
@@ -24,8 +24,8 @@ Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine>=0.26.0rc0
-Requires-Dist: hopeit.fs-storage>=0.26.0rc0
+Requires-Dist: hopeit.engine>=0.26.0rc1
+Requires-Dist: hopeit.fs-storage>=0.26.0rc1
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: numpy>=1.26.4
 

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/pyproject.toml

@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hopeit.dataframes"
-version = "0.26.0rc0"
+version = "0.26.0rc1"
 
 description = "Hopeit Engine Dataframes for Pandas"
 dynamic = ["readme"]
 
 dependencies = [
-    "hopeit.engine>=0.26.0rc0",
-    "hopeit.fs-storage>=0.26.0rc0",
+    "hopeit.engine>=0.26.0rc1",
+    "hopeit.fs-storage>=0.26.0rc1",
     "pandas>=2.2.3",
     "numpy>=1.26.4"
 ]

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/__init__.py

@@ -92,7 +92,7 @@ class DataFrames(Generic[DataFrameT, DataObject]):
     """
 
     @staticmethod
-    def setup(settings: DatasetSerialization):
+    def setup(settings: DatasetSerialization) -> None:
         register_serialization(settings)
 
     @staticmethod

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/datablocks.py

@@ -11,7 +11,7 @@ DataFrameType = TypeVar("DataFrameType")
 
 
 class TempDataBlock(Generic[DataBlockType, DataBlockItemType]):
-    def __init__(self, datatype: Type[DataBlockType], df: pd.DataFrame):
+    def __init__(self, datatype: Type[DataBlockType], df: pd.DataFrame) -> None:
         self.datatype = datatype
         self.df = df
 
@@ -132,7 +132,7 @@ class DataBlocks(Generic[DataBlockType, DataFrameType]):
             ) from e
 
     @classmethod
-    def _adapt_to_schema(cls, datablock: DataBlockType, keys: list[str], df: pd.DataFrame):
+    def _adapt_to_schema(cls, datablock: DataBlockType, keys: list[str], df: pd.DataFrame) -> None:
         for key in keys:
             datatype = find_dataframe_type(getattr(datablock, key).datatype)  # type: ignore[var-annotated]
             valid_df = datatype._from_df(df)._df

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/dataframe.py

@@ -124,7 +124,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         raise NotImplementedError  # must use @dataframe decorator  # pragma: no cover
 
     @staticmethod
-    def __init_from_series__(self, **series: pd.Series):  # pylint: disable=bad-staticmethod-argument
+    def __init_from_series__(self, **series: pd.Series) -> None:  # pylint: disable=bad-staticmethod-argument
         df = pd.DataFrame(series)
         df.index.name = None  # Removes index name to avoid colisions with series name
         if self.__data_object__["validate"]:
@@ -240,9 +240,10 @@ def dataframe(
             return amended_class
         return cls
 
-    def add_dataframe_metadata(cls):
+    def add_dataframe_metadata(cls) -> None:
         serialized_fields = {k: (v.annotation, v) for k, v in fields(cls).items()}
-        dataobject_type = create_model(cls.__name__ + "DataObject", **serialized_fields)
+        dataobject_name = str(cls.__name__) + "DataObject"
+        dataobject_type = create_model(dataobject_name, **serialized_fields)  # type: ignore[call-overload]
         dataobject_type = dataobject(dataobject_type, unsafe=True)
 
         setattr(cls, "DataObject", dataobject_type)
@@ -255,7 +256,7 @@ def dataframe(
             ),
         )
 
-    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool):
+    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool) -> None:
         setattr(
             cls,
             "__data_object__",

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/serialization/files.py

@@ -1,7 +1,7 @@
 """Support for `@dataframes` serialization to files"""
 
 import io
-from typing import Generic, Optional, Type, TypeVar
+from typing import Generic, Literal, Optional, Type, TypeVar
 from uuid import uuid4
 
 import pandas as pd
@@ -17,22 +17,42 @@ except ImportError as e:
 
 from hopeit.dataframes.dataframe import DataFrameMixin
 from hopeit.dataframes.serialization.dataset import Dataset
-from hopeit.dataobjects import DataObject
+from hopeit.dataobjects import DataObject, dataclass, dataobject
+from hopeit.dataobjects.payload import Payload
 from hopeit.fs_storage import FileStorage
 
 DataFrameT = TypeVar("DataFrameT", bound=DataFrameMixin)
 
 
+@dataobject
+@dataclass
+class DatasetFileStorageEngineSettings:
+    """Pyarrow settings for parquet file storage"""
+
+    compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None = None
+    compression_level: int | str | None = None
+
+
 class DatasetFileStorage(Generic[DataFrameT]):
     """Support to store dataframes as files,
     using pandas parquet format support in combination
     with `hopeit.engine` file storage plugins
     """
 
-    def __init__(self, *, location: str, partition_dateformat: Optional[str], **kwargs):
+    def __init__(
+        self,
+        *,
+        location: str,
+        partition_dateformat: Optional[str],
+        storage_settings: dict[str, str | int],
+        **kwargs,
+    ):
         self.storage: FileStorage = FileStorage(
             path=location, partition_dateformat=partition_dateformat
         )
+        self.storage_settings: DatasetFileStorageEngineSettings = Payload.from_obj(
+            storage_settings, DatasetFileStorageEngineSettings
+        )
 
     async def save(self, dataframe: DataFrameT) -> Dataset:
         """Saves @dataframe annotated object as parquet to file system
@@ -42,7 +62,9 @@ class DatasetFileStorage(Generic[DataFrameT]):
         key = f"{datatype.__qualname__.lower()}_{uuid4()}.parquet"
         data = io.BytesIO(
             dataframe._df.to_parquet(  # pylint: disable=protected-access
-                engine="pyarrow"
+                engine="pyarrow",
+                compression=self.storage_settings.compression,
+                compression_level=self.storage_settings.compression_level,
             )
         )
         location = await self.storage.store_file(file_name=key, value=data)

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/serialization/settings.py

@@ -2,7 +2,7 @@
 
 from typing import Optional
 
-from hopeit.dataobjects import dataclass, dataobject
+from hopeit.dataobjects import dataclass, dataobject, field
 
 
 @dataobject
@@ -11,3 +11,4 @@ class DatasetSerialization:
     protocol: str
     location: str
     partition_dateformat: Optional[str] = None
+    storage_settings: dict[str, str | int] = field(default_factory=dict)

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/setup/dataframes.py

@@ -26,11 +26,12 @@ def setup(payload: None, context: EventContext) -> None:
     register_serialization(settings)
 
 
-def register_serialization(settings: DatasetSerialization):
+def register_serialization(settings: DatasetSerialization) -> None:
     impl = find_protocol_impl(settings.protocol)
     storage = impl(
         protocol=settings.protocol,
         location=settings.location,
         partition_dateformat=settings.partition_dateformat,
+        storage_settings=settings.storage_settings,
     )
     setattr(Dataset, "_Dataset__storage", storage)

{hopeit_dataframes-0.26.0rc0 → hopeit_dataframes-0.26.0rc1}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: hopeit.dataframes
-Version: 0.26.0rc0
+Version: 0.26.0rc1
 Summary: Hopeit Engine Dataframes for Pandas
 Author-email: Leo Smerling & Pablo Canto <contact@hopeit.com.ar>, Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
@@ -24,8 +24,8 @@ Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine>=0.26.0rc0
-Requires-Dist: hopeit.fs-storage>=0.26.0rc0
+Requires-Dist: hopeit.engine>=0.26.0rc1
+Requires-Dist: hopeit.fs-storage>=0.26.0rc1
 Requires-Dist: pandas>=2.2.3
 Requires-Dist: numpy>=1.26.4
 

hopeit_dataframes-0.26.0rc1/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,4 @@
+hopeit.engine>=0.26.0rc1
+hopeit.fs-storage>=0.26.0rc1
+pandas>=2.2.3
+numpy>=1.26.4

hopeit_dataframes-0.26.0rc0/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,4 +0,0 @@
-hopeit.engine>=0.26.0rc0
-hopeit.fs-storage>=0.26.0rc0
-pandas>=2.2.3
-numpy>=1.26.4