hopeit.dataframes 0.25.4__tar.gz → 0.26.0rc1__tar.gz

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: hopeit.dataframes
-Version: 0.25.4
-Summary: Hopeit Engine Dataframes Toolkit
-Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
+Version: 0.26.0rc1
+Summary: Hopeit Engine Dataframes for Pandas
+Author-email: Leo Smerling & Pablo Canto <contact@hopeit.com.ar>, Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
 Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
@@ -12,23 +12,22 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.9
-Description-Content-Type: text/plain
-Requires-Dist: hopeit.engine[fs-storage]==0.25.4
-Requires-Dist: pandas
-Requires-Dist: numpy
-Provides-Extra: pyarrow
-Requires-Dist: pyarrow; extra == "pyarrow"
+Description-Content-Type: text/markdown
+Requires-Dist: hopeit.engine>=0.26.0rc1
+Requires-Dist: hopeit.fs-storage>=0.26.0rc1
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: numpy>=1.26.4
 
 # hopeit.engine dataframes plugin
 

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/pyproject.toml

@@ -4,10 +4,21 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "hopeit.dataframes"
-description = "Hopeit Engine Dataframes Toolkit"
-dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
+version = "0.26.0rc1"
+
+description = "Hopeit Engine Dataframes for Pandas"
+dynamic = ["readme"]
+
+dependencies = [
+    "hopeit.engine>=0.26.0rc1",
+    "hopeit.fs-storage>=0.26.0rc1",
+    "pandas>=2.2.3",
+    "numpy>=1.26.4"
+]
+
 license = { text = "Apache 2" }
 authors = [
+    { name = "Leo Smerling & Pablo Canto", email = "contact@hopeit.com.ar" },
     { name = "Leo Smerling", email = "contact@hopeit.com.ar" },
     { name = "Pablo Canto", email = "contact@hopeit.com.ar" },
 ]
@@ -15,10 +26,11 @@ classifiers = [
     "License :: OSI Approved :: Apache Software License",
     "Intended Audience :: Developers",
     "Programming Language :: Python",
-    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
-    "Development Status :: 4 - Beta",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Development Status :: 5 - Production/Stable",
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS :: MacOS X",
     "Operating System :: Microsoft :: Windows",
@@ -26,8 +38,6 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
     "Framework :: AsyncIO",
 ]
-requires-python = ">=3.9"
-
 
 [project.urls]
 "Homepage" = "https://github.com/hopeit-git/hopeit.engine"
@@ -42,10 +52,5 @@ include-package-data = true
 [tool.setuptools.packages.find]
 where = ["src"]
 
-[tool.setuptools.package-data]
-"hopeit.dataframes" = ["py.typed"]
-"hopeit.dataframes.serialization" = ["py.typed"]
-"hopeit.dataframes.setup" = ["py.typed"]
-
 [tool.setuptools.dynamic]
-readme = { file = ["README.md"], content-type = "text/plain" }
+readme = { file = ["README.md"], content-type = "text/markdown" }

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/__init__.py

@@ -92,7 +92,7 @@ class DataFrames(Generic[DataFrameT, DataObject]):
     """
 
     @staticmethod
-    def setup(settings: DatasetSerialization):
+    def setup(settings: DatasetSerialization) -> None:
         register_serialization(settings)
 
     @staticmethod

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/datablocks.py

@@ -11,7 +11,7 @@ DataFrameType = TypeVar("DataFrameType")
 
 
 class TempDataBlock(Generic[DataBlockType, DataBlockItemType]):
-    def __init__(self, datatype: Type[DataBlockType], df: pd.DataFrame):
+    def __init__(self, datatype: Type[DataBlockType], df: pd.DataFrame) -> None:
         self.datatype = datatype
         self.df = df
 
@@ -132,7 +132,7 @@ class DataBlocks(Generic[DataBlockType, DataFrameType]):
             ) from e
 
     @classmethod
-    def _adapt_to_schema(cls, datablock: DataBlockType, keys: list[str], df: pd.DataFrame):
+    def _adapt_to_schema(cls, datablock: DataBlockType, keys: list[str], df: pd.DataFrame) -> None:
         for key in keys:
             datatype = find_dataframe_type(getattr(datablock, key).datatype)  # type: ignore[var-annotated]
             valid_df = datatype._from_df(df)._df

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/dataframe.py

@@ -124,7 +124,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         raise NotImplementedError  # must use @dataframe decorator  # pragma: no cover
 
     @staticmethod
-    def __init_from_series__(self, **series: pd.Series):  # pylint: disable=bad-staticmethod-argument
+    def __init_from_series__(self, **series: pd.Series) -> None:  # pylint: disable=bad-staticmethod-argument
         df = pd.DataFrame(series)
         df.index.name = None  # Removes index name to avoid colisions with series name
         if self.__data_object__["validate"]:
@@ -240,9 +240,10 @@ def dataframe(
             return amended_class
         return cls
 
-    def add_dataframe_metadata(cls):
+    def add_dataframe_metadata(cls) -> None:
         serialized_fields = {k: (v.annotation, v) for k, v in fields(cls).items()}
-        dataobject_type = create_model(cls.__name__ + "DataObject", **serialized_fields)
+        dataobject_name = str(cls.__name__) + "DataObject"
+        dataobject_type = create_model(dataobject_name, **serialized_fields)  # type: ignore[call-overload]
         dataobject_type = dataobject(dataobject_type, unsafe=True)
 
         setattr(cls, "DataObject", dataobject_type)
@@ -255,7 +256,7 @@ def dataframe(
             ),
         )
 
-    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool):
+    def add_dataobject_annotations(cls, unsafe: bool, validate: bool, schema: bool) -> None:
         setattr(
             cls,
             "__data_object__",

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/serialization/dataset.py

@@ -87,7 +87,7 @@ def find_dataframe_type(qual_type_name: str) -> Type[DataFrameT]:
     )
     module = import_module(mod_name)
     datatype = getattr(module, type_name)
-    assert hasattr(
-        datatype, "__dataframe__"
-    ), f"Type {qual_type_name} must be annotated with `@dataframe`."
+    assert hasattr(datatype, "__dataframe__"), (
+        f"Type {qual_type_name} must be annotated with `@dataframe`."
+    )
     return datatype

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/serialization/files.py

@@ -1,7 +1,7 @@
 """Support for `@dataframes` serialization to files"""
 
 import io
-from typing import Generic, Optional, Type, TypeVar
+from typing import Generic, Literal, Optional, Type, TypeVar
 from uuid import uuid4
 
 import pandas as pd
@@ -17,22 +17,42 @@ except ImportError as e:
 
 from hopeit.dataframes.dataframe import DataFrameMixin
 from hopeit.dataframes.serialization.dataset import Dataset
-from hopeit.dataobjects import DataObject
+from hopeit.dataobjects import DataObject, dataclass, dataobject
+from hopeit.dataobjects.payload import Payload
 from hopeit.fs_storage import FileStorage
 
 DataFrameT = TypeVar("DataFrameT", bound=DataFrameMixin)
 
 
+@dataobject
+@dataclass
+class DatasetFileStorageEngineSettings:
+    """Pyarrow settings for parquet file storage"""
+
+    compression: Literal["snappy", "gzip", "brotli", "lz4", "zstd"] | None = None
+    compression_level: int | str | None = None
+
+
 class DatasetFileStorage(Generic[DataFrameT]):
     """Support to store dataframes as files,
     using pandas parquet format support in combination
     with `hopeit.engine` file storage plugins
     """
 
-    def __init__(self, *, location: str, partition_dateformat: Optional[str], **kwargs):
+    def __init__(
+        self,
+        *,
+        location: str,
+        partition_dateformat: Optional[str],
+        storage_settings: dict[str, str | int],
+        **kwargs,
+    ):
         self.storage: FileStorage = FileStorage(
             path=location, partition_dateformat=partition_dateformat
         )
+        self.storage_settings: DatasetFileStorageEngineSettings = Payload.from_obj(
+            storage_settings, DatasetFileStorageEngineSettings
+        )
 
     async def save(self, dataframe: DataFrameT) -> Dataset:
         """Saves @dataframe annotated object as parquet to file system
@@ -42,7 +62,9 @@ class DatasetFileStorage(Generic[DataFrameT]):
         key = f"{datatype.__qualname__.lower()}_{uuid4()}.parquet"
         data = io.BytesIO(
             dataframe._df.to_parquet(  # pylint: disable=protected-access
-                engine="pyarrow"
+                engine="pyarrow",
+                compression=self.storage_settings.compression,
+                compression_level=self.storage_settings.compression_level,
             )
         )
         location = await self.storage.store_file(file_name=key, value=data)

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/serialization/settings.py

@@ -2,7 +2,7 @@
 
 from typing import Optional
 
-from hopeit.dataobjects import dataclass, dataobject
+from hopeit.dataobjects import dataclass, dataobject, field
 
 
 @dataobject
@@ -11,3 +11,4 @@ class DatasetSerialization:
     protocol: str
     location: str
     partition_dateformat: Optional[str] = None
+    storage_settings: dict[str, str | int] = field(default_factory=dict)

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit/dataframes/setup/dataframes.py

@@ -26,11 +26,12 @@ def setup(payload: None, context: EventContext) -> None:
     register_serialization(settings)
 
 
-def register_serialization(settings: DatasetSerialization):
+def register_serialization(settings: DatasetSerialization) -> None:
     impl = find_protocol_impl(settings.protocol)
     storage = impl(
         protocol=settings.protocol,
         location=settings.location,
         partition_dateformat=settings.partition_dateformat,
+        storage_settings=settings.storage_settings,
     )
     setattr(Dataset, "_Dataset__storage", storage)

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,8 +1,8 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: hopeit.dataframes
-Version: 0.25.4
-Summary: Hopeit Engine Dataframes Toolkit
-Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
+Version: 0.26.0rc1
+Summary: Hopeit Engine Dataframes for Pandas
+Author-email: Leo Smerling & Pablo Canto <contact@hopeit.com.ar>, Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
 Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
@@ -12,23 +12,22 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
-Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Development Status :: 5 - Production/Stable
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Operating System :: MacOS :: MacOS X
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.9
-Description-Content-Type: text/plain
-Requires-Dist: hopeit.engine[fs-storage]==0.25.4
-Requires-Dist: pandas
-Requires-Dist: numpy
-Provides-Extra: pyarrow
-Requires-Dist: pyarrow; extra == "pyarrow"
+Description-Content-Type: text/markdown
+Requires-Dist: hopeit.engine>=0.26.0rc1
+Requires-Dist: hopeit.fs-storage>=0.26.0rc1
+Requires-Dist: pandas>=2.2.3
+Requires-Dist: numpy>=1.26.4
 
 # hopeit.engine dataframes plugin
 

{hopeit_dataframes-0.25.4 → hopeit_dataframes-0.26.0rc1}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -1,6 +1,5 @@
 README.md
 pyproject.toml
-setup.py
 src/hopeit.dataframes.egg-info/PKG-INFO
 src/hopeit.dataframes.egg-info/SOURCES.txt
 src/hopeit.dataframes.egg-info/dependency_links.txt

hopeit_dataframes-0.26.0rc1/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,4 @@
+hopeit.engine>=0.26.0rc1
+hopeit.fs-storage>=0.26.0rc1
+pandas>=2.2.3
+numpy>=1.26.4

hopeit_dataframes-0.25.4/setup.py

@@ -1,25 +0,0 @@
-from setuptools import setup
-from os import environ
-
-version = {}
-try:
-    with open("../../../engine/src/hopeit/server/version.py") as fp:
-        exec(fp.read(), version)
-        ENGINE_VERSION = version["ENGINE_VERSION"]
-except FileNotFoundError:
-    ENGINE_VERSION = environ.get("ENGINE_VERSION")
-
-if not ENGINE_VERSION:
-    raise RuntimeError("ENGINE_VERSION is not specified.")
-
-setup(
-    version=ENGINE_VERSION,
-    install_requires=[
-        f"hopeit.engine[fs-storage]=={ENGINE_VERSION}",
-        "pandas",
-        "numpy",
-    ],
-    extras_require={
-        "pyarrow": ["pyarrow"],
-    },
-)

hopeit_dataframes-0.25.4/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.25.4
-pandas
-numpy
-
-[pyarrow]
-pyarrow