hopeit.dataframes 0.25.0b7__tar.gz → 0.25.1__tar.gz

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.0b7
+Version: 0.25.1
 Summary: Hopeit Engine Dataframes Toolkit
-Home-page: https://github.com/hopeit-git/hopeit.engine
-Author: Leo Smerling and Pablo Canto
-Author-email: contact@hopeit.com.ar
+Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
+Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
 Project-URL: Docs: RTD, https://hopeitengine.readthedocs.io/en/latest/
 Project-URL: GitHub: issues, https://github.com/hopeit-git/hopeit.engine/issues
@@ -13,7 +12,6 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -24,9 +22,9 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.25.0b7
+Requires-Python: >=3.9
+Description-Content-Type: text/plain
+Requires-Dist: hopeit.engine[fs-storage]==0.25.1
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

hopeit_dataframes-0.25.1/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools >= 64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hopeit.dataframes"
+description = "Hopeit Engine Dataframes Toolkit"
+dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
+license = { text = "Apache 2" }
+authors = [
+    { name = "Leo Smerling", email = "contact@hopeit.com.ar" },
+    { name = "Pablo Canto", email = "contact@hopeit.com.ar" },
+]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Development Status :: 4 - Beta",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Framework :: AsyncIO",
+]
+requires-python = ">=3.9"
+
+
+[project.urls]
+"Homepage" = "https://github.com/hopeit-git/hopeit.engine"
+"CI: GitHub Actions" = "https://github.com/hopeit-git/hopeit.engine/actions?query=workflow"
+"Docs: RTD" = "https://hopeitengine.readthedocs.io/en/latest/"
+"GitHub: issues" = "https://github.com/hopeit-git/hopeit.engine/issues"
+"GitHub: repo" = "https://github.com/hopeit-git/hopeit.engine"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"hopeit.dataframes" = ["py.typed"]
+"hopeit.dataframes.serialization" = ["py.typed"]
+"hopeit.dataframes.setup" = ["py.typed"]
+
+[tool.setuptools.dynamic]
+readme = { file = ["README.md"], content-type = "text/plain" }

hopeit_dataframes-0.25.1/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup
+from os import environ
+
+version = {}
+try:
+    with open("../../../engine/src/hopeit/server/version.py") as fp:
+        exec(fp.read(), version)
+        ENGINE_VERSION = version["ENGINE_VERSION"]
+except FileNotFoundError:
+    ENGINE_VERSION = environ.get("ENGINE_VERSION")
+
+if not ENGINE_VERSION:
+    raise RuntimeError("ENGINE_VERSION is not specified.")
+
+setup(
+    version=ENGINE_VERSION,
+    install_requires=[
+        f"hopeit.engine[fs-storage]=={ENGINE_VERSION}",
+        "pandas",
+        "numpy",
+    ],
+    extras_require={
+        "pyarrow": ["pyarrow"],
+    },
+)

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit/dataframes/dataframe.py

@@ -1,10 +1,10 @@
 """
 DataFrames type abstractions.
 """
+
 import dataclasses
 from datetime import date, datetime, timezone
-from functools import partial
-from typing import Any, Callable, Dict, Generic, Iterator, List, Type, TypeVar
+from typing import Any, Callable, Dict, Generic, Iterator, List, Type, TypeVar, Union
 
 import numpy as np
 import pandas as pd
@@ -19,30 +19,67 @@ from hopeit.dataobjects import (
     fields,
 )
 from hopeit.dataobjects.payload import Payload
+from pydantic_core import PydanticUndefined
 
 DataFrameT = TypeVar("DataFrameT")
 
 
 @dataclasses.dataclass
-class DataFrameMetadata():
+class DataFrameMetadata:
     columns: List[str]
     fields: Dict[str, FieldInfo]
 
 
 # Functions to do type coercion
-def _series_to_int(x: pd.Series) -> pd.Series:
+def _series_to_int(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(np.int64)
 
 
-def _series_to_float(x: pd.Series) -> pd.Series:
+def _series_to_float(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(np.float64)
 
 
-def _series_to_str(x: pd.Series) -> pd.Series:
+def _series_to_str(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(str)
 
 
-_series_to_utc_datetime = partial(pd.to_datetime, utc=True)
+# Functions to do type coercion
+def _series_to_int_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x[x.notna()].astype(np.int64)
+
+
+def _series_to_float_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x[x.notna()].astype(np.float64)
+
+
+def _series_to_str_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x[x.notna()].astype(str)
+
+
+def _series_to_datetime(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
+    return pd.to_datetime(x)
+
+
+def _series_to_utc_datetime(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
+    return pd.to_datetime(x, utc=True)
+
+
+def _series_to_datetime_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return pd.to_datetime(x)
+
+
+def _series_to_utc_datetime_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return pd.to_datetime(x, utc=True)
 
 
 class DataFrameMixin(Generic[DataFrameT, DataObject]):
@@ -56,11 +93,16 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         int: _series_to_int,
         float: _series_to_float,
         str: _series_to_str,
-        date: pd.to_datetime,
+        date: _series_to_datetime,
         datetime: _series_to_utc_datetime,
+        Union[int, None]: _series_to_int_nullable,
+        Union[float, None]: _series_to_float_nullable,
+        Union[str, None]: _series_to_str_nullable,
+        Union[date, None]: _series_to_datetime_nullable,
+        Union[datetime, None]: _series_to_utc_datetime_nullable,
     }
 
-    def __init__(self) -> None:
+    def __init__(self, **series: pd.Series) -> None:
         # Fields added here only to allow mypy to provide correct type hints
         self.__data_object__: Dict[str, Any] = {}
         self.__dataframe__: DataFrameMetadata = None  # type: ignore
@@ -68,9 +110,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         raise NotImplementedError  # must use @dataframe decorator  # pragma: no cover
 
     @staticmethod
-    def __init_from_series__(
-        self, **series: pd.Series
-    ):  # pylint: disable=bad-staticmethod-argument
+    def __init_from_series__(self, **series: pd.Series):  # pylint: disable=bad-staticmethod-argument
         df = pd.DataFrame(series)
         df.index.name = None  # Removes index name to avoid colisions with series name
         if self.__data_object__["validate"]:
@@ -106,10 +146,7 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         return self._from_df(self.__df[key])
 
     def _to_dataobjects(self) -> List[DataObject]:
-        return [
-            self.DataObject(**fields)
-            for fields in self.__df.to_dict(orient="records")
-        ]
+        return [self.DataObject(**fields) for fields in self.__df.to_dict(orient="records")]
 
     def event_id(self, *args, **kwargs) -> str:
         return ""
@@ -132,9 +169,28 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         else:
             object.__setattr__(self, name, value)
 
-    def _coerce_datatypes(self, df: pd.DataFrame) -> Dict[str, pd.Series]:
+    def _get_series(
+        self,
+        df: pd.DataFrame,
+        field_name: str,
+        field_info: FieldInfo,
+    ) -> pd.Series:
+        try:
+            return df[field_name]
+        except KeyError:
+            default_value = field_info.get_default()
+            if default_value is not PydanticUndefined:
+                return pd.Series([default_value] * len(df))
+            raise
+
+    def _coerce_datatypes(
+        self,
+        df: pd.DataFrame,
+    ) -> Dict[str, pd.Series]:
         return {
-            name: self.DATATYPE_MAPPING[field.annotation](df[name])  # type: ignore
+            name: self.DATATYPE_MAPPING[field.annotation](  # type: ignore[index, operator]
+                name, self._get_series(df, name, field)
+            )
             for name, field in self.__dataframe__.fields.items()
         }
 
@@ -153,7 +209,7 @@ def dataframe(
         if hasattr(cls, "__annotations__") and hasattr(cls, "__dataclass_fields__"):
             amended_class = type(
                 cls.__name__,
-                (DataFrameMixin, ) + cls.__mro__,
+                (DataFrameMixin,) + cls.__mro__,
                 dict(cls.__dict__),
             )
             setattr(amended_class, "__init__", DataFrameMixin.__init_from_series__)
@@ -162,7 +218,7 @@ def dataframe(
 
     def add_dataframe_metadata(cls):
         serialized_fields = {k: (v.annotation, v) for k, v in fields(cls).items()}
-        dataobject_type = create_model(cls.__name__+"DataObject", **serialized_fields)
+        dataobject_type = create_model(cls.__name__ + "DataObject", **serialized_fields)
         dataobject_type = dataobject(dataobject_type, unsafe=True)
 
         setattr(cls, "DataObject", dataobject_type)
@@ -185,17 +241,12 @@ def dataframe(
         setattr(cls, "event_id", StreamEventMixin.event_id)
         setattr(cls, "event_ts", StreamEventMixin.event_ts)
 
-    def set_fields_optional(cls):
-        for _, field in fields(cls).items():
-            field.default = None
-
     def wrap(cls) -> Type[DataFrameMixin]:
         if hasattr(cls, "__dataframe__"):
             return cls
         add_dataframe_metadata(cls)
         amended_class = add_dataframe_mixin(cls)
         add_dataobject_annotations(amended_class, unsafe, validate, schema)
-        set_fields_optional(amended_class)
         return amended_class
 
     if decorated_class is None:

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit/dataframes/serialization/dataset.py

@@ -1,25 +1,37 @@
-"""Dataset objects definition, used as a result of serialized dataframes
-"""
+"""Dataset objects definition, used as a result of serialized dataframes"""
 
 from importlib import import_module
-from typing import Generic, Type, TypeVar
+from typing import Any, Dict, Generic, Type, TypeVar
 
-from hopeit.dataobjects import dataclass, dataobject
+from hopeit.dataobjects import dataclass, dataobject, field
 
 DataFrameT = TypeVar("DataFrameT")
 
 
+class DatasetLoadError(Exception):
+    pass
+
+
 @dataobject
 @dataclass
 class Dataset(Generic[DataFrameT]):
     """Persisted representation of a @dataframe object"""
+
     protocol: str
     partition_key: str
     key: str
     datatype: str
+    schema: Dict[str, Any] = field(default_factory=dict)
 
     async def load(self) -> DataFrameT:
-        return await self.__storage.load(self)  # type: ignore[attr-defined]
+        try:
+            dataframe = await self.__storage.load(self)  # type: ignore[attr-defined]
+            return dataframe
+        except (RuntimeError, IOError, KeyError) as e:
+            raise DatasetLoadError(
+                f"Error {type(e).__name__}: {e} loading dataset of type {self.datatype} "
+                f"at location {self.partition_key}/{self.key}"
+            ) from e
 
     @classmethod
     async def save(cls, dataframe: DataFrameT) -> "Dataset[DataFrameT]":

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit/dataframes/serialization/files.py

@@ -1,5 +1,4 @@
-"""Support for `@dataframes` serialization to files
-"""
+"""Support for `@dataframes` serialization to files"""
 
 import io
 from importlib import import_module
@@ -7,6 +6,7 @@ from typing import Generic, Optional, Type, TypeVar
 from uuid import uuid4
 
 import pandas as pd
+from pydantic import TypeAdapter
 
 try:
     import pyarrow  # type: ignore  # noqa  # pylint: disable=unused-import
@@ -54,14 +54,13 @@ class DatasetFileStorage(Generic[DataFrameT]):
             partition_key=partition_key,
             key=key,
             datatype=f"{datatype.__module__}.{datatype.__qualname__}",
+            schema=TypeAdapter(datatype).json_schema(),
         )
 
     async def load(self, dataset: Dataset) -> EventPayloadType:
         """Loads @dataframe annotated object using Dataset metadata"""
         datatype: Type[DataFrameT] = find_dataframe_type(dataset.datatype)
-        data = await self.storage.get_file(
-            dataset.key, partition_key=dataset.partition_key
-        )
+        data = await self.storage.get_file(dataset.key, partition_key=dataset.partition_key)
         if data is None:
             raise FileNotFoundError(dataset.key)
         df = pd.read_parquet(io.BytesIO(data), engine="pyarrow")

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit/dataframes/serialization/settings.py

@@ -1,5 +1,4 @@
-"""Support for plugin configuration
-"""
+"""Support for plugin configuration"""
 
 from typing import Optional
 

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.0b7
+Version: 0.25.1
 Summary: Hopeit Engine Dataframes Toolkit
-Home-page: https://github.com/hopeit-git/hopeit.engine
-Author: Leo Smerling and Pablo Canto
-Author-email: contact@hopeit.com.ar
+Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
+Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
 Project-URL: Docs: RTD, https://hopeitengine.readthedocs.io/en/latest/
 Project-URL: GitHub: issues, https://github.com/hopeit-git/hopeit.engine/issues
@@ -13,7 +12,6 @@ Project-URL: GitHub: repo, https://github.com/hopeit-git/hopeit.engine
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Intended Audience :: Developers
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -24,9 +22,9 @@ Classifier: Operating System :: Microsoft :: Windows
 Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.25.0b7
+Requires-Python: >=3.9
+Description-Content-Type: text/plain
+Requires-Dist: hopeit.engine[fs-storage]==0.25.1
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit_dataframes-0.25.0b7 → hopeit_dataframes-0.25.1}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 README.md
+pyproject.toml
 setup.py
 src/hopeit.dataframes.egg-info/PKG-INFO
 src/hopeit.dataframes.egg-info/SOURCES.txt
@@ -11,6 +12,8 @@ src/hopeit/dataframes/py.typed
 src/hopeit/dataframes/serialization/__init__.py
 src/hopeit/dataframes/serialization/dataset.py
 src/hopeit/dataframes/serialization/files.py
+src/hopeit/dataframes/serialization/py.typed
 src/hopeit/dataframes/serialization/settings.py
 src/hopeit/dataframes/setup/__init__.py
-src/hopeit/dataframes/setup/dataframes.py
+src/hopeit/dataframes/setup/dataframes.py
+src/hopeit/dataframes/setup/py.typed

hopeit_dataframes-0.25.1/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.25.1
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit_dataframes-0.25.0b7/setup.py

@@ -1,59 +0,0 @@
-import setuptools
-
-version = {}
-with open("../../../engine/src/hopeit/server/version.py") as fp:
-    exec(fp.read(), version)
-
-setuptools.setup(
-    name="hopeit.dataframes",
-    version=version["ENGINE_VERSION"],
-    description="Hopeit Engine Dataframes Toolkit",
-    license="Apache 2",
-    long_description=open("README.md").read(),
-    long_description_content_type="text/markdown",
-    author="Leo Smerling and Pablo Canto",
-    author_email="contact@hopeit.com.ar",
-    url="https://github.com/hopeit-git/hopeit.engine",
-    classifiers=[
-        "License :: OSI Approved :: Apache Software License",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Development Status :: 4 - Beta",
-        "Operating System :: POSIX :: Linux",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: Microsoft :: Windows",
-        "Topic :: Internet :: WWW/HTTP",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Framework :: AsyncIO",
-    ],
-    project_urls={
-        "CI: GitHub Actions": "https://github.com/hopeit-git/hopeit.engine/actions?query=workflow",  # noqa
-        "Docs: RTD": "https://hopeitengine.readthedocs.io/en/latest/",
-        "GitHub: issues": "https://github.com/hopeit-git/hopeit.engine/issues",
-        "GitHub: repo": "https://github.com/hopeit-git/hopeit.engine",
-    },
-    package_dir={"": "src"},
-    packages=[
-        "hopeit.dataframes",
-        "hopeit.dataframes.serialization",
-        "hopeit.dataframes.setup",
-    ],
-    include_package_data=True,
-    package_data={
-        "hopeit.dataframes": ["py.typed"],
-    },
-    python_requires=">=3.8",
-    install_requires=[
-        f"hopeit.engine[fs-storage]=={version['ENGINE_VERSION']}",
-        "pandas",
-        "numpy",
-    ],
-    extras_require={
-        "pyarrow": ["pyarrow"],
-    },
-    entry_points={},
-)

hopeit_dataframes-0.25.0b7/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.25.0b7
-pandas
-numpy
-
-[pyarrow]
-pyarrow