hopeit.dataframes 0.25.0b9__tar.gz → 0.25.2__tar.gz

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.0b9
+Version: 0.25.2
 Summary: Hopeit Engine Dataframes Toolkit
-Home-page: https://github.com/hopeit-git/hopeit.engine
-Author: Leo Smerling and Pablo Canto
-Author-email: contact@hopeit.com.ar
+Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
+Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
 Project-URL: Docs: RTD, https://hopeitengine.readthedocs.io/en/latest/
 Project-URL: GitHub: issues, https://github.com/hopeit-git/hopeit.engine/issues
@@ -24,8 +23,8 @@ Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.25.0b9
+Description-Content-Type: text/plain
+Requires-Dist: hopeit.engine[fs-storage]==0.25.2
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

hopeit_dataframes-0.25.2/pyproject.toml

@@ -0,0 +1,51 @@
+[build-system]
+requires = ["setuptools >= 64", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hopeit.dataframes"
+description = "Hopeit Engine Dataframes Toolkit"
+dynamic = ["version", "readme", "dependencies", "optional-dependencies"]
+license = { text = "Apache 2" }
+authors = [
+    { name = "Leo Smerling", email = "contact@hopeit.com.ar" },
+    { name = "Pablo Canto", email = "contact@hopeit.com.ar" },
+]
+classifiers = [
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Development Status :: 4 - Beta",
+    "Operating System :: POSIX :: Linux",
+    "Operating System :: MacOS :: MacOS X",
+    "Operating System :: Microsoft :: Windows",
+    "Topic :: Internet :: WWW/HTTP",
+    "Topic :: Software Development :: Libraries :: Python Modules",
+    "Framework :: AsyncIO",
+]
+requires-python = ">=3.9"
+
+
+[project.urls]
+"Homepage" = "https://github.com/hopeit-git/hopeit.engine"
+"CI: GitHub Actions" = "https://github.com/hopeit-git/hopeit.engine/actions?query=workflow"
+"Docs: RTD" = "https://hopeitengine.readthedocs.io/en/latest/"
+"GitHub: issues" = "https://github.com/hopeit-git/hopeit.engine/issues"
+"GitHub: repo" = "https://github.com/hopeit-git/hopeit.engine"
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.package-data]
+"hopeit.dataframes" = ["py.typed"]
+"hopeit.dataframes.serialization" = ["py.typed"]
+"hopeit.dataframes.setup" = ["py.typed"]
+
+[tool.setuptools.dynamic]
+readme = { file = ["README.md"], content-type = "text/plain" }

hopeit_dataframes-0.25.2/setup.py

@@ -0,0 +1,25 @@
+from setuptools import setup
+from os import environ
+
+version = {}
+try:
+    with open("../../../engine/src/hopeit/server/version.py") as fp:
+        exec(fp.read(), version)
+        ENGINE_VERSION = version["ENGINE_VERSION"]
+except FileNotFoundError:
+    ENGINE_VERSION = environ.get("ENGINE_VERSION")
+
+if not ENGINE_VERSION:
+    raise RuntimeError("ENGINE_VERSION is not specified.")
+
+setup(
+    version=ENGINE_VERSION,
+    install_requires=[
+        f"hopeit.engine[fs-storage]=={ENGINE_VERSION}",
+        "pandas",
+        "numpy",
+    ],
+    extras_require={
+        "pyarrow": ["pyarrow"],
+    },
+)

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit/dataframes/__init__.py

@@ -118,9 +118,9 @@ class DataFrames(Generic[DataFrameT, DataObject]):
         return datatype._from_dataobjects(dataobjects)  # type: ignore  # pylint: disable=protected-access
 
     @staticmethod
-    def to_dataobjects(obj: DataFrameT) -> List[DataObject]:
+    def to_dataobjects(obj: DataFrameT, *, normalize_null_values: bool = False) -> List[DataObject]:
         """Converts `@dataframe` object to a list of standard `@dataobject`s"""
-        return obj._to_dataobjects()  # type: ignore  # pylint: disable=protected-access
+        return obj._to_dataobjects(normalize_null_values)  # type: ignore  # pylint: disable=protected-access
 
     @staticmethod
     def from_array(datatype: Type[DataFrameT], array: np.ndarray) -> DataFrameT:

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit/dataframes/dataframe.py

@@ -4,8 +4,7 @@ DataFrames type abstractions.
 
 import dataclasses
 from datetime import date, datetime, timezone
-from functools import partial
-from typing import Any, Callable, Dict, Generic, Iterator, List, Type, TypeVar
+from typing import Any, Callable, Dict, Generic, Iterator, List, Type, TypeVar, Union
 
 import numpy as np
 import pandas as pd
@@ -20,6 +19,7 @@ from hopeit.dataobjects import (
     fields,
 )
 from hopeit.dataobjects.payload import Payload
+from pydantic_core import PydanticUndefined
 
 DataFrameT = TypeVar("DataFrameT")
 
@@ -31,19 +31,65 @@ class DataFrameMetadata:
 
 
 # Functions to do type coercion
-def _series_to_int(x: pd.Series) -> pd.Series:
+def _series_to_int(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(np.int64)
 
 
-def _series_to_float(x: pd.Series) -> pd.Series:
+def _series_to_bool(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
+    return x.astype(bool)
+
+
+def _series_to_float(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(np.float64)
 
 
-def _series_to_str(x: pd.Series) -> pd.Series:
+def _series_to_str(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
     return x.astype(str)
 
 
-_series_to_utc_datetime = partial(pd.to_datetime, utc=True)
+# Functions to do type coercion
+def _series_to_int_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x.dropna().astype(np.int64)
+
+
+def _series_to_bool_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x.dropna().astype(bool)
+
+
+def _series_to_float_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x.dropna().astype(np.float64)
+
+
+def _series_to_str_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return x.dropna().astype(str)
+
+
+def _series_to_datetime(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
+    return pd.to_datetime(x)
+
+
+def _series_to_utc_datetime(field_name: str, x: pd.Series) -> pd.Series:
+    if x.isnull().values.any():  # type: ignore[union-attr]
+        raise ValueError(f"Field `{field_name}` is not nullable")
+    return pd.to_datetime(x, utc=True)
+
+
+def _series_to_datetime_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return pd.to_datetime(x.dropna())
+
+
+def _series_to_utc_datetime_nullable(_field_name: str, x: pd.Series) -> pd.Series:
+    return pd.to_datetime(x.dropna(), utc=True)
 
 
 class DataFrameMixin(Generic[DataFrameT, DataObject]):
@@ -53,15 +99,24 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
     Do not use this class directly, instead use `@dataframe` class decorator.
     """
 
+    DataFrameValueType = Union[int, bool, float, str, date, datetime, None]
+
     DATATYPE_MAPPING = {
         int: _series_to_int,
+        bool: _series_to_bool,
         float: _series_to_float,
         str: _series_to_str,
-        date: pd.to_datetime,
+        date: _series_to_datetime,
         datetime: _series_to_utc_datetime,
+        Union[int, None]: _series_to_int_nullable,
+        Union[bool, None]: _series_to_bool_nullable,
+        Union[float, None]: _series_to_float_nullable,
+        Union[str, None]: _series_to_str_nullable,
+        Union[date, None]: _series_to_datetime_nullable,
+        Union[datetime, None]: _series_to_utc_datetime_nullable,
     }
 
-    def __init__(self) -> None:
+    def __init__(self, **series: pd.Series) -> None:
         # Fields added here only to allow mypy to provide correct type hints
         self.__data_object__: Dict[str, Any] = {}
         self.__dataframe__: DataFrameMetadata = None  # type: ignore
@@ -104,7 +159,17 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
     def __getitem__(self, key) -> "DataFrameT":
         return self._from_df(self.__df[key])
 
-    def _to_dataobjects(self) -> List[DataObject]:
+    def _normalize_null_values(
+        self, value: Union[DataFrameValueType, pd.Timestamp]
+    ) -> DataFrameValueType:
+        return None if pd.isnull(value) else value
+
+    def _to_dataobjects(self, normalize_null_values: bool) -> List[DataObject]:
+        if normalize_null_values:
+            return [
+                self.DataObject(**{k: self._normalize_null_values(v) for k, v in fields.items()})
+                for fields in self.__df.to_dict(orient="records")
+            ]
         return [self.DataObject(**fields) for fields in self.__df.to_dict(orient="records")]
 
     def event_id(self, *args, **kwargs) -> str:
@@ -128,9 +193,28 @@ class DataFrameMixin(Generic[DataFrameT, DataObject]):
         else:
             object.__setattr__(self, name, value)
 
-    def _coerce_datatypes(self, df: pd.DataFrame) -> Dict[str, pd.Series]:
+    def _get_series(
+        self,
+        df: pd.DataFrame,
+        field_name: str,
+        field_info: FieldInfo,
+    ) -> pd.Series:
+        try:
+            return df[field_name]
+        except KeyError:
+            default_value = field_info.get_default()
+            if default_value is not PydanticUndefined:
+                return pd.Series([default_value] * len(df))
+            raise
+
+    def _coerce_datatypes(
+        self,
+        df: pd.DataFrame,
+    ) -> Dict[str, pd.Series]:
         return {
-            name: self.DATATYPE_MAPPING[field.annotation](df[name])  # type: ignore
+            name: self.DATATYPE_MAPPING[field.annotation](  # type: ignore[index, operator]
+                name, self._get_series(df, name, field)
+            )
             for name, field in self.__dataframe__.fields.items()
         }
 
@@ -181,17 +265,12 @@ def dataframe(
         setattr(cls, "event_id", StreamEventMixin.event_id)
         setattr(cls, "event_ts", StreamEventMixin.event_ts)
 
-    def set_fields_optional(cls):
-        for _, field in fields(cls).items():
-            field.default = None
-
     def wrap(cls) -> Type[DataFrameMixin]:
         if hasattr(cls, "__dataframe__"):
             return cls
         add_dataframe_metadata(cls)
         amended_class = add_dataframe_mixin(cls)
         add_dataobject_annotations(amended_class, unsafe, validate, schema)
-        set_fields_optional(amended_class)
         return amended_class
 
     if decorated_class is None:

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit/dataframes/serialization/dataset.py

@@ -1,13 +1,17 @@
 """Dataset objects definition, used as a result of serialized dataframes"""
 
 from importlib import import_module
-from typing import Generic, Type, TypeVar
+from typing import Any, Dict, Generic, Type, TypeVar
 
-from hopeit.dataobjects import dataclass, dataobject
+from hopeit.dataobjects import dataclass, dataobject, field
 
 DataFrameT = TypeVar("DataFrameT")
 
 
+class DatasetLoadError(Exception):
+    pass
+
+
 @dataobject
 @dataclass
 class Dataset(Generic[DataFrameT]):
@@ -17,9 +21,17 @@ class Dataset(Generic[DataFrameT]):
     partition_key: str
     key: str
     datatype: str
+    schema: Dict[str, Any] = field(default_factory=dict)
 
     async def load(self) -> DataFrameT:
-        return await self.__storage.load(self)  # type: ignore[attr-defined]
+        try:
+            dataframe = await self.__storage.load(self)  # type: ignore[attr-defined]
+            return dataframe
+        except (RuntimeError, IOError, KeyError) as e:
+            raise DatasetLoadError(
+                f"Error {type(e).__name__}: {e} loading dataset of type {self.datatype} "
+                f"at location {self.partition_key}/{self.key}"
+            ) from e
 
     @classmethod
     async def save(cls, dataframe: DataFrameT) -> "Dataset[DataFrameT]":

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit/dataframes/serialization/files.py

@@ -6,6 +6,7 @@ from typing import Generic, Optional, Type, TypeVar
 from uuid import uuid4
 
 import pandas as pd
+from pydantic import TypeAdapter
 
 try:
     import pyarrow  # type: ignore  # noqa  # pylint: disable=unused-import
@@ -53,6 +54,7 @@ class DatasetFileStorage(Generic[DataFrameT]):
             partition_key=partition_key,
             key=key,
             datatype=f"{datatype.__module__}.{datatype.__qualname__}",
+            schema=TypeAdapter(datatype).json_schema(),
         )
 
     async def load(self, dataset: Dataset) -> EventPayloadType:

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit.dataframes.egg-info/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.1
 Name: hopeit.dataframes
-Version: 0.25.0b9
+Version: 0.25.2
 Summary: Hopeit Engine Dataframes Toolkit
-Home-page: https://github.com/hopeit-git/hopeit.engine
-Author: Leo Smerling and Pablo Canto
-Author-email: contact@hopeit.com.ar
+Author-email: Leo Smerling <contact@hopeit.com.ar>, Pablo Canto <contact@hopeit.com.ar>
 License: Apache 2
+Project-URL: Homepage, https://github.com/hopeit-git/hopeit.engine
 Project-URL: CI: GitHub Actions, https://github.com/hopeit-git/hopeit.engine/actions?query=workflow
 Project-URL: Docs: RTD, https://hopeitengine.readthedocs.io/en/latest/
 Project-URL: GitHub: issues, https://github.com/hopeit-git/hopeit.engine/issues
@@ -24,8 +23,8 @@ Classifier: Topic :: Internet :: WWW/HTTP
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Framework :: AsyncIO
 Requires-Python: >=3.9
-Description-Content-Type: text/markdown
-Requires-Dist: hopeit.engine[fs-storage]==0.25.0b9
+Description-Content-Type: text/plain
+Requires-Dist: hopeit.engine[fs-storage]==0.25.2
 Requires-Dist: pandas
 Requires-Dist: numpy
 Provides-Extra: pyarrow

{hopeit_dataframes-0.25.0b9 → hopeit_dataframes-0.25.2}/src/hopeit.dataframes.egg-info/SOURCES.txt

@@ -1,4 +1,5 @@
 README.md
+pyproject.toml
 setup.py
 src/hopeit.dataframes.egg-info/PKG-INFO
 src/hopeit.dataframes.egg-info/SOURCES.txt
@@ -11,6 +12,8 @@ src/hopeit/dataframes/py.typed
 src/hopeit/dataframes/serialization/__init__.py
 src/hopeit/dataframes/serialization/dataset.py
 src/hopeit/dataframes/serialization/files.py
+src/hopeit/dataframes/serialization/py.typed
 src/hopeit/dataframes/serialization/settings.py
 src/hopeit/dataframes/setup/__init__.py
-src/hopeit/dataframes/setup/dataframes.py
+src/hopeit/dataframes/setup/dataframes.py
+src/hopeit/dataframes/setup/py.typed

hopeit_dataframes-0.25.2/src/hopeit.dataframes.egg-info/requires.txt

@@ -0,0 +1,6 @@
+hopeit.engine[fs-storage]==0.25.2
+pandas
+numpy
+
+[pyarrow]
+pyarrow

hopeit_dataframes-0.25.0b9/setup.py

@@ -1,58 +0,0 @@
-import setuptools
-
-version = {}
-with open("../../../engine/src/hopeit/server/version.py") as fp:
-    exec(fp.read(), version)
-
-setuptools.setup(
-    name="hopeit.dataframes",
-    version=version["ENGINE_VERSION"],
-    description="Hopeit Engine Dataframes Toolkit",
-    license="Apache 2",
-    long_description=open("README.md").read(),
-    long_description_content_type="text/markdown",
-    author="Leo Smerling and Pablo Canto",
-    author_email="contact@hopeit.com.ar",
-    url="https://github.com/hopeit-git/hopeit.engine",
-    classifiers=[
-        "License :: OSI Approved :: Apache Software License",
-        "Intended Audience :: Developers",
-        "Programming Language :: Python",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Development Status :: 4 - Beta",
-        "Operating System :: POSIX :: Linux",
-        "Operating System :: MacOS :: MacOS X",
-        "Operating System :: Microsoft :: Windows",
-        "Topic :: Internet :: WWW/HTTP",
-        "Topic :: Software Development :: Libraries :: Python Modules",
-        "Framework :: AsyncIO",
-    ],
-    project_urls={
-        "CI: GitHub Actions": "https://github.com/hopeit-git/hopeit.engine/actions?query=workflow",  # noqa
-        "Docs: RTD": "https://hopeitengine.readthedocs.io/en/latest/",
-        "GitHub: issues": "https://github.com/hopeit-git/hopeit.engine/issues",
-        "GitHub: repo": "https://github.com/hopeit-git/hopeit.engine",
-    },
-    package_dir={"": "src"},
-    packages=[
-        "hopeit.dataframes",
-        "hopeit.dataframes.serialization",
-        "hopeit.dataframes.setup",
-    ],
-    include_package_data=True,
-    package_data={
-        "hopeit.dataframes": ["py.typed"],
-    },
-    python_requires=">=3.9",
-    install_requires=[
-        f"hopeit.engine[fs-storage]=={version['ENGINE_VERSION']}",
-        "pandas",
-        "numpy",
-    ],
-    extras_require={
-        "pyarrow": ["pyarrow"],
-    },
-    entry_points={},
-)

hopeit_dataframes-0.25.0b9/src/hopeit.dataframes.egg-info/requires.txt

@@ -1,6 +0,0 @@
-hopeit.engine[fs-storage]==0.25.0b9
-pandas
-numpy
-
-[pyarrow]
-pyarrow