interloper-pandas 0.2.0__tar.gz

interloper_pandas-0.2.0/PKG-INFO

@@ -0,0 +1,14 @@
+Metadata-Version: 2.3
+Name: interloper-pandas
+Version: 0.2.0
+Summary: Interloper pandas integration: DataFrame normalizer and adapter
+Author: Guillaume Onfroy
+Author-email: Guillaume Onfroy <guillaume@digitlcloud.com>
+Requires-Dist: pandas>=1.5
+Requires-Dist: interloper-core
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+
+# interloper-pandas
+
+Pandas DataFrame normalizer and adapter for Interloper.

interloper_pandas-0.2.0/README.md

@@ -0,0 +1,3 @@
+# interloper-pandas
+
+Pandas DataFrame normalizer and adapter for Interloper.

interloper_pandas-0.2.0/pyproject.toml

@@ -0,0 +1,43 @@
+# ###############
+# PROJECT / UV
+# ###############
+[project]
+name = "interloper-pandas"
+version = "0.2.0"
+description = "Interloper pandas integration: DataFrame normalizer and adapter"
+readme = "README.md"
+authors = [{ name = "Guillaume Onfroy", email = "guillaume@digitlcloud.com" }]
+requires-python = ">=3.10"
+dependencies = [
+    "pandas>=1.5",
+    "interloper-core",
+]
+
+[build-system]
+requires = ["uv_build>=0.9.5,<0.10.0"]
+build-backend = "uv_build"
+
+[tool.uv.sources]
+interloper-core = { workspace = true }
+
+# ###############
+# RUFF
+# ###############
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = ["E", "I", "UP", "ANN001", "ANN201", "ANN202"]
+
+[tool.ruff.lint.per-file-ignores]
+"__init__.py" = ["F401", "F403"]
+"tests/**" = ["ANN", "F811"]
+
+# ###############
+# PYRIGHT
+# ###############
+[tool.pyright]
+include = ["src"]
+typeCheckingMode = "basic"
+reportMissingParameterType = true
+ignore = ["libs/**", "tests/**", "scripts/**"]

interloper_pandas-0.2.0/src/interloper_pandas/__init__.py

@@ -0,0 +1,6 @@
+"""Interloper pandas integration: DataFrame normalizer and IO adapter."""
+
+from interloper_pandas.adapter import DataFrameAdapter
+from interloper_pandas.normalizer import DataFrameNormalizer
+
+__all__ = ["DataFrameAdapter", "DataFrameNormalizer"]

interloper_pandas-0.2.0/src/interloper_pandas/adapter.py

@@ -0,0 +1,44 @@
+"""DataFrame adapter for converting between pandas DataFrames and database rows."""
+
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from interloper.errors import AdapterError
+from interloper.io.adapter import DataAdapter
+
+
+class DataFrameAdapter(DataAdapter):
+    """Adapter for pandas ``DataFrame``.
+
+    Converts between ``DataFrame`` and ``list[dict]`` row format used by
+    :class:`~interloper.io.database.DatabaseIO`.
+    """
+
+    def to_rows(self, data: Any) -> list[dict[str, Any]]:
+        """Convert a ``DataFrame`` to a list of row dicts.
+
+        Args:
+            data: A pandas ``DataFrame``
+
+        Returns:
+            Rows as list of dicts
+
+        Raises:
+            TypeError: If *data* is not a ``DataFrame``
+        """
+        if not isinstance(data, pd.DataFrame):
+            raise AdapterError(f"DataFrameAdapter expects a pandas DataFrame, got {type(data).__name__}.")
+        return data.to_dict("records")
+
+    def from_rows(self, rows: list[dict[str, Any]]) -> Any:
+        """Convert rows to a pandas ``DataFrame``.
+
+        Args:
+            rows: Raw rows from the database
+
+        Returns:
+            A pandas ``DataFrame``
+        """
+        return pd.DataFrame(rows)

interloper_pandas-0.2.0/src/interloper_pandas/normalizer.py

@@ -0,0 +1,136 @@
+"""DataFrame-native normalizer for pandas DataFrames."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+import pandas as pd
+from interloper.normalizer.base import Normalizer
+from interloper.schema import infer_schema, reconcile_schema, validate_schema
+from pydantic import BaseModel
+
+
+@dataclass
+class DataFrameNormalizer(Normalizer):
+    """Type-native normalizer for pandas ``DataFrame`` asset data.
+
+    Accepts a ``DataFrame`` and returns a ``DataFrame`` — all transformations
+    are performed using native pandas operations for efficiency.
+
+    Usage::
+
+        @asset(normalizer=DataFrameNormalizer())
+        def my_asset(context):
+            return pd.DataFrame({"UserName": ["alice"], "Address": ["NYC"]})
+
+    Inherits all configuration fields from :class:`Normalizer`:
+    ``normalize_columns``, ``flatten_max_level``, ``flatten_separator``,
+    ``fill_missing``, ``infer``.
+    """
+
+    def normalize(self, data: Any) -> pd.DataFrame:
+        """Normalize *data* to a ``DataFrame`` with configured transformations.
+
+        If the input is already a ``DataFrame``, operates on it directly.
+        Otherwise, coerces to ``list[dict]`` first, then converts to
+        ``DataFrame``.
+
+        Args:
+            data: Raw asset output (``DataFrame`` or any type supported by
+                the base :class:`Normalizer`).
+
+        Returns:
+            Normalized ``DataFrame``.
+        """
+        if isinstance(data, pd.DataFrame):
+            df = data
+        else:
+            # Coerce to list[dict] using base class, then convert to DataFrame
+            rows = self._coerce(data)
+            df = pd.DataFrame(rows)
+
+        if df.empty:
+            return df
+
+        if self.flatten_max_level is None or self.flatten_max_level > 0:
+            df = self._flatten_dataframe(df)
+
+        if self.normalize_columns:
+            df = df.rename(columns=self.column_name)
+
+        # fill_missing is a no-op for DataFrames — pandas naturally fills
+        # missing columns with NaN when constructed from heterogeneous data.
+
+        return df
+
+    def infer_schema(self, data: pd.DataFrame) -> type[BaseModel]:
+        """Infer a Pydantic model from a ``DataFrame``.
+
+        Converts the DataFrame to records and delegates to the shared
+        schema inference logic.
+
+        Args:
+            data: Normalized ``DataFrame`` (output of :meth:`normalize`).
+
+        Returns:
+            A dynamically created Pydantic ``BaseModel`` subclass.
+        """
+        rows = data.to_dict("records")
+        return infer_schema(rows)
+
+    def validate_schema(
+        self,
+        data: pd.DataFrame,
+        schema: type[BaseModel],
+        *,
+        strict: bool = False,
+    ) -> None:
+        """Validate a ``DataFrame`` against a Pydantic schema.
+
+        Converts the DataFrame to records and validates each row.
+
+        Args:
+            data: Normalized ``DataFrame``.
+            schema: Pydantic model class to validate against.
+            strict: When ``True``, reject extra and missing required fields.
+        """
+        rows = data.to_dict("records")
+        validate_schema(rows, schema, strict=strict)
+
+    def reconcile(
+        self,
+        data: pd.DataFrame,
+        schema: type[BaseModel],
+    ) -> pd.DataFrame:
+        """Reconcile a ``DataFrame`` against a Pydantic schema.
+
+        Aligns columns to the schema (drops extras, adds missing) and
+        coerces values to the schema's types via Pydantic ``model_validate``.
+
+        Args:
+            data: Normalized ``DataFrame``.
+            schema: Pydantic model class describing the target shape.
+
+        Returns:
+            Reconciled ``DataFrame``.
+        """
+        rows = data.to_dict("records")
+        reconciled = reconcile_schema(rows, schema)
+        return pd.DataFrame(reconciled)
+
+    def _flatten_dataframe(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Flatten nested dicts in DataFrame cells using separator-joined keys.
+
+        Any cell value that is a ``dict`` is expanded into separate columns
+        with keys joined by :attr:`flatten_separator`.
+
+        Args:
+            df: Input ``DataFrame`` potentially containing dict-valued cells.
+
+        Returns:
+            Flattened ``DataFrame``.
+        """
+        rows = df.to_dict("records")
+        flattened = [self._flatten_dict(row) for row in rows]
+        return pd.DataFrame(flattened)