flowtask 5.13.8__tar.gz → 5.13.9__tar.gz

{flowtask-5.13.8 → flowtask-5.13.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.8
+Version: 5.13.9
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -50,7 +50,9 @@ Requires-Dist: nltk>=3.10.3
 Requires-Dist: jdcal==1.4.1
 Requires-Dist: html5lib==1.1
 Requires-Dist: shapely>=2.1.2
+Requires-Dist: pyproj>=3.6
 Requires-Dist: tzwhere==3.0.3
+Requires-Dist: timezonefinder>=9.0.0
 Requires-Dist: tabulate==0.9.0
 Requires-Dist: python-magic==0.4.27
 Requires-Dist: pytomlpp==1.0.13

{flowtask-5.13.8 → flowtask-5.13.9}/flowtask/components/CopyToPg.py

@@ -1,42 +1,40 @@
-import re
 import asyncio
-import multiprocessing
-import gc
 import csv
-from decimal import Decimal
 import datetime
+import gc
+import multiprocessing
+import re
+from collections.abc import Sequence
+from decimal import Decimal
 from io import BytesIO
-from typing import Sequence, Union
-import pandas as pd
-from pandas.api.types import (
-    is_datetime64_any_dtype,
-    is_integer_dtype,
-    is_float_dtype,
-    is_bool_dtype
-)
+from typing import Any, Optional
+
+import asyncpg
 import numpy as np
-from typing import Optional
-from querysource.types.dt.transforms import to_wall_clock
 import orjson
-import asyncpg
-from datamodel import Field
+import pandas as pd
+import shapely
+from asyncdb.exceptions import DataError, StatementError
+from asyncdb.models import Model
 from asyncpg.exceptions import (
-    StringDataRightTruncationError,
-    UniqueViolationError,
     ForeignKeyViolationError,
     NotNullViolationError,
+    StringDataRightTruncationError,
+    UniqueViolationError,
+)
+from datamodel import Field
+from pandas.api.types import (
+    is_datetime64_any_dtype,
 )
 from pgvector.asyncpg import register_vector
-from asyncdb.exceptions import StatementError, DataError
-from asyncdb.models import Model
+from querysource.types.dt.transforms import to_wall_clock
+
 # Dataintegration components:
 from ..exceptions import (
     ComponentError,
-    DataNotFound,
 )
 from ..interfaces.copy_to import CopyTo, dtypes
-from ..utils.json import json_decoder, json_encoder
-
+from ..utils.json import json_decoder
 
 # PostgreSQL type names (as reported by asyncpg) that store plain text.
 PG_TEXT_TYPES = frozenset({"text", "varchar", "bpchar", "character varying", "character", "name"})
@@ -89,6 +87,31 @@ def quote_ddl_columns(sql: str, columns: list[str]) -> str:
     return sql
 
 
+def _to_ewkb(value: Any) -> bytes:
+    """Encode WKT, EWKT, or a Shapely geometry as SRID-bearing EWKB.
+
+    Args:
+        value: Geometry text or a Shapely geometry.
+
+    Returns:
+        Binary EWKB with the source SRID, defaulting to EPSG:4326.
+    """
+    geometry = value
+    srid = 0
+    if isinstance(value, str):
+        text = value.strip()
+        match = re.match(r"^SRID=(\d+);(.*)$", text, flags=re.IGNORECASE | re.DOTALL)
+        if match:
+            srid = int(match.group(1))
+            text = match.group(2)
+        geometry = shapely.from_wkt(text)
+    if not srid:
+        srid = shapely.get_srid(geometry) or 4326
+    if shapely.get_srid(geometry) != srid:
+        geometry = shapely.set_srid(geometry, srid)
+    return shapely.to_wkb(geometry, include_srid=True)
+
+
 class CopyToPg(CopyTo):
     """
     CopyToPg
@@ -113,6 +136,7 @@ class CopyToPg(CopyTo):
     | array_columns  |  No      | List of column names containing JSON arrays. These columns will be formatted     |
     |                |          | appropriately before insertion.                                                  |
     |                |          | Requires `use_buffer` to be True.                                                |
+    | geometry_columns | No     | Mapping of columns to PostGIS types, or a list defaulting to geography Polygon.  |
     | use_quoting    |  No      | Boolean flag indicating whether to use quoting for CSV data insertion (optional).|
     |                |          | Defaults to False.                                                               |
     |                |          | Using quoting can be helpful for data containing special characters.             |
@@ -171,7 +195,7 @@ class CopyToPg(CopyTo):
                 stack_info=True
             )
 
-    def extract_copied(self, result: Union[str, Sequence[str], None]) -> int:
+    def extract_copied(self, result: str | Sequence[str] | None) -> int:
         if result is None:
             return 0
         if isinstance(result, str):
@@ -225,6 +249,12 @@ class CopyToPg(CopyTo):
                 f = (col, Optional[list], Field(default=None, nullable=True, db_type='vector'))
             elif col in self._array_columns:
                 f = (col, Optional[list], Field(default=None, nullable=True, db_type='text[]'))
+            elif col in getattr(self, "_geometry_columns", {}):
+                f = (
+                    col,
+                    Optional[str],
+                    Field(default=None, nullable=True, db_type=self._geometry_columns[col]),
+                )
             elif col in self._binary_columns:
                 f = (col, Optional[bytes], Field(default=None, nullable=True, db_type='bytea'))
             else:
@@ -321,14 +351,17 @@ class CopyToPg(CopyTo):
                 await asyncio.sleep(5e-3)
                 break  # exit loop
             except (asyncpg.exceptions.QueryCanceledError, StatementError) as e:
-                if "canceling statement due to statement timeout" in str(e) or "another operation is in progress" in str(e):  # noqa
+                if (
+                    "canceling statement due to statement timeout" in str(e)
+                    or "another operation is in progress" in str(e)
+                ):
                     retry_count += 1
                     self._logger.warning(
-                        f"CopyToPg Error: {str(e)}, Retrying... {retry_count}/{max_retries}"
+                        f"CopyToPg Error: {e!s}, Retrying... {retry_count}/{max_retries}"
                     )
                     if retry_count > max_retries:
                         raise ComponentError(
-                            f"CopyToPg Error: {str(e)}, Max Retries reached"
+                            f"CopyToPg Error: {e!s}, Max Retries reached"
                         ) from e
                     else:
                         # Create a new connection an wait until repeat operation:
@@ -362,6 +395,30 @@ class CopyToPg(CopyTo):
             self._logger.error(f"Error getting column types: {e}")
             return {}
 
+    async def _register_postgis_codecs(self, engine) -> None:
+        """Register binary EWKB codecs for installed PostGIS spatial types.
+
+        Args:
+            engine: Underlying asyncpg connection used to register codecs.
+
+        Raises:
+            ComponentError: If neither PostGIS spatial type is installed.
+        """
+        rows = await engine.fetch(
+            "SELECT t.typname, n.nspname FROM pg_type t JOIN pg_namespace n ON n.oid = t.typnamespace "
+            "WHERE t.typname IN ('geometry', 'geography')"
+        )
+        if not rows:
+            raise ComponentError("PostGIS extension not installed in target database")
+        for row in rows:
+            await engine.set_type_codec(
+                row["typname"],
+                schema=row["nspname"],
+                encoder=_to_ewkb,
+                decoder=shapely.from_wkb,
+                format="binary",
+            )
+
     def _coerce_text_columns(self, pg_column_types: list[dict] | dict) -> None:
         """Stringify scalar values bound for columns the table stores as text.
 
@@ -380,7 +437,12 @@ class CopyToPg(CopyTo):
             col["name"] for col in pg_column_types
             if col.get("pg_type") in PG_TEXT_TYPES
         ]
-        skip = set(self._json_columns) | set(self._array_columns) | set(self._vector_columns)
+        skip = (
+            set(self._json_columns)
+            | set(self._array_columns)
+            | set(self._vector_columns)
+            | set(getattr(self, "_geometry_columns", {}))
+        )
         scalars = (bool, int, float, Decimal, np.generic, datetime.date, datetime.datetime, pd.Timestamp)
         converted = []
         for col in text_cols:
@@ -483,6 +545,15 @@ class CopyToPg(CopyTo):
                     kw = {}
                     if hasattr(self, "use_quoting"):
                         kw = {"quoting": csv.QUOTE_NONNUMERIC}
+                    for col in getattr(self, "_geometry_columns", {}):
+                        if col in self.data.columns:
+                            self.data[col] = self.data[col].map(
+                                lambda geometry: (
+                                    f"SRID={shapely.get_srid(geometry) or 4326};{geometry.wkt}"
+                                    if isinstance(geometry, shapely.Geometry)
+                                    else geometry
+                                )
+                            )
                     self.data.to_csv(s_buf, index=None, header=None, **kw)
                     s_buf.seek(0)
                     if hasattr(self, "clean_df"):
@@ -505,6 +576,8 @@ class CopyToPg(CopyTo):
                                 format="binary",
                             )
                             await register_vector(conn.engine())
+                            if self._geometry_columns:
+                                await self._register_postgis_codecs(conn.engine())
                             # Saving as CSV to the table
                             result = await conn.engine().copy_to_table(
                                 table_name=self.tablename,
@@ -596,7 +669,10 @@ class CopyToPg(CopyTo):
                             if col in self.data.columns:
                                 # Ensure array values are Python lists
                                 self.data[col] = self.data[col].apply(
-                                    lambda x: None if x is None else x if isinstance(x, list) else eval(x) if isinstance(x, str) and x.startswith('[') else [x]  # noqa
+                                    lambda x: None if x is None
+                                    else x if isinstance(x, list)
+                                    else eval(x) if isinstance(x, str) and x.startswith('[')
+                                    else [x]
                                 )
                     # Final NaT cleanup for all columns - ensure we have no NaT values before sending to PostgreSQL
                     for col in self.data.columns:
@@ -624,6 +700,8 @@ class CopyToPg(CopyTo):
                             format="binary"
                         )
                         await register_vector(conn.engine())
+                        if self._geometry_columns:
+                            await self._register_postgis_codecs(conn.engine())
 
                         if self._debug:
                             for c in self.data.columns:

flowtask-5.13.9/flowtask/components/Frankfurter.py

@@ -0,0 +1,348 @@
+"""Frankfurter — exchange-rate component over the Frankfurter v2 REST API."""
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Callable
+from datetime import date as date_type
+from io import StringIO
+from typing import Any
+
+import backoff
+import httpx
+import orjson
+import pandas as pd
+
+from ..exceptions import ComponentError, ConfigError, DataNotFound
+from ..interfaces.flow import FlowComponent
+from ..interfaces.http import HTTPService
+
+API_URL: str = "https://api.frankfurter.dev"
+TYPES: tuple[str, ...] = ("rates", "rate", "convert", "currencies", "currency", "providers")
+METADATA_TYPES: tuple[str, ...] = ("currencies", "currency", "providers")
+FORMATS: tuple[str, ...] = ("ndjson", "csv", "json")
+GROUPS: tuple[str, ...] = ("week", "month")
+ACCEPT_BY_FORMAT: dict[str, str] = {
+    "ndjson": "application/x-ndjson",
+    "csv": "text/csv",
+    "json": "application/json",
+}
+
+
+class Frankfurter(HTTPService, FlowComponent):
+    """
+    Frankfurter
+
+        Overview
+
+            Consumes the Frankfurter v2 exchange-rates REST API with an async httpx client. It
+            returns latest, historical, provider, and metadata exchange-rate data as a pandas
+            DataFrame by default, or as the raw response payload.
+
+    :widths: auto
+
+        | type             | No | rates (default), rate, convert, currencies, currency, or providers. |
+        | base             | No | Base currency ISO code; required for rate, convert, and currency. |
+        | quotes           | No | Target currencies, as a list or comma-separated string. |
+        | date             | No | Historical date in YYYY-MM-DD format. |
+        | start_date       | No | Time-series start date, sent as from. |
+        | end_date         | No | Time-series end date, sent as to. |
+        | group            | No | week or month; requires start_date. |
+        | provider         | No | Provider route key, exclusive with providers. |
+        | providers        | No | Provider filter list, exclusive with provider. |
+        | expand_providers | No | Adds contributing providers to every rate row. |
+        | scope            | No | currencies only: all includes legacy currencies. |
+        | amount           | No | Positive convert amount, added to the result with converted. |
+        | inverse          | No | Adds inverse_rate (1 / rate) computed locally, with no second request. |
+        | format           | No | ndjson (default), csv, or json. Metadata is always json. |
+        | as_dataframe     | No | Return a DataFrame when true, otherwise the raw payload. |
+        | api_url          | No | API base URL, defaulting to https://api.frankfurter.dev. |
+        | timeout          | No | Request timeout in seconds, default 30. |
+        | max_retries      | No | Attempts for transport and server errors, default 3. |
+
+    :consumes: none
+    :produces: dataframe
+
+        Example:
+
+        ```yaml
+        - Frankfurter:
+            type: rates
+            base: USD
+            quotes: [EUR, GBP]
+            start_date: "2026-10-01"
+            end_date: "2026-10-05"
+            format: ndjson
+            inverse: true
+            as_dataframe: true
+        ```
+    """
+
+    _version: str = "1.0.0"
+    accept: str = "application/json"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop | None = None,
+        job: Callable | None = None,
+        stat: Callable | None = None,
+        **kwargs: Any,
+    ) -> None:
+        self.type: str = kwargs.get("type", "rates")
+        self.base: str | None = kwargs.get("base")
+        self.quotes: list[str] | str | None = kwargs.get("quotes")
+        self.date: str | None = kwargs.get("date")
+        self.start_date: str | None = kwargs.get("start_date")
+        self.end_date: str | None = kwargs.get("end_date")
+        self.group: str | None = kwargs.get("group")
+        self.provider: str | None = kwargs.get("provider")
+        self.providers: list[str] | str | None = kwargs.get("providers")
+        self.expand_providers: bool = kwargs.get("expand_providers", False)
+        self.scope: str | None = kwargs.get("scope")
+        self.amount: float | None = kwargs.get("amount")
+        self.inverse: bool = kwargs.get("inverse", False)
+        self.format: str = kwargs.get("format", "ndjson")
+        self.as_dataframe: bool = kwargs.get("as_dataframe", True)
+        self.api_url: str = kwargs.get("api_url", API_URL)
+        self.max_retries: int = kwargs.get("max_retries", 3)
+        self._injected_client: httpx.AsyncClient | None = kwargs.pop("client", None)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.timeout = kwargs.get("timeout", 30)
+
+    async def start(self, **kwargs: Any) -> bool:
+        """Validate configuration and initialize an owned or injected HTTP client."""
+        self._validate()
+        if self._injected_client is not None:
+            self._client = self._injected_client
+        else:
+            self._client = httpx.AsyncClient(base_url=self.api_url, timeout=httpx.Timeout(self.timeout))
+        return True
+
+    async def run(self) -> Any:
+        """Fetch, parse, and publish the configured Frankfurter result."""
+        path, params, headers = self._build_request()
+        response = await self._fetch(path, params, headers)
+        self._raise_for_status(response)
+        result = await self._parse(response)
+        if self.inverse and isinstance(result, pd.DataFrame):
+            result = self._apply_inverse(result)
+        if self.type == "convert" and isinstance(result, pd.DataFrame):
+            result = self._apply_conversion(result)
+        self.add_metric("URL", str(response.url))
+        self.add_metric("TYPE", self.type)
+        self.add_metric("FORMAT", self.format)
+        self.add_metric("CONTENT_TYPE", response.headers.get("content-type", ""))
+        if isinstance(result, pd.DataFrame):
+            self.add_metric("NUMROWS", len(result.index))
+            self.add_metric("NUMCOLS", len(result.columns))
+        self._result = result
+        return self._result
+
+    async def close(self) -> bool:
+        """Close only the HTTP client that this component created."""
+        if hasattr(self, "_client") and self._client is not None and self._injected_client is None:
+            await self._client.aclose()
+        return True
+
+    def _validate(self) -> None:
+        """Validate component configuration before issuing any HTTP request."""
+        if self.type not in TYPES:
+            raise ConfigError(f"Invalid type: {self.type}")
+        if self.format not in FORMATS:
+            raise ConfigError(f"Invalid format: {self.format}")
+        if self.group is not None and self.group not in GROUPS:
+            raise ConfigError(f"Invalid group: {self.group}")
+        if self.group is not None and not self.start_date:
+            raise ConfigError("group requires start_date")
+        if self.end_date and not self.start_date:
+            raise ConfigError("end_date requires start_date")
+        if self.date and (self.start_date or self.end_date):
+            raise ConfigError("date is exclusive with start_date and end_date")
+
+        parsed_dates: dict[str, date_type] = {}
+        for name in ("date", "start_date", "end_date"):
+            value = getattr(self, name)
+            if value:
+                try:
+                    parsed_dates[name] = date_type.fromisoformat(value)
+                except (TypeError, ValueError) as exc:
+                    raise ConfigError(f"Invalid {name}: {value}") from exc
+        if self.start_date and self.end_date and parsed_dates["end_date"] < parsed_dates["start_date"]:
+            raise ConfigError("end_date must be greater than or equal to start_date")
+
+        self.quotes = self._normalize_codes(self.quotes)
+        self.providers = self._normalize_codes(self.providers)
+        if self.base:
+            self.base = self.base.upper()
+        if self.provider:
+            self.provider = self.provider.upper()
+
+        if self.type == "rate" and (not self.base or not self.quotes or len(self.quotes) != 1):
+            raise ConfigError("rate requires base and exactly one quote")
+        if self.type == "convert":
+            if not self.base:
+                raise ConfigError("convert requires base")
+            if not isinstance(self.amount, (int, float)) or isinstance(self.amount, bool) or self.amount <= 0:
+                raise ConfigError("convert requires amount greater than zero")
+        if self.inverse:
+            if self.type in METADATA_TYPES:
+                raise ConfigError(f"inverse is not supported for type: {self.type}")
+            if not self.as_dataframe:
+                raise ConfigError("inverse requires as_dataframe")
+        if self.provider and self.providers:
+            raise ConfigError("provider is exclusive with providers")
+        if self.type == "currency" and not self.base:
+            raise ConfigError("currency requires base")
+        if self.type in METADATA_TYPES and self.format != "json":
+            self._logger.warning("Frankfurter metadata endpoints always use JSON; forcing format=json")
+            self.format = "json"
+
+    @staticmethod
+    def _normalize_codes(value: list[str] | str | None) -> list[str] | None:
+        """Normalize a comma-separated or list currency/provider value to uppercase codes."""
+        if value is None:
+            return None
+        values = value.split(",") if isinstance(value, str) else value
+        return [str(item).strip().upper() for item in values if str(item).strip()]
+
+    def _build_request(self) -> tuple[str, dict[str, str], dict[str, str]]:
+        """Build a v2 Frankfurter path, query parameters, and request headers."""
+        path: str
+        params: dict[str, str] = {}
+        if self.type in ("rates", "convert"):
+            path = "/v2/rates"
+            if self.provider:
+                path = f"/v2/providers/{self.provider}/rates"
+            self._add_rate_params(params)
+        elif self.type == "rate":
+            quote = self.quotes[0]
+            path = f"/v2/rate/{self.base}/{quote}"
+            if self.provider:
+                path = f"/v2/providers/{self.provider}/rate/{self.base}/{quote}"
+            self._add_date_params(params)
+            if self.providers:
+                params["providers"] = ",".join(self.providers)
+        elif self.type == "currencies":
+            path = "/v2/currencies"
+            if self.scope:
+                params["scope"] = self.scope
+        elif self.type == "currency":
+            path = f"/v2/currency/{self.base}"
+        else:
+            path = "/v2/providers"
+            if self.provider:
+                path = f"/v2/providers/{self.provider}"
+
+        if self.format == "csv" and self.type not in METADATA_TYPES:
+            path = f"{path}.csv"
+        return path, params, {"Accept": ACCEPT_BY_FORMAT[self.format]}
+
+    def _add_rate_params(self, params: dict[str, str]) -> None:
+        """Add rate-endpoint query parameters to a request parameter mapping."""
+        if self.base:
+            params["base"] = self.base
+        if self.quotes:
+            params["quotes"] = ",".join(self.quotes)
+        self._add_date_params(params)
+        if self.group:
+            params["group"] = self.group
+        if self.providers:
+            params["providers"] = ",".join(self.providers)
+        if self.expand_providers:
+            params["expand"] = "providers"
+
+    def _add_date_params(self, params: dict[str, str]) -> None:
+        """Add mutually-exclusive historical-date or time-series query parameters."""
+        if self.date:
+            params["date"] = self.date
+        elif self.start_date:
+            params["from"] = self.start_date
+            if self.end_date:
+                params["to"] = self.end_date
+
+    async def _fetch(self, path: str, params: dict[str, str], headers: dict[str, str]) -> httpx.Response:
+        """Fetch a response, retrying transport failures and 5xx statuses with backoff."""
+        async def request() -> httpx.Response:
+            response = await self._client.get(path, params=params, headers=headers)
+            if response.status_code >= 500:
+                raise httpx.HTTPStatusError("Frankfurter server error", request=response.request, response=response)
+            return response
+
+        retry_request = backoff.on_exception(
+            backoff.expo,
+            (httpx.TransportError, httpx.HTTPStatusError),
+            max_tries=self.max_retries,
+            jitter=backoff.full_jitter,
+        )(request)
+        try:
+            return await retry_request()
+        except httpx.HTTPStatusError as exc:
+            raise ComponentError(f"Frankfurter {exc.response.status_code}: server error") from exc
+        except httpx.TransportError as exc:
+            raise ComponentError(f"Frankfurter transport error: {exc}") from exc
+
+    def _raise_for_status(self, response: httpx.Response) -> None:
+        """Map Frankfurter API errors to the framework's component exceptions."""
+        if response.status_code < 400:
+            return
+        message = response.text
+        try:
+            payload = orjson.loads(response.text)
+            message = payload.get("message", message) if isinstance(payload, dict) else message
+        except orjson.JSONDecodeError:
+            pass
+        if response.status_code == 404:
+            raise DataNotFound(f"Frankfurter: {message}")
+        raise ComponentError(f"Frankfurter {response.status_code}: {message}")
+
+    async def _parse(self, response: httpx.Response) -> pd.DataFrame | str | list | dict:
+        """Parse a response according to its Content-Type and output configuration."""
+        text = response.text
+        if not text.strip():
+            raise DataNotFound("Frankfurter: No data was found.")
+        content_type = response.headers.get("content-type", "").split(";", 1)[0].strip().lower()
+        if content_type == "application/x-ndjson":
+            if not self.as_dataframe:
+                return text
+            result: pd.DataFrame | str | list | dict = pd.read_json(StringIO(text), lines=True)
+        elif content_type == "text/csv":
+            if not self.as_dataframe:
+                return text
+            result = pd.read_csv(StringIO(text))
+        elif content_type == "application/json":
+            payload = orjson.loads(text)
+            if not payload:
+                raise DataNotFound("Frankfurter: No data was found.")
+            if not self.as_dataframe:
+                return payload
+            result = await self.create_dataframe(payload)
+        else:
+            raise ComponentError(f"Frankfurter: Unsupported content type: {content_type}")
+
+        if not isinstance(result, pd.DataFrame) or result.empty:
+            raise DataNotFound("Frankfurter: No data was found.")
+        if "date" in result.columns:
+            result["date"] = pd.to_datetime(result["date"], format="%Y-%m-%d")
+        if "rate" in result.columns:
+            result["rate"] = result["rate"].astype("float64")
+        return result
+
+    def _apply_inverse(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Add the reciprocal quote-to-base rate, computed locally from the same response.
+
+        The API exposes no multi-base query, so the opposite direction would otherwise cost
+        one extra request per base. ``inverse_rate`` is ``1 / rate``; a zero rate yields NaN
+        instead of infinity.
+        """
+        if "rate" not in df.columns:
+            raise ComponentError("Frankfurter: inverse requires a rate column")
+        df = df.copy()
+        rate = df["rate"].astype("float64")
+        df["inverse_rate"] = (1.0 / rate.where(rate != 0.0)).astype("float64")
+        return df
+
+    def _apply_conversion(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Add configured amount and unrounded converted-value columns to a rates frame."""
+        df = df.copy()
+        df["amount"] = float(self.amount)
+        df["converted"] = df["amount"] * df["rate"]
+        return df