flowtask 5.13.2__tar.gz → 5.13.3__tar.gz

{flowtask-5.13.2 → flowtask-5.13.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.2
+Version: 5.13.3
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0

{flowtask-5.13.2 → flowtask-5.13.3}/flowtask/components/BestBuy.py

@@ -35,14 +35,17 @@ from ..interfaces.flow import FlowComponent
 from ..interfaces import HTTPService, SeleniumService
 from ..interfaces.http import ua
 from ..utils.bestbuy_greedy import (
-    NOT_ENABLED_STATES,
     CoverageMap,
     SeedCoverage,
+    SkuFulfillment,
     StoreAvailability,
     Target,
+    availability_types,
+    fulfillment_slice,
     greedy_next_seed,
     load_coverage,
     parse_fulfillment,
+    parse_sku_fulfillment,
     save_coverage,
 )
 
@@ -106,6 +109,25 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
     | coverage_file              | No  | Coverage map JSON (default <storage_path>/bestbuy_coverage.json; in-memory when unset). |
     | proxy_session_max_requests | No  | Requests before a sticky proxy session is rotated (default 50). |
     | proxy_session_max_age      | No  | Seconds before a sticky proxy session is rotated (default 540). |
+    | debug_payload              | No  | availability only: log the full GraphQL JSON of every call; with storage_path also dump it to <storage_path>/payloads/ (default false). |
+
+    ``type: availability`` output columns (one row per store × sku). Legacy contract:
+    ``enabled``, ``locationId``, ``location_data``, ``availability`` (pickup ``fulfillmentType``),
+    ``inStoreAvailability`` / ``inStoreAvailable`` (pickup eligible AND inventory in store),
+    ``onShelfDisplay`` (kept for compatibility; the API has no shelf-display signal, so it stays
+    ``NULL`` unless the input already carries it), ``pickupEligible``,
+    ``availablePickupQuantity``, ``availableInStoreQuantity``, ``checked``.
+    Extended columns: ``availabilityTypes`` (``IN_STORE,PICKUP,SHIPPING,DELIVERY`` summary),
+    ``condition``; pickup detail ``pickupMinHours``, ``pickupMinDate``, ``pickupMaxDate``,
+    ``pickupFulfillDate``, ``pickupDisplayDateType``, ``pickupInStoreOnly``, ``pickupPreorderable``,
+    ``pickupBackordered``, ``pickupWhenAvailable``, ``pickupRestrictions``, ``storeDistance``;
+    shipping (for the request zipcode) ``shippingEligible``, ``shippingZipcode``, ``shippingBackordered``,
+    ``shippingInStoreOnly``, ``shippingPreorderable``, ``fulfillByVendor``, ``directToBoxEligible``,
+    ``shippingSameDay``, ``shippingDefaultOption``, ``shippingFastestDate``, ``shippingFastestPrice``,
+    ``shippingFreeDate``, ``shippingOptions`` (list of dicts); home delivery ``deliveryEligible``,
+    ``deliverable``, ``deliveryBackordered``, ``deliveryInStoreOnly``, ``installationEligible``;
+    and ``fulfillment_data``, the raw JSON slice (button states, store, pickup entry, shipping,
+    delivery) the columns were derived from.
 
     :consumes: dataframe
     :produces: dataframe
@@ -196,6 +218,8 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
         self._pool_size: int = kwargs.get('concurrency', 12)
         self.proxy_session_max_requests: int = kwargs.get('proxy_session_max_requests', 50)
         self.proxy_session_max_age: float = kwargs.get('proxy_session_max_age', 540)
+        # Debug: dump the raw GraphQL availability payload (log + optional file)
+        self.debug_payload: bool = kwargs.get('debug_payload', False)
         self._session_pool: list[ProxySession] = []
         self._pool_lock = asyncio.Lock()
         self._pool_cursor: int = 0
@@ -989,7 +1013,10 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             await self._release_session(session, status=r.status_code)
             if r.status_code == 200:
                 self._num_iterations += 1
-                return r.json()
+                payload = r.json()
+                if self.debug_payload:
+                    await self._dump_availability_payload(payload, sku, store_id, nearby)
+                return payload
             if r.status_code in (522, 502, 503, 504) and attempt < 3:
                 await asyncio.sleep(0.5 * attempt)
                 continue
@@ -1005,6 +1032,36 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             return None
         return None
 
+    async def _dump_availability_payload(self, payload: dict, sku: str, store_id: str,
+                                         nearby: bool) -> None:
+        """Log the raw GraphQL fulfillment payload and optionally persist it to disk.
+
+        Enabled with ``debug_payload: true``. The full JSON is written to the logger
+        and, when ``storage_path`` is configured, saved as
+        ``<storage_path>/payloads/availability_<sku>_<store_id>[_nearby].json``.
+
+        Args:
+            payload: Decoded JSON returned by ``/gateway/graphql/fulfillment``.
+            sku: Product SKU of the request.
+            store_id: Seed store id of the request.
+            nearby: Whether the request asked for nearby stores (bulk mode seed call).
+        """
+        pretty = json.dumps(payload, indent=2, ensure_ascii=False, default=str)
+        self._logger.info(
+            f"BestBuy availability payload sku={sku} store={store_id} nearby={nearby}:\n{pretty}"
+        )
+        if not self.storage_path:
+            return
+        suffix = "_nearby" if nearby else ""
+        out_dir = Path(self.storage_path).joinpath("payloads")
+        out_file = out_dir.joinpath(f"availability_{sku}_{store_id}{suffix}.json")
+        try:
+            await asyncio.to_thread(out_dir.mkdir, parents=True, exist_ok=True)
+            await asyncio.to_thread(out_file.write_text, pretty, encoding="utf-8")
+            self._logger.debug(f"BestBuy: availability payload saved to {out_file}")
+        except OSError as exc:
+            self._logger.warning(f"BestBuy: could not save availability payload: {exc}")
+
     def _build_oxylabs_us_proxy_url(self, sessid: str | None = None) -> Optional[str]:
         """Build an Oxylabs Residential URL pinned to a US sticky session.
 
@@ -1254,46 +1311,150 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             if avail is None:
                 self._write_unavailable(idx, result, sku)
                 return row
-            self._write_row(idx, avail, brand)
+            self._write_row(idx, avail, brand, parse_sku_fulfillment(result, sku))
             return row
 
+    # Extended availability columns -> default value when the column is created.
+    EXTENDED_AVAILABILITY_COLUMNS: dict[str, Any] = {
+        'availabilityTypes': None,
+        'condition': None,
+        'pickupMinHours': None,
+        'pickupMinDate': None,
+        'pickupMaxDate': None,
+        'pickupFulfillDate': None,
+        'pickupDisplayDateType': None,
+        'pickupInStoreOnly': None,
+        'pickupPreorderable': None,
+        'pickupBackordered': None,
+        'pickupWhenAvailable': None,
+        'pickupRestrictions': None,
+        'storeDistance': None,
+        'shippingEligible': None,
+        'shippingZipcode': None,
+        'shippingBackordered': None,
+        'shippingInStoreOnly': None,
+        'shippingPreorderable': None,
+        'fulfillByVendor': None,
+        'directToBoxEligible': None,
+        'shippingSameDay': None,
+        'shippingDefaultOption': None,
+        'shippingFastestDate': None,
+        'shippingFastestPrice': None,
+        'shippingFreeDate': None,
+        'shippingOptions': None,
+        'deliveryEligible': None,
+        'deliverable': None,
+        'deliveryBackordered': None,
+        'deliveryInStoreOnly': None,
+        'installationEligible': None,
+        'fulfillment_data': None,
+    }
+
+    def _ensure_extended_columns(self) -> None:
+        """Create any missing extended availability column (idempotent, object dtype)."""
+        missing = [c for c in self.EXTENDED_AVAILABILITY_COLUMNS if c not in self.data.columns]
+        for column in missing:
+            self.data[column] = pd.Series(
+                [self.EXTENDED_AVAILABILITY_COLUMNS[column]] * len(self.data),
+                index=self.data.index, dtype=object,
+            )
+
     def _write_unavailable(self, idx: Any, data: dict, sku: str) -> None:
-        """No fulfillment info in the response: write only ``enabled`` + ``checked`` (legacy behaviour)."""
-        fo = (data.get('data') or {}).get('fulfillmentOptions') or {}
-        states = fo.get('buttonStates') or []
-        state = ''
-        for bs in states:
-            if (bs.get('skuId') or bs.get('sku')) == sku:
-                state = bs.get('buttonState') or ''
-                break
-        if not state and states:
-            state = states[0].get('buttonState') or ''
-        self.data.loc[idx, 'enabled'] = state not in NOT_ENABLED_STATES
+        """No per-store pickup info in the response: write the SKU-level columns and ``checked``.
+
+        ``enabled`` still comes from the button state (legacy behaviour); shipping / delivery
+        columns and ``fulfillment_data`` are filled when the payload carries them, so a
+        ship-only SKU is still fully described.
+        """
+        info = parse_sku_fulfillment(data, sku)
+        self.data.loc[idx, 'enabled'] = info.enabled
+        self._write_sku_columns(idx, None, info)
         self.data.loc[idx, 'checked'] = True
 
-    def _write_row(self, idx: Any, avail: StoreAvailability, brand: Any) -> None:
+    def _write_sku_columns(self, idx: Any, avail: StoreAvailability | None,
+                           info: SkuFulfillment) -> None:
+        """Write the SKU-level (button state, shipping, delivery) columns plus the JSON slice.
+
+        Args:
+            idx: DataFrame index label.
+            avail: Per-store availability when known (used for the summary / JSON), else None.
+            info: SKU-level fulfillment info parsed from the same response.
+        """
+        self._ensure_extended_columns()
+        d = self.data
+        types = availability_types(avail, info)
+        d.at[idx, 'availabilityTypes'] = ','.join(types) if types else None
+        d.at[idx, 'condition'] = (avail.condition if avail and avail.condition else None) or info.condition
+        d.at[idx, 'shippingEligible'] = info.shipping_eligible
+        d.at[idx, 'shippingZipcode'] = info.shipping_zipcode
+        d.at[idx, 'shippingBackordered'] = info.shipping_backordered
+        d.at[idx, 'shippingInStoreOnly'] = info.shipping_in_store_only
+        d.at[idx, 'shippingPreorderable'] = info.shipping_preorderable
+        d.at[idx, 'fulfillByVendor'] = info.fulfill_by_vendor
+        d.at[idx, 'directToBoxEligible'] = info.direct_to_box_eligible
+        d.at[idx, 'shippingSameDay'] = info.shipping_same_day
+        d.at[idx, 'shippingDefaultOption'] = info.shipping_default_option
+        d.at[idx, 'shippingFastestDate'] = info.shipping_fastest_date
+        d.at[idx, 'shippingFastestPrice'] = info.shipping_fastest_price
+        d.at[idx, 'shippingFreeDate'] = info.shipping_free_date
+        d.at[idx, 'shippingOptions'] = (
+            [o.model_dump() for o in info.shipping_options] if info.shipping_options else None
+        )
+        d.at[idx, 'deliveryEligible'] = info.delivery_eligible
+        d.at[idx, 'deliverable'] = info.deliverable
+        d.at[idx, 'deliveryBackordered'] = info.delivery_backordered
+        d.at[idx, 'deliveryInStoreOnly'] = info.delivery_in_store_only
+        installation = [
+            x for x in (
+                avail.installation_eligible if avail else None,
+                info.shipping_installation_eligible,
+                info.delivery_installation_eligible,
+            ) if x is not None
+        ]
+        d.at[idx, 'installationEligible'] = any(installation) if installation else None
+        d.at[idx, 'fulfillment_data'] = fulfillment_slice(avail, info)
+
+    def _write_row(self, idx: Any, avail: StoreAvailability, brand: Any,
+                   info: SkuFulfillment | None = None) -> None:
         """Write the output-contract columns for one DataFrame row and set checked=True.
 
         Args:
             idx: DataFrame index label.
             avail: Normalized availability for that row's (store, sku).
             brand: Brand value to copy into the row.
+            info: SKU-level fulfillment info (button state, shipping, delivery) parsed from the
+                same response. When omitted only the per-store columns are written.
         """
         is_in_store_pickup = (
             avail.pickup_eligible and avail.instore_inventory and avail.fulfillment_type == 'PICKUP'
         )
-        self.data.loc[idx, 'enabled'] = avail.enabled
-        self.data.loc[idx, 'locationId'] = avail.store.get('storeId') or avail.store_id
-        self.data.at[idx, 'brand'] = brand
-        self.data.at[idx, 'location_data'] = avail.store
-        self.data.loc[idx, 'availability'] = avail.fulfillment_type
-        self.data.loc[idx, 'inStoreAvailability'] = is_in_store_pickup
-        self.data.loc[idx, 'inStoreAvailable'] = is_in_store_pickup
-        self.data.loc[idx, 'onShelfDisplay'] = avail.instore_inventory
-        self.data.loc[idx, 'pickupEligible'] = avail.pickup_eligible
-        self.data.loc[idx, 'availablePickupQuantity'] = avail.quantity if avail.pickup_eligible else 0
-        self.data.loc[idx, 'availableInStoreQuantity'] = avail.quantity if is_in_store_pickup else 0
-        self.data.loc[idx, 'checked'] = True
+        self._ensure_extended_columns()
+        d = self.data
+        d.loc[idx, 'enabled'] = avail.enabled
+        d.loc[idx, 'locationId'] = avail.store.get('storeId') or avail.store_id
+        d.at[idx, 'brand'] = brand
+        d.at[idx, 'location_data'] = avail.store
+        d.loc[idx, 'availability'] = avail.fulfillment_type
+        d.loc[idx, 'inStoreAvailability'] = is_in_store_pickup
+        d.loc[idx, 'inStoreAvailable'] = is_in_store_pickup
+        d.loc[idx, 'pickupEligible'] = avail.pickup_eligible
+        d.loc[idx, 'availablePickupQuantity'] = avail.quantity if avail.pickup_eligible else 0
+        d.loc[idx, 'availableInStoreQuantity'] = avail.quantity if is_in_store_pickup else 0
+        # pickup detail
+        d.at[idx, 'pickupMinHours'] = avail.min_pickup_hours
+        d.at[idx, 'pickupMinDate'] = avail.min_date
+        d.at[idx, 'pickupMaxDate'] = avail.max_date
+        d.at[idx, 'pickupFulfillDate'] = avail.fulfill_date
+        d.at[idx, 'pickupDisplayDateType'] = avail.display_date_type
+        d.at[idx, 'pickupInStoreOnly'] = avail.in_store_only
+        d.at[idx, 'pickupPreorderable'] = avail.preorderable
+        d.at[idx, 'pickupBackordered'] = avail.backordered
+        d.at[idx, 'pickupWhenAvailable'] = avail.when_available_flag
+        d.at[idx, 'pickupRestrictions'] = avail.restrictions
+        d.at[idx, 'storeDistance'] = avail.distance
+        if info is not None:
+            self._write_sku_columns(idx, avail, info)
+        d.loc[idx, 'checked'] = True
 
     def column_exists(self, column: str, default_val: Any = None):
         if column not in self.data.columns:
@@ -1501,12 +1662,13 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
         self.column_exists('locationId')
         self.column_exists('availability')
         self.column_exists('inStoreAvailability')
-        self.column_exists('onShelfDisplay', False)
+        self.column_exists('onShelfDisplay')  # not provided by the API: NULL unless set upstream
         self.column_exists('availableInStoreQuantity', 0)
         self.column_exists('inStoreAvailable', False)
         self.column_exists('pickupEligible', False)
         self.column_exists('availablePickupQuantity', 0)
         self.column_exists('enabled', False)
+        self._ensure_extended_columns()
         self.data['checked'] = False  # Add 'checked' flag column
         self._logger.info(
             f"BestBuy availability: {len(self.data)} rows, bulk_mode={self.bulk_mode}"
@@ -1591,11 +1753,12 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             return
         by_store, returned, _max = parse_fulfillment(data, sku, store_id)
         avail = by_store.get(store_id) or (by_store.get(returned[0]) if returned else None)
+        info = parse_sku_fulfillment(data, sku)
         for idx in idxs:
             if avail is None:
                 self._write_unavailable(idx, data, sku)
             else:
-                self._write_row(idx, avail, self.data.at[idx, 'brand'])
+                self._write_row(idx, avail, self.data.at[idx, 'brand'], info)
 
     def _apply_response(self, sku: str, seed: str, data: dict, targets: dict[str, Target]) -> set[str]:
         """Parse one nearby response, write every covered target row, update coverage.
@@ -1604,13 +1767,14 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             The target store ids covered by the response.
         """
         by_store, returned, max_dist = parse_fulfillment(data, sku, seed)
+        info = parse_sku_fulfillment(data, sku)
         covered: set[str] = set()
         for sid, avail in by_store.items():
             if sid not in targets:
                 continue
             idxs = self._row_index.get((sid, sku), [])
             for idx in idxs:
-                self._write_row(idx, avail, self.data.at[idx, 'brand'])
+                self._write_row(idx, avail, self.data.at[idx, 'brand'], info)
             if idxs:
                 self._seed_resolved.add((sid, sku))
             covered.add(sid)

{flowtask-5.13.2 → flowtask-5.13.3}/flowtask/components/CopyToPg.py

@@ -37,6 +37,13 @@ from ..interfaces.copy_to import CopyTo, dtypes
 from ..utils.json import json_decoder, json_encoder
 
 
+# PostgreSQL type names (as reported by asyncpg) that store plain text.
+PG_TEXT_TYPES = frozenset({"text", "varchar", "bpchar", "character varying", "character", "name"})
+
+# Column names that collide with attributes used internally by the dynamic
+# asyncdb Model while rendering the CREATE TABLE statement.
+RESERVED_MODEL_COLUMNS = frozenset({"columns", "Meta"})
+
 # adding support for primary keys on raw tables
 pk_sentence = """ALTER TABLE {schema}.{table}
 ADD CONSTRAINT {schema}_{table}_pkey PRIMARY KEY({fields});"""
@@ -44,6 +51,43 @@ unique_sentence = """ALTER TABLE {schema}.{table}
 ADD CONSTRAINT unq_{schema}_{table} UNIQUE({fields});"""
 
 
+def quote_identifier(name: str) -> str:
+    """Quote a PostgreSQL identifier so its case is preserved.
+
+    Args:
+        name: Column name as it appears in the DataFrame.
+
+    Returns:
+        The name wrapped in double quotes, with embedded quotes escaped.
+    """
+    return '"' + name.replace('"', '""') + '"'
+
+
+def quote_ddl_columns(sql: str, columns: list[str]) -> str:
+    """Quote the column identifiers of an asyncdb-rendered ``CREATE TABLE``.
+
+    asyncdb emits one `` <name> <type> ...`` line per column with the name
+    unquoted, so PostgreSQL folds it to lowercase and a later ``COPY`` using
+    the DataFrame's exact (quoted) names fails with *Undefined Column*.
+
+    Args:
+        sql: DDL returned by ``Model.model(dialect="sql")``.
+        columns: Column names in the order they were given to the model.
+
+    Returns:
+        The same DDL with every column identifier double-quoted.
+    """
+    for col in columns:
+        sql = re.sub(
+            rf"^(\s*){re.escape(col)}(?=\s)",
+            lambda m, c=col: f"{m.group(1)}{quote_identifier(c)}",
+            sql,
+            count=1,
+            flags=re.MULTILINE,
+        )
+    return sql
+
+
 class CopyToPg(CopyTo):
     """
     CopyToPg
@@ -145,6 +189,17 @@ class CopyToPg(CopyTo):
         return total
 
     async def _create_table(self):
+        """Create the destination table from the DataFrame columns.
+
+        Builds a dynamic ``asyncdb`` Model from the DataFrame dtypes and
+        renders it as a ``CREATE TABLE`` statement. Column names that
+        collide with attributes the model machinery needs (``columns``,
+        ``Meta``) cannot be represented and raise a ``ComponentError``.
+
+        Raises:
+            ComponentError: when no primary key is defined, a column name
+                is reserved, or the DDL execution fails.
+        """
         _pk = self.create_table.get("pk", None)
         _unq = self.create_table.get("unique", None)
         _drop = self.create_table.get("drop", False)
@@ -154,6 +209,13 @@ class CopyToPg(CopyTo):
             )
         # extracting columns:
         columns = self.data.columns.tolist()
+        reserved = RESERVED_MODEL_COLUMNS.intersection(columns)
+        if reserved:
+            raise ComponentError(
+                f"CopyToPg: Error on Table Creation: column name(s) "
+                f"{sorted(reserved)} are reserved by the table model; "
+                "rename them before CopyToPg."
+            )
         cols = []
         for col in columns:
             if col in self._json_columns:
@@ -166,22 +228,37 @@ class CopyToPg(CopyTo):
                 f = (col, Optional[bytes], Field(default=None, nullable=True, db_type='bytea'))
             else:
                 datatype = self.data.dtypes[col]
-                try:
-                    t = dtypes[str(datatype)]
-                except KeyError:
-                    t = str
-                f = (col, t)
+                if is_datetime64_any_dtype(datatype):
+                    # any precision (ns/us/ms/s); tz-aware keeps the offset
+                    db_type = (
+                        "timestamp with time zone"
+                        if isinstance(datatype, pd.DatetimeTZDtype)
+                        else "timestamp without time zone"
+                    )
+                    f = (
+                        col,
+                        Optional[datetime.datetime],
+                        Field(default=None, nullable=True, db_type=db_type),
+                    )
+                else:
+                    try:
+                        t = dtypes[str(datatype)]
+                    except KeyError:
+                        t = str
+                    f = (col, t)
             cols.append(f)
         try:
             cls = Model.make_model(
                 name=self.tablename, schema=self.schema, fields=cols
             )
-            # Create model instance with None values for all columns
-            init_kwargs = {field[0]: None for field in cols}
-            mdl = cls(**init_kwargs)
-            
-            if sql := mdl.model(dialect="sql"):
-                print("SQL IS ", sql)
+            # Call the classmethod through its underlying function: a column
+            # named "model" becomes a field on the dynamic class and would
+            # shadow ``Model.model`` (resolving to None).
+            if sql := Model.model.__func__(cls, dialect="sql"):
+                # Quote identifiers so the table keeps the DataFrame's exact
+                # column names (the COPY path quotes them too).
+                sql = quote_ddl_columns(sql, columns)
+                self._logger.debug(f"Create Table SQL: {sql}")
                 async with await self._connection.connection() as conn:
                     if _drop is True:
                         result, error = await conn.execute(
@@ -198,7 +275,7 @@ class CopyToPg(CopyTo):
                     pk = pk_sentence.format(
                         schema=self.schema,
                         table=self.tablename,
-                        fields=",".join(_pk),
+                        fields=",".join(quote_identifier(c) for c in _pk),
                     )
                     _primary, error = await conn.execute(sentence=pk)
                     self._logger.debug(
@@ -209,7 +286,7 @@ class CopyToPg(CopyTo):
                         unique = unique_sentence.format(
                             schema=self.schema,
                             table=self.tablename,
-                            fields=",".join(_unq),
+                            fields=",".join(quote_identifier(c) for c in _unq),
                         )
                         _unique, error = await conn.execute(sentence=unique)
                         self._logger.debug(
@@ -284,6 +361,38 @@ class CopyToPg(CopyTo):
             self._logger.error(f"Error getting column types: {e}")
             return {}
 
+    def _coerce_text_columns(self, pg_column_types: list[dict] | dict) -> None:
+        """Stringify scalar values bound for columns the table stores as text.
+
+        ``object`` columns holding ints, floats, bools or dates (for example a
+        ``pickup_min_hours`` that is ``None`` on most rows) are created as
+        ``varchar`` but the binary COPY receives the raw Python value and
+        asyncpg rejects it with *expected str, got int*.
+
+        Args:
+            pg_column_types: Result of :meth:`_get_column_types`; the
+                ``{}`` fallback returned on lookup errors is ignored.
+        """
+        if not isinstance(pg_column_types, list):
+            return
+        text_cols = [
+            col["name"] for col in pg_column_types
+            if col.get("pg_type") in PG_TEXT_TYPES
+        ]
+        skip = set(self._json_columns) | set(self._array_columns) | set(self._vector_columns)
+        scalars = (bool, int, float, Decimal, np.generic, datetime.date, datetime.datetime, pd.Timestamp)
+        converted = []
+        for col in text_cols:
+            if col not in self.data.columns or col in skip:
+                continue
+            series = self.data[col]
+            mask = series.map(lambda v: isinstance(v, scalars))
+            if mask.any():
+                self.data[col] = series.where(~mask, series.map(str))
+                converted.append(col)
+        if converted:
+            self._logger.debug(f"CopyToPg: coerced to text for {self.schema}.{self.tablename}: {converted}")
+
     async def _copy_dataframe(self):
         # insert data directly into table
         columns = list(self.data.columns)
@@ -437,7 +546,6 @@ class CopyToPg(CopyTo):
                     async with await self._connection.connection() as conn:
                         # Get PostgreSQL column types
                         pg_column_types = await self._get_column_types(conn)
-                        # TODO: using the column types to refine the conversion
 
                     # Handle datetime columns - replace NaT with None
                     datetime_cols = self.data.select_dtypes(include=['datetime64[ns]', 'datetime64[ns, UTC]'])
@@ -494,6 +602,8 @@ class CopyToPg(CopyTo):
 
                     # 1️⃣ Turn every NaN / NA / NaT into Python None
                     self.data = self.data.astype(object).where(pd.notnull(self.data), None)
+                    # 2️⃣ Numbers / bools heading to text columns must be strings for COPY
+                    self._coerce_text_columns(pg_column_types)
                     tuples = list(zip(*map(self.data.get, self.data)))
 
                     async with await self._connection.connection() as conn:

flowtask-5.13.3/flowtask/documentation/generated/components/BestBuy.doc.json

@@ -0,0 +1,80 @@
+{
+  "name": "BestBuy",
+  "version": "1.0.0",
+  "category": "Other",
+  "description": "Extract Best Buy information (stores, products, reviews, product availability) by combining the Best Buy API with web scraping. ``type: availability`` checks per-store pickup availability through the GraphQL fulfillment endpoint over sticky Oxylabs US sessions. With ``bulk_mode: true`` it runs a greedy set-cover over the endpoint's nearby-store fan-out, so the call count drops from stores x skus to roughly seeds x skus.",
+  "examples": [
+    "BestBuy:\n      type: availability\n      brand: Epson\n      bulk_mode: true\n      storage_path: /home/ubuntu/symbits/bestbuy/\n      concurrency: 12"
+  ],
+  "schema": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "type": "object",
+    "title": "BestBuy",
+    "description": "Extract Best Buy information (stores, products, reviews, product availability) by combining the Best Buy API with web scraping. ``type: availability`` checks per-store pickup availability through the GraphQL fulfillment endpoint over sticky Oxylabs US sessions. With ``bulk_mode: true`` it runs a greedy set-cover over the endpoint's nearby-store fan-out, so the call count drops from stores x skus to roughly seeds x skus.",
+    "properties": {
+      "type": {
+        "type": "string",
+        "description": "Function to run: availability, product, reviews, products, products_by_brand, stores, product_api."
+      },
+      "brand": {
+        "type": "string",
+        "description": "Brand to filter or tag products with."
+      },
+      "product_info": {
+        "type": "string",
+        "description": "Fetch extra product information (default false)."
+      },
+      "use_proxy": {
+        "type": "string",
+        "description": "Opt in to the proxy for non-availability scraping (default false)."
+      },
+      "use_storage": {
+        "type": "string",
+        "description": "Reuse a same-day CSV result from storage_path (default false)."
+      },
+      "storage_path": {
+        "type": "string",
+        "description": "Directory for stored results and the default coverage file. Required when use_storage is true."
+      },
+      "chunk_size": {
+        "type": "string",
+        "description": "Rows per processing chunk (default 100)."
+      },
+      "task_parts": {
+        "type": "string",
+        "description": "Number of task partitions (default 10)."
+      },
+      "concurrency": {
+        "type": "string",
+        "description": "Max concurrent requests and sticky proxy-session pool size (default 12)."
+      },
+      "bulk_mode": {
+        "type": "string",
+        "description": "availability only: greedy set-cover over GraphQL nearby stores (default false = one call per row). The `enabled` column comes from the SKU button state returned with the seed call."
+      },
+      "coverage_file": {
+        "type": "string",
+        "description": "Coverage map JSON (default <storage_path>/bestbuy_coverage.json; in-memory when unset)."
+      },
+      "proxy_session_max_requests": {
+        "type": "string",
+        "description": "Requests before a sticky proxy session is rotated (default 50)."
+      },
+      "proxy_session_max_age": {
+        "type": "string",
+        "description": "Seconds before a sticky proxy session is rotated (default 540)."
+      },
+      "debug_payload": {
+        "type": "string",
+        "description": "availability only: log the full GraphQL JSON of every call; with storage_path also dump it to <storage_path>/payloads/ (default false)."
+      }
+    },
+    "required": [
+      "type"
+    ],
+    "x-flowtask-io": {
+      "consumes": "dataframe",
+      "produces": "dataframe"
+    }
+  }
+}