flowtask 5.13.3__tar.gz → 5.13.5__tar.gz

{flowtask-5.13.3 → flowtask-5.13.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.3
+Version: 5.13.5
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0
@@ -22,14 +22,14 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Framework :: AsyncIO
-Requires-Python: <3.14,>=3.11
+Requires-Python: <3.14,>=3.11.4
 Description-Content-Type: text/markdown
 License-File: LICENSE
 Requires-Dist: borax==3.5.0
 Requires-Dist: PyDrive==1.3.1
 Requires-Dist: chardet==5.2.0
 Requires-Dist: aiohttp-jinja2==1.6
-Requires-Dist: asyncssh[bcrypt,fido2,pkcs11,pyopenssl]==2.23.0
+Requires-Dist: asyncssh[bcrypt,fido2,pkcs11,pyopenssl]<3,>=2.24.0
 Requires-Dist: pyxlsb==1.0.10
 Requires-Dist: python-calamine==0.2.3
 Requires-Dist: pyecharts==2.0.8
@@ -46,7 +46,7 @@ Requires-Dist: adal==1.2.7
 Requires-Dist: xlrd>=2.0.2
 Requires-Dist: xmltodict<1.0.0,>=0.14.2
 Requires-Dist: zeep[async]>=4.3.3
-Requires-Dist: nltk>=3.9.3
+Requires-Dist: nltk>=3.10.3
 Requires-Dist: jdcal==1.4.1
 Requires-Dist: html5lib==1.1
 Requires-Dist: shapely>=2.1.2
@@ -70,7 +70,7 @@ Requires-Dist: aiofiles>=24.1.0
 Requires-Dist: filetype==1.2.0
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: aioftp==0.23.1
-Requires-Dist: py7zr==0.22.0
+Requires-Dist: py7zr<2,>=1.1.3
 Requires-Dist: rarfile==4.2
 Requires-Dist: python-pptx==1.0.2
 Requires-Dist: google-auth>=2.35.0
@@ -101,7 +101,7 @@ Requires-Dist: pyheif==0.8.0
 Requires-Dist: filetype==1.2.0
 Requires-Dist: exif==1.6.1
 Requires-Dist: pillow-avif-plugin==1.5.2
-Requires-Dist: pillow-heif==0.22.0
+Requires-Dist: pillow-heif<2,>=1.3.0
 Requires-Dist: pgvector==0.4.1
 Requires-Dist: proxylists>=0.14.0
 Requires-Dist: async-notify[all]>=1.4.0

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/BestBuy.py

@@ -104,11 +104,12 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
     | storage_path               | No  | Directory for stored results and the default coverage file. Required when use_storage is true. |
     | chunk_size                 | No  | Rows per processing chunk (default 100). |
     | task_parts                 | No  | Number of task partitions (default 10). |
-    | concurrency                | No  | Max concurrent requests and sticky proxy-session pool size (default 12). |
+    | concurrency                | No  | Max concurrent requests and sticky proxy-session pool size (default 6). |
     | bulk_mode                  | No  | availability only: greedy set-cover over GraphQL nearby stores (default false = one call per row). The `enabled` column comes from the SKU button state returned with the seed call. |
     | coverage_file              | No  | Coverage map JSON (default <storage_path>/bestbuy_coverage.json; in-memory when unset). |
     | proxy_session_max_requests | No  | Requests before a sticky proxy session is rotated (default 50). |
     | proxy_session_max_age      | No  | Seconds before a sticky proxy session is rotated (default 540). |
+    | request_timeout            | No  | Hard deadline in seconds for one availability request through the proxy (default 45). |
     | debug_payload              | No  | availability only: log the full GraphQL JSON of every call; with storage_path also dump it to <storage_path>/payloads/ (default false). |
 
     ``type: availability`` output columns (one row per store × sku). Legacy contract:
@@ -140,7 +141,7 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
       brand: Epson
       bulk_mode: true
       storage_path: /home/ubuntu/symbits/bestbuy/
-      concurrency: 12
+      concurrency: 6
     ```
     """
     _version = "1.0.0"
@@ -205,19 +206,20 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             "User-Agent": random.choice(ua),
             **self.headers
         }
-        # Cap concurrent storeAvailability requests so the Oxylabs gateway
-        # doesn't return 522 from a single residential exit IP under load.
-        # Each request rotates to a fresh sticky session, so the bottleneck
-        # is gateway capacity rather than per-IP load — 12 is safe and gives
-        # roughly 3x throughput over the original semaphore=10 (which was
-        # bottlenecked by 403s, not network capacity).
-        self.semaphore = asyncio.Semaphore(kwargs.get('concurrency', 12))
+        # Cap concurrent storeAvailability requests: under load the Oxylabs
+        # gateway fails the CONNECT tunnel with 522 (httpx.ProxyError) when
+        # sticky residential exit nodes can't reach bestbuy.com. 12 produced
+        # those 522s in production; 6 keeps the pool smaller and steadier.
+        self.semaphore = asyncio.Semaphore(kwargs.get('concurrency', 6))
         # Selenium driver is not thread-safe: serialize all execute_async_script calls
         self._selenium_semaphore = asyncio.Semaphore(1)
         # FEAT-561: sticky Oxylabs session pool (K = concurrency)
-        self._pool_size: int = kwargs.get('concurrency', 12)
+        self._pool_size: int = kwargs.get('concurrency', 6)
         self.proxy_session_max_requests: int = kwargs.get('proxy_session_max_requests', 50)
         self.proxy_session_max_age: float = kwargs.get('proxy_session_max_age', 540)
+        # Hard per-request deadline: a stalled Oxylabs CONNECT can outlive httpx's
+        # per-operation timeouts and freeze the sequential bulk seed loop.
+        self.request_timeout: float = kwargs.get('request_timeout', 45)
         # Debug: dump the raw GraphQL availability payload (log + optional file)
         self.debug_payload: bool = kwargs.get('debug_payload', False)
         self._session_pool: list[ProxySession] = []
@@ -993,11 +995,19 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
                     if attempt < 3:
                         continue
                     return None
-                r = await client.get(url, headers=api_headers)
+                r = await asyncio.wait_for(
+                    client.get(url, headers=api_headers), timeout=self.request_timeout
+                )
             except (httpx.ConnectError, httpx.ConnectTimeout, httpx.ReadTimeout,
-                    httpx.RemoteProtocolError) as ex:
+                    httpx.RemoteProtocolError, httpx.ProxyError, asyncio.TimeoutError) as ex:
+                # ProxyError: the Oxylabs CONNECT tunnel failed (e.g. "522 status code 522"
+                # when the sticky exit node can't reach bestbuy.com) — retry on a fresh session.
                 await self._release_session(session, status=None, error=ex)
                 if attempt < 3:
+                    self._logger.warning(
+                        f"GraphQL fulfillment attempt {attempt}/3 failed for sku={sku} "
+                        f"store={store_id}: {ex!r}; retrying."
+                    )
                     await asyncio.sleep(0.5 * attempt)
                     continue
                 self._logger.warning(
@@ -1018,6 +1028,10 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
                     await self._dump_availability_payload(payload, sku, store_id, nearby)
                 return payload
             if r.status_code in (522, 502, 503, 504) and attempt < 3:
+                self._logger.warning(
+                    f"GraphQL fulfillment attempt {attempt}/3 for sku={sku} store={store_id}: "
+                    f"HTTP {r.status_code}; retrying."
+                )
                 await asyncio.sleep(0.5 * attempt)
                 continue
             if r.status_code == 403 and attempt < 3:
@@ -1244,7 +1258,8 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             status in (403, 429)
             or (status is not None and status >= 500)
             or isinstance(error, (
-                httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError
+                httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError,
+                httpx.ProxyError, asyncio.TimeoutError,
             ))
         )
         if burn and not session.burned:
@@ -1719,16 +1734,25 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
             self._row_index.setdefault((sid, sku), []).append(idx)
 
         if targets and skus:
+            self._logger.info(
+                f"BestBuy bulk: {len(skus)} SKUs x {len(targets)} stores; "
+                f"resolving sku {skus[0]} (1/{len(skus)}) dynamically."
+            )
             order = await self._resolve_sku_dynamic(
                 skus[0], {t for t in targets if (t, skus[0]) in self._row_index}, targets
             )
-            for sku in skus[1:]:
+            for n, sku in enumerate(skus[1:], start=2):
+                self._logger.info(
+                    f"BestBuy bulk: sku {sku} ({n}/{len(skus)}); seeds={self._num_seeds}, "
+                    f"http_calls={self._num_iterations}."
+                )
                 uncovered = await self._resolve_sku_replay(sku, order, targets)
                 if uncovered:
                     await self._resolve_sku_dynamic(sku, uncovered, targets)
 
             # Fallback: every (store, sku) not covered by a seed response.
             leftovers = [pair for pair in self._row_index if pair not in self._seed_resolved]
+            self._logger.info(f"BestBuy bulk: {len(leftovers)} (store, sku) pairs to fallback.")
             await self._processing_tasks([self._fallback_pair(*pair) for pair in leftovers])
 
         total_pairs = len(self._row_index)
@@ -1803,6 +1827,9 @@ class BestBuy(FlowComponent, SeleniumService, HTTPService):
                 pending -= got
                 covered |= got | {seed}
             # on failure the neighbours stay pending; the seed goes to fallback
+            self._logger.debug(
+                f"BestBuy bulk: sku={sku} seed={seed} ok={bool(data)}; {len(pending)} stores pending."
+            )
         return order
 
     async def _resolve_sku_replay(self, sku: str, seeds: list[str],

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/FileRead.py

@@ -4,6 +4,7 @@ from pathlib import Path, PurePath
 import io
 import aiofiles
 from ..exceptions import FileNotFound, FileError
+from ..conf import TASK_PATH
 from ..utils.json import json_decoder
 from ..interfaces.file_base import FileBase
 from ..interfaces.dataframes.pandas import PandasDataframe
@@ -69,8 +70,12 @@ class FileRead(FileBase, PandasDataframe):
     async def start(self, **kwargs):
         """Check for File and Directory information."""
         if not hasattr(self, "directory"):
-            if self._use_taskstorage:
-                self.directory = self._taskstore.get_path().joinpath(self._program, 'files')
+            # resolve <program>/files from the task storage the task was loaded
+            # from (e.g. --storage=private); storages without a path use TASK_PATH.
+            base = self._taskstore.get_path() if self._taskstore else None
+            task_files = Path(base or TASK_PATH, self._program, "files")
+            if self._use_taskstorage or task_files.is_dir():
+                self.directory = task_files
             else:
                 self.directory = self._filestore.get_directory('')
         await super().start(**kwargs)
@@ -96,7 +101,7 @@ class FileRead(FileBase, PandasDataframe):
                     )
         elif hasattr(self, "filename"):
             if isinstance(self.filename, str):
-                file = Path(file)
+                file = Path(self.directory).joinpath(self.filename)
             elif isinstance(self.filename, PurePath):
                 file = self.filename.resolve()
             else:

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/NetworkNinja/models/forms.py

@@ -7,6 +7,7 @@ from datamodel import BaseModel, Field
 from datamodel.parsers.json import json_encoder
 from querysource.conf import default_dsn, async_default_dsn
 from querysource.outputs.tables import PgOutput
+from ....conf import pg_server_settings
 from .abstract import AbstractPayload
 from .organization import Organization
 from .client import Client
@@ -465,7 +466,9 @@ class FormData(AbstractPayload):
 
         logger.debug(f"Processing form update: {previous_form} -> {new_form}")
 
-        async with await AsyncDB('pg', dsn=default_dsn).connection() as conn:
+        async with await AsyncDB(
+            'pg', dsn=default_dsn, server_settings=pg_server_settings('NetworkNinja')
+        ).connection() as conn:
             try:
                 # Use PL/pgSQL to handle the UPDATE exception properly
                 transaction = f"""

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/NetworkNinja/router.py

@@ -34,7 +34,8 @@ from ...interfaces.http import HTTPService
 from ...conf import (
     NETWORKNINJA_API_KEY,
     NETWORKNINJA_BASE_URL,
-    NETWORKNINJA_ENV
+    NETWORKNINJA_ENV,
+    pg_pool_options,
 )
 from .models import NetworkNinja_Map, NN_Order
 from .models.abstract import AbstractPayload
@@ -491,8 +492,13 @@ class NetworkNinja(HTTPService, FlowComponent):
         return final_form_ids
 
     async def close(self):
-        if self._pool:
-            await self._pool.close()
+        """Release the PostgreSQL pool (safe to call more than once)."""
+        pool, self._pool = self._pool, None
+        if pool:
+            try:
+                await pool.close()
+            except Exception as exc:  # pylint: disable=W0718
+                self._logger.warning(f"Error closing NetworkNinja pool: {exc!s}")
 
     def _evaluate_input(self):
         if self.previous or self.input is not None:
@@ -526,13 +532,31 @@ class NetworkNinja(HTTPService, FlowComponent):
         return True
 
     async def run(self):
-        """Run NetworkNinja Router."""
+        """Run NetworkNinja Router.
+
+        Opens a short-lived PostgreSQL pool for the duration of the action.
+        The task runner only calls ``close()`` when ``run()`` returns, so the
+        pool is released here as well whenever the action raises (for example
+        ``DataNotFound`` from ``get_batches`` on an empty NetworkNinja queue).
+        """
+        self._pool = AsyncPool(
+            'pg',
+            dsn=default_dsn,
+            **pg_pool_options(suffix='NetworkNinja'),
+        )
+        await self._pool.connect()
+        try:
+            return await self._run_action()
+        except BaseException:
+            await self.close()
+            raise
+
+    async def _run_action(self):
+        """Execute the configured NetworkNinja action and return its result."""
         tasks = []
         fn = getattr(self, self._action)
         self.add_metric("ACTION", self._action)
         self._result = {}
-        self._pool = AsyncPool('pg', dsn=default_dsn, max_clients=1000)
-        await self._pool.connect()
         if self._action == 'get_batch':
             try:
                 result = await fn()

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/OpenWithBase.py

@@ -19,7 +19,7 @@ from ..exceptions import FileNotFound, ComponentError
 from ..utils.mail import MailMessage
 from ..utils import check_empty
 from ..parsers.maps import open_map, open_model
-from ..conf import TASK_PATH, DEFAULT_ENCODING
+from ..conf import TASK_PATH, DEFAULT_ENCODING, pg_server_settings
 from ..interfaces.flow import FlowComponent
 from ..utils.constants import excel_based
 
@@ -224,7 +224,12 @@ class OpenWithBase(FlowComponent):
                         event_loop = asyncio.get_running_loop()
                     except RuntimeError:
                         event_loop = asyncio.get_event_loop()
-                    db = AsyncDB("pg", dsn=asyncpg_url, loop=event_loop)
+                    db = AsyncDB(
+                        "pg",
+                        dsn=asyncpg_url,
+                        loop=event_loop,
+                        server_settings=pg_server_settings(self.__class__.__name__),
+                    )
                     async with await db.connection() as conn:
                         result, error = await conn.query(discover)
                         if error:

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/PlanogramCompliance.py

@@ -8,10 +8,12 @@ from pathlib import Path
 from PIL import Image
 from asyncdb import AsyncDB
 from querysource.conf import default_dsn
+from ..conf import pg_server_settings
 import re
 from parrot.conf import PLANOGRAM_FOLDER
 from parrot.clients.google import GoogleGenAIClient, GoogleModel
 from parrot_pipelines.models import PlanogramConfig, EndcapGeometry
+from parrot_pipelines.planogram.plan import PlanogramCompliance as PlanogramPipeline
 from ..interfaces.flow import FlowComponent
 from ..interfaces.pipelines.parrot import AIPipeline
 from ..exceptions import ComponentError, ConfigError
@@ -54,6 +56,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
     | id_column              | No  | DataFrame column used as per-row image id (default ``photo_id``)|
     | reference_images       | No  | Mapping key -> image file or directory under ``images/``;       |
     |                        |     | overrides the ``reference_images`` column of the row            |
+    |                        |     | (ignored for types without references, e.g. ``ink_wall``)       |
     | llm_backend            | No  | LLM backend as ``provider:model``; overrides the ``llm_backend``|
     |                        |     | column of the row (default: the ai-parrot package default)      |
     | llm_config             | No  | Arguments of an explicit Google GenAI client; when given it wins|
@@ -69,8 +72,10 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
     Returns the input DataFrame plus: ``planogram_id``, ``overall_compliance_score``,
     ``overall_compliant``, ``compliance_analysis``, ``compliance_analysis_markdown``,
     ``compliance_analysis_json`` (includes assessment status, coverage, strict score and
-    pipeline errors), ``compliance_by_shelf``, ``detections``, ``overlay_image`` and
-    ``overlay_path``.
+    pipeline errors and ``products_found``), ``compliance_by_shelf``, ``detections``,
+    ``products_found`` (one entry per expected slot: model, sku, brand, found, misplaced,
+    status, confidence, facings, facings_found, observed; empty without ai-parrot FEAT-645),
+    ``overlay_image`` and ``overlay_path``.
 
         Example:
 
@@ -198,7 +203,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                 "planogram_config_table is not specified"
             )
         planogram_query = f"SELECT * FROM {self._planogram_config_table} WHERE config_name = $1"
-        db = AsyncDB('pg', dsn=default_dsn)
+        db = AsyncDB('pg', dsn=default_dsn, server_settings=pg_server_settings('PlanogramCompliance'))
         async with await db.connection() as conn:
             result = await conn.fetch_one(planogram_query, self._planogram_name)
         if not result:
@@ -253,19 +258,27 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
             self.confidence_threshold = row['confidence_threshold']
         if row.get('detection_model'):
             self.detection_model = row['detection_model']
-        # Task reference images win; otherwise use the ones stored in the row.
-        references = reference_images or self._reference_images
-        if not references:
-            references = self._db_reference_images(
-                self._decode_json_column(row, 'reference_images')
+        # DB value takes precedence, the task value is the fallback.
+        planogram_type = row.get('planogram_type') or self._planogram_type
+        layout_config = self._decode_json_column(row, 'planogram_config') or {}
+        references: Dict[str, Union[Path, List[Path]]] = {}
+        if self._uses_reference_images(planogram_type, layout_config):
+            # Task reference images win; otherwise use the ones stored in the row.
+            references = reference_images or self._prepare_task_references()
+            if not references:
+                references = self._db_reference_images(
+                    self._decode_json_column(row, 'reference_images')
+                )
+        else:
+            self._logger.debug(
+                f"Planogram type '{planogram_type}' does not use reference images; skipping them"
             )
         try:
             planogram_config = PlanogramConfig(
                 planogram_id=row.get('planogram_id'),
                 config_name=self._planogram_name,
-                # DB value takes precedence, the task value is the fallback.
-                planogram_type=row.get('planogram_type') or self._planogram_type,
-                planogram_config=self._decode_json_column(row, 'planogram_config') or {},
+                planogram_type=planogram_type,
+                planogram_config=layout_config,
                 endcap_geometry=geometry,
                 roi_detection_prompt=row.get('roi_detection_prompt') or None,
                 object_identification_prompt=row.get('object_identification_prompt') or None,
@@ -282,6 +295,40 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
         self._logger.debug('Planogram config loaded: %s', planogram_config)
         return planogram_config
 
+    def _uses_reference_images(self, planogram_type: str, layout_config: Dict[str, Any]) -> bool:
+        """Ask the pipeline whether this planogram type consumes reference images.
+
+        Args:
+            planogram_type: Resolved planogram type of the configuration.
+            layout_config: The ``planogram_config`` column (may override ``layout_profile.references``).
+
+        Returns:
+            False for types that identify products without references (e.g. ``ink_wall``); True
+            when the installed ai-parrot predates ``uses_reference_images``.
+
+        Raises:
+            ConfigError: Unknown planogram type or an invalid ``layout_profile`` override.
+        """
+        uses_references = getattr(PlanogramPipeline, 'uses_reference_images', None)
+        if uses_references is None:
+            return True
+        try:
+            return bool(uses_references(planogram_type, layout_config))
+        except ValueError as e:
+            raise ConfigError(
+                f"Invalid planogram configuration: {e}"
+            ) from e
+
+    def _prepare_task_references(self) -> Dict[str, Union[Path, List[Path]]]:
+        """Resolve the task ``reference_images`` once, only for types that use them.
+
+        Returns:
+            The cached mapping of catalogue key to a Path or a list of Paths.
+        """
+        if not self._reference_images and self.reference_images:
+            self._reference_images = self._prepare_reference_images()
+        return self._reference_images
+
     @staticmethod
     def _decode_json_column(row: Dict[str, Any], column: str) -> Any:
         """Return a JSONB column of the row as a Python object.
@@ -379,16 +426,12 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                 f"Image column '{self.image_column}' not found in DataFrame"
             )
 
-        # Prepare reference images list - convert to string paths as expected by pipeline
-        self._reference_images = self._prepare_reference_images()
-
         # Initialize LLM for pipeline
         self._llm = self._initialize_llm()
 
-        # Generate Planogram Configuration:
-        self.planogram_config = await self.get_planogram_config(
-            reference_images=self._reference_images
-        )
+        # Generate Planogram Configuration; reference images are resolved there,
+        # only when the planogram type uses them.
+        self.planogram_config = await self.get_planogram_config()
 
         self._logger.info(
             f"PlanogramPipeline started with {len(self.data)} rows to process"
@@ -472,6 +515,26 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
         """
         return result.get('compliance_results') or result.get('step3_compliance_results') or []
 
+    @staticmethod
+    def _products_found(result: Dict[str, Any]) -> List[Dict[str, Any]]:
+        """Return the slot presence list of a pipeline result as JSON-safe dicts.
+
+        Args:
+            result: Pipeline result dictionary.
+
+        Returns:
+            ``model_dump(mode="json")`` of each ``products_found`` item (dict items kept as-is);
+            an empty list when the key is missing (ai-parrot without FEAT-645).
+        """
+        items = result.get('products_found') or []
+        products: List[Dict[str, Any]] = []
+        for item in items:
+            if hasattr(item, 'model_dump'):
+                products.append(item.model_dump(mode='json'))
+            elif isinstance(item, dict):
+                products.append(item)
+        return products
+
     def _create_markdown_report(self, result: Dict[str, Any], overlay_path: str = "") -> str:
         """Generate a markdown report from pipeline results."""
         md = []
@@ -519,6 +582,29 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                         md.append(f"    - Matched: {text_result.matched_features}")
             md.append("")
 
+        products_found = self._products_found(result)
+        if products_found:
+            found = [p for p in products_found if p.get('found') is True]
+            not_found = [p for p in products_found if p.get('found') is False]
+            unknown = [p for p in products_found if p.get('found') is None]
+            misplaced = [p for p in products_found if p.get('misplaced')]
+            md.append("## Slot Presence")
+            md.append(
+                f"- **Found:** {len(found)} | **Not found:** {len(not_found)} "
+                f"| **Unknown:** {len(unknown)} | **Misplaced:** {len(misplaced)}"
+            )
+            if not_found:
+                md.append("- Not found:")
+                for p in not_found:
+                    line = f"  - {p.get('model') or p.get('display_name')}"
+                    if p.get('observed'):
+                        line += f" (observed: {p.get('observed')})"
+                    md.append(line)
+            if unknown:
+                md.append("- Unknown:")
+                for p in unknown:
+                    md.append(f"  - {p.get('model') or p.get('display_name')}")
+            md.append("")
         errors = result.get('errors') or []
         if errors:
             md.append("## Pipeline Errors")
@@ -602,6 +688,14 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
             # list of ComplianceResult objects
             compliance_results = self._compliance_results(result)
             pipeline_errors = list(result.get('errors') or [])
+            products_found = self._products_found(result)
+            if 'products_found' not in result:
+                self._logger.debug(
+                    f"Row {row_index}: pipeline result has no 'products_found' (ai-parrot without FEAT-645)"
+                )
+            skipped = len(result.get('products_found') or []) - len(products_found)
+            if skipped > 0:
+                self._logger.debug(f"Row {row_index}: skipped {skipped} unsupported 'products_found' item(s)")
             if pipeline_errors:
                 self._logger.warning(
                     f"Row {row_index}: pipeline reported errors: {pipeline_errors}"
@@ -618,6 +712,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                 'detection_source': result.get('detection_source'),
                 'resolved_backend': result.get('resolved_backend'),
                 'errors': pipeline_errors,
+                'products_found': products_found,
                 'step3_compliance_results': [
                     {
                         'shelf_level': cr.shelf_level,
@@ -722,6 +817,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                 # Add the new fields to the return dictionary
                 'compliance_by_shelf': compliance_by_shelf,
                 'detections': detections,
+                'products_found': products_found,
                 'status': 'success',
                 'error': None
             }
@@ -743,6 +839,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
                 # Add new fields with error state
                 'compliance_by_shelf': {"error": str(e)},
                 'detections': [{"error": str(e)}],
+                'products_found': [],
                 'status': 'error',
                 'error': str(e)
             }
@@ -772,13 +869,14 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
             # NEW: Initialize columns for the new data
             'compliance_by_shelf': None,
             'detections': None,
+            'products_found': None,
             'overlay_path': ""
         }
 
         for col, default in new_cols.items():
             # Use pd.NA for object types to allow for proper handling of missing values
             enhanced_df[col] = pd.NA if default is None else default
-            if col in ['overlay_image', 'compliance_by_shelf', 'detections']:
+            if col in ['overlay_image', 'compliance_by_shelf', 'detections', 'products_found']:
                 enhanced_df[col] = enhanced_df[col].astype('object')
 
         # Populate results
@@ -795,6 +893,7 @@ class PlanogramCompliance(AIPipeline, FlowComponent):
             # NEW: Populate the new columns
             enhanced_df.at[idx, 'compliance_by_shelf'] = result['compliance_by_shelf']
             enhanced_df.at[idx, 'detections'] = result.get('detections', [])
+            enhanced_df.at[idx, 'products_found'] = result.get('products_found', [])
             enhanced_df.at[idx, 'planogram_id'] = result.get('planogram_id', None)
 
         return enhanced_df

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/ProductReportBot.py

@@ -17,6 +17,7 @@ from parrot.models.google import ConversationalScriptConfig, FictionalSpeaker
 from parrot.conf import STATIC_DIR
 from asyncdb import AsyncDB
 from querysource.conf import default_dsn
+from ..conf import pg_server_settings
 from parrot.models.responses import AgentResponse, AIMessage
 from parrot.models.basic import CompletionUsage
 from pathlib import Path
@@ -826,7 +827,7 @@ class ProductReportBot(ParrotAgentConfigMixin, ParrotBot, FlowComponent):
             )
 
         query = query_file.read_text()
-        db = AsyncDB('pg', dsn=default_dsn)
+        db = AsyncDB('pg', dsn=default_dsn, server_settings=pg_server_settings('ProductReportBot'))
         async with await db.connection() as conn:
             product_data, error = await conn.query(query, model)
             if error:
@@ -992,7 +993,7 @@ class ProductReportBot(ParrotAgentConfigMixin, ParrotBot, FlowComponent):
                 response_dict.pop(field, None)
             
            
-            db = AsyncDB('pg', dsn=default_dsn)
+            db = AsyncDB('pg', dsn=default_dsn, server_settings=pg_server_settings('ProductReportBot'))
             async with await db.connection() as conn:
                 ProductResponse.Meta.connection = conn
                 ProductResponse.Meta.schema = program_slug

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/SchedulingVisits.py

@@ -913,8 +913,8 @@ class SchedulingVisits(FlowComponent):
 
         self._logger.info("=== MARKET CAPACITY ANALYSIS ===")
 
-        for market in df['market'].unique():
-            market_data = df[df['market'] == market]
+        for market in df[self._market_name].unique():
+            market_data = df[df[self._market_name] == market]
             store_count = len(market_data)
 
             # Calculate visits needed per month, and the headcount that
@@ -3365,7 +3365,7 @@ class SchedulingVisits(FlowComponent):
                         'employee_position': employee_position,
                         'store_id': store_row['store_id'],
                         'store_name': store_row.get('store_name', 'Unknown'),
-                        'market': store_row['market'],
+                        'market': store_row[self._market_name],
                         'market_id': store_row.get('market_id'),
                         'store_position': store_row['store_position'],
                         'scheduled_date': scheduled_date,

{flowtask-5.13.3 → flowtask-5.13.5}/flowtask/components/tMap/tMap.py

@@ -9,6 +9,7 @@ from navconfig.logging import logging
 from asyncdb import AsyncDB
 from asyncdb.exceptions import NoDataFound
 from querysource.conf import asyncpg_url
+from ...conf import pg_server_settings
 from querysource.types.dt import transforms as qsdfunctions
 from ...exceptions import ComponentError
 from ...parsers.maps import open_map, open_model
@@ -142,7 +143,12 @@ class tMap(FlowComponent):
                         event_loop = asyncio.get_running_loop()
                     except RuntimeError:
                         event_loop = asyncio.get_event_loop()
-                    db = AsyncDB("pg", dsn=asyncpg_url, loop=event_loop)
+                    db = AsyncDB(
+                        "pg",
+                        dsn=asyncpg_url,
+                        loop=event_loop,
+                        server_settings=pg_server_settings("tMap"),
+                    )
                     async with await db.connection() as conn:
                         result, error = await conn.query(discover)
                         if error: