flowtask 5.13.0__tar.gz → 5.13.1__tar.gz

{flowtask-5.13.0 → flowtask-5.13.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flowtask
-Version: 5.13.0
+Version: 5.13.1
 Summary: Framework for Task orchestration
 Author-email: Jesus Lara <jesuslarag@gmail.com>
 License: Apache-2.0

{flowtask-5.13.0 → flowtask-5.13.1}/flowtask/__main__.py

@@ -185,7 +185,14 @@ def main():
     from .utils.uv import install_uvloop
     from .utils import cPrint
     install_uvloop()
-    loop = asyncio.get_event_loop()
+    # `get_event_loop()` stopped creating a loop when none is set: it is
+    # deprecated on 3.10+, and uvloop 0.22 raises outright instead of falling
+    # back. Without this the CLI dies before the first log line.
+    try:
+        loop = asyncio.get_event_loop()
+    except RuntimeError:
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
     loop.slow_callback_duration = 0.2
     try:
         result = loop.run_until_complete(task(loop))

{flowtask-5.13.0 → flowtask-5.13.1}/flowtask/components/MarketClustering.py

@@ -27,12 +27,19 @@ from ..interfaces.OverpassGeocoding import (
     ReverseGeocodeResult,
     DEFAULT_OVERPASS_URL,
 )
+from ..interfaces.GoogleReverseGeocoding import GoogleReverseGeocoder
+from ..interfaces.NominatimGeocoding import (
+    NominatimReverseGeocoder,
+    DEFAULT_NOMINATIM_URL,
+    DEFAULT_NOMINATIM_USER_AGENT,
+    PUBLIC_MIN_INTERVAL,
+)
 from ..exceptions import (
     DataNotFound,
     ConfigError,
     ComponentError,
 )
-from ..conf import ROUTING_DETOUR_FACTOR
+from ..conf import ROUTING_DETOUR_FACTOR, GOOGLE_PLACES_API_KEY
 from ..interfaces.routing import RoutingService, DistanceMatrix
 from .SchedulingVisits import SchedulingVisits
 
@@ -50,6 +57,9 @@ from .SchedulingVisits import SchedulingVisits
 # through the actual RoutingService instance and its own average_speed.
 _INTRA_CLUSTER_AVERAGE_SPEED_MPH = 40.0
 
+# Reverse geocoders accepted by `geocoding_backend` (resolve_centroid_location).
+GEOCODING_BACKENDS = frozenset({'overpass', 'google', 'nominatim'})
+
 
 def meters_to_miles(m):
     return m * 0.000621371
@@ -942,8 +952,9 @@ class MarketClustering(FlowComponent):
             plus ``centroid_city``, ``centroid_state``, ``centroid_county``
             and ``centroid_postcode``. The point resolved is
             ``centroid_lat``/``centroid_lon`` itself — the market's centre,
-            not a member store — at **one Overpass query per market**,
-            issued once the layout is final.
+            not a member store — at **one geocoder query per market**,
+            issued once the layout is final, against the backend chosen by
+            ``geocoding_backend``.
 
             A centroid is a geometric mean and can land on a parking lot or
             an empty field: a point with nothing addressable nearby degrades
@@ -960,6 +971,33 @@ class MarketClustering(FlowComponent):
             rotated through on retry.
         geocode_concurrency (int, default 8): Maximum in-flight
             reverse-geocode queries.
+        geocoding_backend (str, default ``'overpass'``): Reverse geocoder used
+            by ``resolve_centroid_location``. ``'overpass'`` queries
+            ``overpass_url``; ``'google'`` uses the Google Geocoding API
+            (paid, most accurate; needs ``google_api_key`` or
+            ``GOOGLE_PLACES_API_KEY``); ``'nominatim'`` uses OpenStreetMap's
+            Nominatim — the public server by default, throttled to one
+            request per second as its usage policy requires (~2 minutes for
+            120 markets). Google and Nominatim retry HTTP 429/5xx (and
+            Google's ``OVER_QUERY_LIMIT``) with exponential backoff,
+            honouring ``Retry-After``.
+        google_api_key (str, optional): Key for ``geocoding_backend='google'``;
+            defaults to ``GOOGLE_PLACES_API_KEY`` from the configuration.
+        nominatim_url (str, default public server): Nominatim base URL, e.g.
+            a self-hosted ``http://localhost:8080``.
+        nominatim_email (str, optional): Contact address sent to Nominatim,
+            recommended by the public server's usage policy.
+        nominatim_user_agent (str, optional): Identifying ``User-Agent`` for
+            Nominatim; defaults to ``flowtask/<version>``.
+        nominatim_min_interval (float, default 1.0): Seconds between Nominatim
+            requests. Keep 1.0 on the public server; ``0`` for a self-hosted
+            instance.
+        routing_backend (str, optional): Backend for the road-mile lookups
+            made by the sub-cluster anchoring and fly-out probe passes:
+            ``'valhalla'``, ``'osrm'`` or ``'geodesic'`` (offline
+            great-circle × detour factor, zero network calls). Defaults to
+            ``ROUTING_BACKEND`` from the configuration. The
+            ``no_clustering`` path never routes, whatever this is set to.
         isolation_values (list, optional): Values of ``isolation_column``
             forming the isolated partition. Stores inside and outside the
             partition are clustered separately and NEVER share a market —
@@ -1864,6 +1902,40 @@ class MarketClustering(FlowComponent):
         else:
             self.overpass_url_fallback = list(_overpass_fallback)
         self.geocode_concurrency: int = kwargs.pop('geocode_concurrency', 8)
+        # Which reverse geocoder resolve_centroid_location talks to.
+        self.geocoding_backend: str = str(
+            kwargs.pop('geocoding_backend', 'overpass') or 'overpass'
+        ).strip().lower()
+        if self.geocoding_backend not in GEOCODING_BACKENDS:
+            raise ConfigError(
+                f"geocoding_backend must be one of {sorted(GEOCODING_BACKENDS)}, "
+                f"got {self.geocoding_backend!r}"
+            )
+        self.google_api_key: Optional[str] = (
+            kwargs.pop('google_api_key', None) or GOOGLE_PLACES_API_KEY
+        )
+        if (
+            self.resolve_centroid_location
+            and self.geocoding_backend == 'google'
+            and not self.google_api_key
+        ):
+            raise ConfigError(
+                "geocoding_backend='google' needs google_api_key "
+                "(or GOOGLE_PLACES_API_KEY in the environment)"
+            )
+        self.nominatim_url: str = kwargs.pop('nominatim_url', DEFAULT_NOMINATIM_URL)
+        self.nominatim_email: Optional[str] = kwargs.pop('nominatim_email', None)
+        self.nominatim_user_agent: str = kwargs.pop(
+            'nominatim_user_agent', DEFAULT_NOMINATIM_USER_AGENT
+        )
+        self.nominatim_min_interval: float = float(
+            kwargs.pop('nominatim_min_interval', PUBLIC_MIN_INTERVAL)
+        )
+
+        # Road-mile backend for the sub-cluster / fly-out passes. None keeps
+        # RoutingService's configured default (ROUTING_BACKEND); 'geodesic'
+        # runs them fully offline, without a Valhalla instance.
+        self.routing_backend: Optional[str] = kwargs.pop('routing_backend', None)
 
         # -- Global employee-budget mode (FEAT-240) --------------------------
         # A hard cap on total hires for the entire run. Its presence (with
@@ -5284,7 +5356,7 @@ class MarketClustering(FlowComponent):
         n_unplaced = 0
         n_beyond_cap = 0
         stores_beyond_cap = 0
-        async with RoutingService() as routing:
+        async with RoutingService(backend=self.routing_backend) as routing:
             for subcluster in subclusters:
                 medoid = subcluster['medoid']
                 candidates = sorted(
@@ -5536,7 +5608,7 @@ class MarketClustering(FlowComponent):
                 cid = self._data.at[idx, self._cluster_id]
                 by_market.setdefault(cid, []).append(idx)
 
-            async with RoutingService() as routing:
+            async with RoutingService(backend=self.routing_backend) as routing:
                 for cid in sorted(by_market.keys(), key=str):
                     idxs = sorted(by_market[cid])
                     centroid = centroids.get(cid)
@@ -9565,10 +9637,51 @@ class MarketClustering(FlowComponent):
             points[cid] = (float(lat), float(lon))
         return points
 
+    def _geocoder_label(self) -> str:
+        """Human-readable backend + endpoint, for the resolution logs."""
+        if self.geocoding_backend == 'google':
+            return "Google Geocoding"
+        if self.geocoding_backend == 'nominatim':
+            return f"Nominatim ({self.nominatim_url})"
+        return f"Overpass ({self.overpass_url})"
+
+    def _build_reverse_geocoder(
+        self,
+    ) -> Union[OverpassReverseGeocoder, GoogleReverseGeocoder, NominatimReverseGeocoder]:
+        """Instantiate the reverse geocoder selected by `geocoding_backend`.
+
+        Returns:
+            An un-entered geocoder; use it with ``async with``. All three
+            share the ``reverse_many`` contract and return
+            `ReverseGeocodeResult` values.
+        """
+        if self.geocoding_backend == 'google':
+            return GoogleReverseGeocoder(
+                api_key=self.google_api_key,
+                concurrency=self.geocode_concurrency,
+            )
+        if self.geocoding_backend == 'nominatim':
+            # The public server allows one request per second; the
+            # min_interval throttle already serialises, so concurrency
+            # only matters for a self-hosted instance with no throttle.
+            return NominatimReverseGeocoder(
+                url=self.nominatim_url,
+                email=self.nominatim_email,
+                user_agent=self.nominatim_user_agent,
+                min_interval=self.nominatim_min_interval,
+                concurrency=self.geocode_concurrency,
+            )
+        return OverpassReverseGeocoder(
+            url=self.overpass_url,
+            fallback_urls=self.overpass_url_fallback,
+            concurrency=self.geocode_concurrency,
+        )
+
     async def _resolve_centroid_locations(self) -> None:
         """Reverse-geocode each market's centroid into a street address.
 
-        One Overpass query per market — the centroid itself, not a member
+        One query per market against `geocoding_backend` (Overpass, Google
+        or Nominatim) — the centroid itself, not a member
         store — de-duplicated and run with bounded concurrency, so a market
         of 400 stores still costs exactly one lookup.
 
@@ -9593,22 +9706,18 @@ class MarketClustering(FlowComponent):
             )
             return
 
+        label = self._geocoder_label()
         self._logger.info(
             f"Resolving centroid location for {len(points)} markets via "
-            f"Overpass ({self.overpass_url})"
+            f"{label}"
         )
         try:
-            async with OverpassReverseGeocoder(
-                url=self.overpass_url,
-                fallback_urls=self.overpass_url_fallback,
-                concurrency=self.geocode_concurrency,
-            ) as geocoder:
+            async with self._build_reverse_geocoder() as geocoder:
                 resolved = await geocoder.reverse_many(points.values())
         except Exception as exc:  # noqa: BLE001 - never lose a layout over a label
             self._logger.warning(
                 f"Centroid location resolution failed against "
-                f"{self.overpass_url} ({exc}); every centroid_address will "
-                f"be empty."
+                f"{label} ({exc}); every centroid_address will be empty."
             )
             return
 

{flowtask-5.13.0 → flowtask-5.13.1}/flowtask/components/ParrotAgent.py

@@ -452,7 +452,10 @@ class ParrotAgent(ParrotAgentConfigMixin, AgentBase, FlowComponent):
         # keeps its fields inside `output_column` rather than as columns of
         # their own, so look there too — otherwise `attach_columns` would work
         # per row (where the result is merged into the context) but not here.
-        for column in {c for ch in config.channels for c in ch.attach_columns}:
+        columns = {c for ch in config.channels for c in ch.attach_columns}
+        if config.storage is not None:
+            columns |= set(config.storage.columns)
+        for column in columns:
             values = self._collect_paths(df, column)
             if values:
                 context[column] = values[: config.max_attachments]

flowtask-5.13.1/flowtask/interfaces/GoogleReverseGeocoding.py

@@ -0,0 +1,171 @@
+"""Reverse geocoding through the Google Geocoding API.
+
+The paid but most accurate of the ``geocoding_backend`` options: one
+``latlng`` request per point against
+``https://maps.googleapis.com/maps/api/geocode/json``. Google reports
+throttling inside a 200 body (``status: OVER_QUERY_LIMIT``), so those
+answers are retried with the same exponential backoff as HTTP 429/5xx.
+A ``REQUEST_DENIED`` (bad or unauthorised key) is final and stops every
+later lookup in the same session instead of repeating the error per market.
+
+Typical use::
+
+    async with GoogleReverseGeocoder(api_key=GOOGLE_API_KEY) as geocoder:
+        results = await geocoder.reverse_many(market_centroids)
+"""
+from __future__ import annotations
+
+from typing import Any, Mapping, Optional
+
+from .OverpassGeocoding import ReverseGeocodeResult, _haversine_m, _valid_postcode
+from .ReverseGeocoding import BaseReverseGeocoder
+
+
+DEFAULT_GOOGLE_GEOCODE_URL = "https://maps.googleapis.com/maps/api/geocode/json"
+
+# Body statuses meaning "try again later" rather than "no answer".
+_RETRY_STATUSES = frozenset({"OVER_QUERY_LIMIT", "UNKNOWN_ERROR"})
+# Body statuses meaning the key or request is unusable: retrying is futile.
+_FATAL_STATUSES = frozenset({"REQUEST_DENIED", "INVALID_REQUEST"})
+
+# Address-component types that name the municipality, in preference order.
+_CITY_TYPES = (
+    "locality",
+    "postal_town",
+    "sublocality_level_1",
+    "sublocality",
+    "administrative_area_level_3",
+)
+
+
+def _components(result: Mapping[str, Any]) -> dict[str, dict]:
+    """Index a result's address components by type (first one wins)."""
+    indexed: dict[str, dict] = {}
+    for component in result.get("address_components") or []:
+        for kind in component.get("types") or []:
+            indexed.setdefault(kind, component)
+    return indexed
+
+
+class GoogleReverseGeocoder(BaseReverseGeocoder):
+    """Resolve coordinates to street addresses via Google Geocoding.
+
+    Args:
+        api_key: Google Maps Platform key with the Geocoding API enabled.
+        url: Geocoding endpoint (overridable for tests or a proxy).
+        language: ``language`` parameter for the returned names.
+        **kwargs: Transport options forwarded to :class:`BaseReverseGeocoder`
+            (``max_retries``, ``backoff_base``, ``max_backoff``,
+            ``concurrency``, ``timeout``, ``session``...).
+
+    Raises:
+        ValueError: If ``api_key`` is empty.
+    """
+
+    def __init__(
+        self,
+        api_key: str,
+        url: str = DEFAULT_GOOGLE_GEOCODE_URL,
+        language: str = "en",
+        **kwargs: Any,
+    ) -> None:
+        if not api_key:
+            raise ValueError("GoogleReverseGeocoder requires an api_key")
+        super().__init__(**kwargs)
+        self.api_key = api_key
+        self.url = url
+        self.language = language
+        self._denied = False
+
+    @property
+    def name(self) -> str:
+        return "Google Geocoding"
+
+    def _retry_payload(self, payload: Mapping[str, Any]) -> bool:
+        return payload.get("status") in _RETRY_STATUSES
+
+    @staticmethod
+    def _parse(payload: Optional[Mapping[str, Any]], lat: float, lon: float) -> ReverseGeocodeResult:
+        """Turn one Geocoding API payload into a :class:`ReverseGeocodeResult`.
+
+        The street comes from the first result carrying both a
+        ``street_number`` and a ``route`` (Google orders results from most
+        to least specific); without one, the first result with a ``route``
+        gives a street-only answer. City, state, county and postcode are
+        taken from that result and, where it lacks them, from the broader
+        results that follow.
+
+        Args:
+            payload: Decoded JSON, or ``None`` for a failed request.
+            lat: Latitude originally queried, for the distance.
+            lon: Longitude originally queried.
+
+        Returns:
+            A result whose ``degraded`` flag is ``False`` only when a house
+            number and street were matched.
+        """
+        if not payload or payload.get("status") != "OK":
+            return ReverseGeocodeResult()
+        results = payload.get("results") or []
+        if not results:
+            return ReverseGeocodeResult()
+
+        indexed = [(result, _components(result)) for result in results]
+        primary = next(
+            (pair for pair in indexed if "street_number" in pair[1] and "route" in pair[1]),
+            None,
+        ) or next((pair for pair in indexed if "route" in pair[1]), None) or indexed[0]
+        primary_result, primary_parts = primary
+
+        def _pick(kinds: tuple[str, ...], short: bool = False) -> Optional[str]:
+            for _, parts in [primary, *indexed]:
+                for kind in kinds:
+                    component = parts.get(kind)
+                    if component:
+                        return component.get("short_name" if short else "long_name")
+            return None
+
+        housenumber = (primary_parts.get("street_number") or {}).get("long_name")
+        street = (primary_parts.get("route") or {}).get("long_name")
+        location = (primary_result.get("geometry") or {}).get("location") or {}
+        distance = (
+            _haversine_m(lat, lon, float(location["lat"]), float(location["lng"]))
+            if "lat" in location and "lng" in location
+            else None
+        )
+        return ReverseGeocodeResult(
+            housenumber=housenumber,
+            street=street,
+            city=_pick(_CITY_TYPES),
+            state=_pick(("administrative_area_level_1",), short=True),
+            postcode=_valid_postcode(_pick(("postal_code",))),
+            county=_pick(("administrative_area_level_2",)),
+            distance_m=distance,
+            degraded=not (housenumber and street),
+        )
+
+    async def reverse(self, lat: float, lon: float) -> ReverseGeocodeResult:
+        """Resolve one coordinate through the Geocoding API.
+
+        Args:
+            lat: Latitude to resolve.
+            lon: Longitude to resolve.
+
+        Returns:
+            The parsed result; an empty degraded one when the request
+            failed, returned ``ZERO_RESULTS`` or the key was rejected.
+        """
+        if self._denied:
+            return ReverseGeocodeResult()
+        payload = await self._get_json(
+            self.url,
+            {"latlng": f"{lat},{lon}", "key": self.api_key, "language": self.language},
+        )
+        status = (payload or {}).get("status")
+        if status in _FATAL_STATUSES:
+            self._denied = status == "REQUEST_DENIED"
+            self._logger.error(
+                "Google Geocoding %s: %s", status, (payload or {}).get("error_message", "")
+            )
+        return self._parse(payload, lat, lon)
+

flowtask-5.13.1/flowtask/interfaces/NominatimGeocoding.py

@@ -0,0 +1,149 @@
+"""Reverse geocoding through Nominatim, OpenStreetMap's geocoder.
+
+The key-free ``geocoding_backend`` option. By default it targets the
+public server, ``https://nominatim.openstreetmap.org``, and obeys its
+usage policy (https://operations.osmfoundation.org/policies/nominatim/):
+
+* **at most one request per second**, enforced across concurrent callers
+  (``min_interval=1.0``), so ~120 market centroids take ~2 minutes;
+* an identifying ``User-Agent`` on every request (and an optional
+  ``email`` contact parameter);
+* exponential backoff on HTTP 429/5xx, honouring ``Retry-After``.
+
+A self-hosted instance can lift the throttle with ``min_interval=0``.
+
+Typical use::
+
+    async with NominatimReverseGeocoder(email="ops@example.com") as geocoder:
+        results = await geocoder.reverse_many(market_centroids)
+"""
+from __future__ import annotations
+
+from typing import Any, Mapping, Optional
+
+from ..version import __version__
+from .OverpassGeocoding import ReverseGeocodeResult, _haversine_m, _valid_postcode
+from .ReverseGeocoding import BaseReverseGeocoder
+
+
+DEFAULT_NOMINATIM_URL = "https://nominatim.openstreetmap.org"
+DEFAULT_NOMINATIM_USER_AGENT = (
+    f"flowtask/{__version__} (+https://github.com/phenobarbital/flowtask)"
+)
+# Public-server policy: an absolute maximum of 1 request per second.
+PUBLIC_MIN_INTERVAL = 1.0
+
+# Address keys naming the municipality, in preference order.
+_CITY_KEYS = ("city", "town", "village", "hamlet", "municipality", "suburb")
+# Address keys naming the street, in preference order.
+_STREET_KEYS = ("road", "pedestrian", "footway", "residential", "path")
+
+
+class NominatimReverseGeocoder(BaseReverseGeocoder):
+    """Resolve coordinates to street addresses via Nominatim ``/reverse``.
+
+    Args:
+        url: Nominatim base URL (without ``/reverse``).
+        email: Contact address sent as the ``email`` parameter, as the
+            public usage policy recommends for any non-trivial use.
+        user_agent: Identifying ``User-Agent``; the policy rejects generic
+            library defaults.
+        language: ``accept-language`` for the returned names.
+        zoom: Detail level; 18 asks for a building/address.
+        min_interval: Seconds between request starts (1.0 for the public
+            server; ``0`` for a self-hosted one).
+        **kwargs: Transport options forwarded to :class:`BaseReverseGeocoder`
+            (``max_retries``, ``backoff_base``, ``max_backoff``,
+            ``concurrency``, ``timeout``, ``session``...).
+    """
+
+    def __init__(
+        self,
+        url: str = DEFAULT_NOMINATIM_URL,
+        email: Optional[str] = None,
+        user_agent: str = DEFAULT_NOMINATIM_USER_AGENT,
+        language: str = "en",
+        zoom: int = 18,
+        min_interval: float = PUBLIC_MIN_INTERVAL,
+        **kwargs: Any,
+    ) -> None:
+        super().__init__(
+            min_interval=min_interval,
+            user_agent=user_agent or DEFAULT_NOMINATIM_USER_AGENT,
+            **kwargs,
+        )
+        self.url = (url or DEFAULT_NOMINATIM_URL).rstrip("/")
+        self.email = email
+        self.language = language
+        self.zoom = zoom
+
+    @property
+    def name(self) -> str:
+        return "Nominatim"
+
+    @staticmethod
+    def _state_code(address: Mapping[str, Any]) -> Optional[str]:
+        """Prefer the ISO 3166-2 subdivision code (``US-GA`` -> ``GA``)."""
+        iso = address.get("ISO3166-2-lvl4")
+        if iso and "-" in iso:
+            return iso.split("-", 1)[1]
+        return address.get("state")
+
+    @staticmethod
+    def _parse(payload: Optional[Mapping[str, Any]], lat: float, lon: float) -> ReverseGeocodeResult:
+        """Turn one ``/reverse?format=jsonv2`` payload into a result.
+
+        Args:
+            payload: Decoded JSON, or ``None`` for a failed request.
+            lat: Latitude originally queried, for the distance.
+            lon: Longitude originally queried.
+
+        Returns:
+            A result whose ``degraded`` flag is ``False`` only when a house
+            number and street were matched. ``{"error": ...}`` (nothing
+            within reach, e.g. open water) yields an empty degraded result.
+        """
+        if not payload or payload.get("error"):
+            return ReverseGeocodeResult()
+        address = payload.get("address") or {}
+        housenumber = address.get("house_number")
+        street = next((address[key] for key in _STREET_KEYS if address.get(key)), None)
+        city = next((address[key] for key in _CITY_KEYS if address.get(key)), None)
+        distance: Optional[float] = None
+        try:
+            distance = _haversine_m(lat, lon, float(payload["lat"]), float(payload["lon"]))
+        except (KeyError, TypeError, ValueError):
+            pass
+        return ReverseGeocodeResult(
+            housenumber=housenumber,
+            street=street,
+            city=city,
+            state=NominatimReverseGeocoder._state_code(address),
+            postcode=_valid_postcode(address.get("postcode")),
+            county=address.get("county"),
+            distance_m=distance,
+            degraded=not (housenumber and street),
+        )
+
+    async def reverse(self, lat: float, lon: float) -> ReverseGeocodeResult:
+        """Resolve one coordinate through Nominatim ``/reverse``.
+
+        Args:
+            lat: Latitude to resolve.
+            lon: Longitude to resolve.
+
+        Returns:
+            The parsed result; an empty degraded one on failure.
+        """
+        params: dict[str, Any] = {
+            "format": "jsonv2",
+            "lat": lat,
+            "lon": lon,
+            "zoom": self.zoom,
+            "addressdetails": 1,
+            "accept-language": self.language,
+        }
+        if self.email:
+            params["email"] = self.email
+        payload = await self._get_json(f"{self.url}/reverse", params)
+        return self._parse(payload, lat, lon)