flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/GoogleA4.py

@@ -0,0 +1,150 @@
+from typing import Dict
+from navconfig.logging import logging
+from querysource.exceptions import DataNotFound as QSNotFound
+from ..exceptions import ComponentError, DataNotFound
+from .QSBase import QSBase
+
+
+class GoogleA4(QSBase):
+    """
+    GoogleA4
+
+    Overview
+
+        The GoogleA4 class is a component for interacting with Google Analytics 4 (GA4) to fetch and transform report data.
+        It extends the QSBase class and provides methods for retrieving reports and transforming the data into a specified format.
+
+    .. table:: Properties
+       :widths: auto
+
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | Name         | Required | Summary                                                              |
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | datalist     |   Yes    |  Method for reports                                                  |
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | subtask      |   Yes    |  Identifiers of property and metrics                                 |
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | type         |   Yes    | Defines the type of data handled by the component, set to "report".  |
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | _driver      |   Yes    | Specifies the driver used by the component, set to "ga".             |
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | _metrics     |   Yes    | A dictionary mapping GA4 metrics to their corresponding output names.|
+    +--------------+----------+-----------+----------------------------------------------------------+
+    | _qs          |   Yes    | Instance of the QSBase class used to interact with the data source.  |
+    +--------------+----------+-----------+----------------------------------------------------------+
+
+        Raises:
+            DataNotFound: If no data is found.
+            ComponentError: If any other error occurs during execution.
+
+    Return
+
+    The methods in this class return the requested report data from Google Analytics 4, formatted according to the specific requirements of the component.
+
+    
+
+        Example:
+
+        ```yaml
+        GoogleA4:
+          type: report
+          property_id: '306735132'
+          pattern:
+            start_date:
+            - date_diff
+            - value: now
+              diff: 14
+              mode: days
+              mask: '%Y-%m-%d'
+            end_date:
+            - today
+            - mask: '%Y-%m-%d'
+          dimensions:
+          - mobileDeviceBranding
+          - mobileDeviceModel
+          metric:
+          - sessions
+          - totalUsers
+          - newUsers
+          - engagedSessions
+          - sessionsPerUser
+          company_id: 52
+          ga4_dimension: 10
+        ```
+
+    """
+    type = "report"
+    _driver = "ga"
+    _metrics: Dict = {
+        "sessions": "sessions",
+        "totalUsers": "total_users",
+        "newUsers": "new_users",
+        "engagedSessions": "engaged_users",
+        "sessionsPerUser": "per_user",
+    }
+
+    async def report(self):
+        try:
+            resultset = await self._qs.report()
+            if not resultset:
+                raise DataNotFound(
+                    "No data found on GA4."
+                )
+            result = []
+            # TODO: making a better data-transformation
+            conditions = self._kwargs.get('conditions', {})
+            start_date = conditions.get('start_date', None)
+            end_date = conditions.get('end_date', None)
+            dimensions = conditions.get('dimensions', [])
+            if not start_date or not end_date:
+                raise ComponentError(
+                    "Google Analytics 4 ERROR: Start and end dates are required."
+                )
+            company_id = conditions.get('company_id', None)
+            if not company_id:
+                raise ComponentError(
+                    "Google Analytics 4 ERROR: Company ID is required."
+                )
+            ga4_dimension = conditions.get('ga4_dimension', None)
+            self.add_metric(
+                "START_DATE", start_date
+            )
+            self.add_metric("END_DATE", end_date)
+            self.add_metric("COMPANY", company_id)
+            self.add_metric("DIMENSION", ga4_dimension)
+            for row in resultset:
+                res = {}
+                res["start_date"] = start_date
+                res["end_date"] = end_date
+                res["company_id"] = company_id
+                res["dimension"] = dimensions
+                if "ga4_dimension" in self._variables:
+                    res["ga4_dimension"] = self._variables["ga4_dimension"]
+                elif "ga4_dimension" in conditions:
+                    res["ga4_dimension"] = ga4_dimension
+                d = {}
+                for dimension in conditions.get('dimensions', []):
+                    d[dimension] = row[dimension]
+                res["dimensions"] = d
+                metrics = {}
+                for metric in conditions["metric"]:
+                    metrics[metric] = row[metric]
+                    try:
+                        new_metric = self._metrics[metric]
+                        res[new_metric] = row[metric]
+                    except KeyError:
+                        pass
+                res["metric"] = metrics
+                result.append(res)
+            return result
+        except DataNotFound:
+            raise
+        except QSNotFound as err:
+            raise DataNotFound(
+                f"GA4 Not Found: {err}"
+            ) from err
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(
+                f"Google Analytics 4 ERROR: {err!s}"
+            ) from err

flowtask/components/GoogleGeoCoding.py

@@ -0,0 +1,344 @@
+from collections.abc import Callable
+import asyncio
+from aiohttp.resolver import AsyncResolver
+import aiohttp
+import logging
+import backoff
+import pandas as pd
+from datamodel.parsers.json import json_encoder
+from ..conf import GOOGLE_API_KEY
+from ..components import FlowComponent
+from ..exceptions import ComponentError
+
+
+logging.getLogger("aiohttp").setLevel(logging.WARNING)
+logging.getLogger("httpcore").setLevel(logging.WARNING)
+
+
+class GoogleGeoCoding(FlowComponent):
+    """
+    Google GeoCoding
+
+    Overview
+
+    This component retrieves geographical coordinates (latitude and longitude)
+    for a given set of addresses using Google Maps Geocoding API.
+    It utilizes asynchronous processing to handle multiple requests concurrently and
+    offers error handling for various scenarios.
+
+    .. table:: Properties
+    :widths: auto
+
+        +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+        | Name                  | Required | Description                                                                                          |
+        +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+        | data (pd.DataFrame)   |   Yes    | Pandas DataFrame containing the addresses. Requires a column with the address information.           |
+        +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+        | columns (list)        |   Yes    | List of column names in the DataFrame that contain the address components (e.g., ["street", "city"]).|
+        +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+
+    Return
+
+        The component modifies the input DataFrame by adding new columns named
+        'latitude', 'longitude', 'formatted_address', 'place_id' and 'zipcode' containing the retrieved
+        geocoding information for each address. The original DataFrame is returned.
+    
+
+        Example:
+
+        ```yaml
+        GoogleGeoCoding:
+          skipError: skip
+          place_prefix: account_name
+          use_find_place: true
+          return_pluscode: true
+          chunk_size: 50
+          keywords:
+          - electronics_store
+          columns:
+          - street_address
+          - city
+          - state_code
+          - zipcode
+        ```
+
+    """  # noqa
+    base_url: str = "https://maps.googleapis.com/maps/api/geocode/json"
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.check_field = kwargs.get('comparison_field', 'formatted_address')
+        self.use_find_place: bool = kwargs.get('use_find_place', False)
+        self.return_pluscode: bool = kwargs.get('return_pluscode', False)
+        self.place_prefix: str = kwargs.get('place_prefix', None)
+        self._wait_time: float = kwargs.get('wait_time', 0.1)
+        super(GoogleGeoCoding, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.semaphore = asyncio.Semaphore(10)
+
+    async def start(self, **kwargs):
+        self._counter: int = 0
+        if self.previous:
+            self.data = self.input
+        if not hasattr(self, 'columns'):
+            raise RuntimeError(
+                'GoogleGeoCoding requires a Column Attribute'
+            )
+        if not isinstance(self.columns, list):
+            raise RuntimeError(
+                'GoogleGeoCoding requires a Column Attribute as list'
+            )
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError(
+                "Incompatible Pandas Dataframe", status=404
+            )
+        if not GOOGLE_API_KEY:
+            raise ComponentError(
+                "Google API Key is missing", status=404
+            )
+        return True
+
+    async def find_place(
+        self,
+        address: str,
+        place_prefix: str = None,
+        fields="place_id,plus_code"
+    ) -> tuple:
+        """Searches for a place using the Google Places API.
+
+        Args:
+            idx: row index
+            row: pandas row
+            return_pluscode: return the Google +Code
+            place_prefix: adding a prefix to address
+
+        Returns:
+            The Place ID of the first matching result, or None if no results are found.
+        """
+        base_url = "https://maps.googleapis.com/maps/api/place/findplacefromtext/json"
+        params = {
+            "input": address,
+            "inputtype": "textquery",
+            "fields": fields,
+            "key": GOOGLE_API_KEY
+        }
+        async with aiohttp.ClientSession() as session:
+            async with session.get(base_url, params=params) as response:
+                if response.status == 200:
+                    result = await response.json()
+                    if result['status'] == 'OK' and result["candidates"]:
+                        candidate = result["candidates"][0]
+                        place_id = candidate["place_id"]
+                        plus_code = candidate.get('plus_code', {}).get('compound_code', None)
+                        global_code = candidate.get('plus_code', {}).get('global_code', None)
+                        return place_id, plus_code, global_code
+        return None, None, None
+
+    @backoff.on_exception(
+        backoff.expo,
+        (aiohttp.ClientError, asyncio.TimeoutError),
+        max_tries=2
+    )
+    async def get_coordinates(
+        self,
+        idx,
+        row,
+        return_pluscode: bool = False,
+        place_prefix: str = None
+    ):
+        async with self.semaphore:
+            street_address = self.columns[0]
+            if pd.notnull(row[street_address]):
+                try:
+                    address = ', '.join(
+                        [
+                            str(row[column]) for column in self.columns
+                            if column is not None and pd.notna(row[column])
+                        ]
+                    )
+                except (ValueError, TypeError, KeyError):
+                    address = row[street_address]
+                if not address:
+                    return idx, None
+                if place_prefix:
+                    try:
+                        place_prefix = row[place_prefix]
+                    except (ValueError, KeyError):
+                        pass
+                    address = f"{place_prefix} {address}"
+                more_params = {}
+                if hasattr(self, 'keywords'):
+                    keywords = []
+                    for element in self.keywords:
+                        keywords.append(f"keyword:{element}")
+                    more_params = {
+                        "components": "|".join(keywords)
+                    }
+                params = {
+                    "address": address,
+                    **more_params,
+                    "key": GOOGLE_API_KEY
+                }
+                self._logger.notice(
+                    f"Looking for {address}"
+                )
+                try:
+                    # Total timeout for the request
+                    timeout = aiohttp.ClientTimeout(total=20)
+                    resolver = AsyncResolver(
+                        nameservers=["1.1.1.1", "8.8.8.8"]
+                    )
+                    connector = aiohttp.TCPConnector(
+                        limit=100,
+                        resolver=resolver
+                    )
+                    session_args = {
+                        "connector": connector,
+                        "timeout": timeout,
+                        "json_serialize": json_encoder
+                    }
+                    async with aiohttp.ClientSession(**session_args) as session:
+                        async with session.get(self.base_url, params=params) as response:
+                            if response.status == 200:
+                                result = await response.json()
+                                if result['status'] == 'OK':
+                                    data = result['results'][0]
+                                    more_args = {}
+                                    plus_code = None
+                                    global_code = None
+                                    # Check if it's a subpremise or has an empty plus_code
+                                    if "subpremise" in data["types"] or not data.get("plus_code", {}).get("global_code", None):  # noqa
+                                        # Refine the search:
+                                        if self.use_find_place is True:
+                                            place_id, plus_code, global_code = await self.find_place(
+                                                address, place_prefix
+                                            )
+                                        else:
+                                            place_id = data.get('place_id', None)
+                                    else:
+                                        place_id = data.get('place_id', None)
+                                        try:
+                                            plus_code = data.get("plus_code", {}).get('compound_code')
+                                            global_code = data.get("plus_code", {}).get('global_code')
+                                        except KeyError:
+                                            pass
+                                    # extract all information:
+                                    if return_pluscode is True:
+                                        more_args = {
+                                            "plus_code": plus_code,
+                                            "global_code": global_code
+                                        }
+                                    # Extract postal code
+                                    postal_code = None
+                                    for component in data['address_components']:
+                                        if 'postal_code' in component['types']:
+                                            postal_code = component['long_name']
+                                            break
+                                    latitude = data['geometry']['location']['lat']
+                                    longitude = data['geometry']['location']['lng']
+                                    formatted_address = data['formatted_address']
+                                    self._counter += 1
+                                    # Avoid overwhelming Google APIs
+                                    await asyncio.sleep(self._wait_time)
+                                    return idx, {
+                                        "latitude": latitude,
+                                        "longitude": longitude,
+                                        "formatted_address": formatted_address,
+                                        "place_id": place_id,
+                                        "zipcode": postal_code,
+                                        **more_args
+                                    }
+                                else:
+                                    error = result.get('error_message', result)
+                                    status = result.get('status', 'Unknown')
+                                    self._logger.error(
+                                        f"{status}: {error}"
+                                    )
+                except asyncio.TimeoutError as exc:
+                    self._logger.error(
+                        f"TimeoutException: {exc}"
+                    )
+                    return idx, None
+                except TypeError as exc:
+                    self._logger.error(
+                        f"TypeError: {exc}"
+                    )
+                    return idx, None
+            return idx, None
+
+    def column_exists(self, column: str):
+        if column not in self.data.columns:
+            self._logger.warning(
+                f"Column {column} does not exist in the dataframe"
+            )
+            self.data[column] = None
+            return False
+        return True
+
+    def chunkify(self, lst, n):
+        """Split list lst into chunks of size n."""
+        for i in range(0, len(lst), n):
+            yield lst[i:i + n]
+
+    async def run(self):
+        # initialize columns:
+        self.column_exists('place_id')
+        self.column_exists('latitude')
+        self.column_exists('longitude')
+        self.column_exists('formatted_address')
+        self.column_exists('zipcode')
+        self._counter = 0
+        tasks = [
+            self.get_coordinates(
+                idx,
+                row,
+                return_pluscode=self.return_pluscode,
+                place_prefix=self.place_prefix
+            ) for idx, row in self.data.iterrows()
+            if pd.isnull(row[self.check_field])
+        ]
+        results = []
+        for chunk in self.chunkify(tasks, self.chunk_size):
+            result = await asyncio.gather(*chunk, return_exceptions=True)
+            if result:
+                for res in result:
+                    if isinstance(res, Exception):
+                        # Handle the exception
+                        self._logger.error(
+                            f"Task failed with exception: {res}"
+                        )
+                        continue
+                    results.append(res)
+                # else:
+                #     results += result
+        results_list = []
+        for idx, result in results:
+            if result:
+                result['idx'] = idx  # Add the index to the result dictionary
+                results_list.append(result)
+        if results_list:
+            results_df = pd.DataFrame(results_list)
+            results_df.set_index('idx', inplace=True)
+            # If necessary, reindex results_df to match self.data
+            results_df = results_df.reindex(self.data.index)
+            # Directly assign columns from results_df to self.data
+            for column in results_df.columns:
+                mask = results_df[column].notnull()
+                indices = results_df.index[mask]
+                self.data.loc[indices, column] = results_df.loc[indices, column]
+        self.add_metric("DOWNLOADED", self._counter)
+        # if self._debug is True:
+        print(self.data)
+        print("::: Printing Column Information === ")
+        for column, t in self.data.dtypes.items():
+            print(column, "->", t, "->", self.data[column].iloc[0])
+        self._result = self.data
+        return self._result
+
+    async def close(self):
+        pass