flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/google.py

@@ -0,0 +1,335 @@
+from typing import Union
+from collections.abc import Callable
+from functools import partial
+import asyncio
+import aiohttp
+from aiohttp.resolver import AsyncResolver
+import pandas as pd
+import ssl
+from datamodel.parsers.json import json_encoder
+from proxylists import check_address
+from proxylists.proxies import (
+    FreeProxy,
+    Oxylabs
+)
+from ..conf import GOOGLE_API_KEY, GOOGLE_PLACES_API_KEY
+from ..exceptions import ComponentError
+from ..components import FlowComponent
+
+
+# Monkey-Patching for <3.11 TLS Support
+setattr(
+    asyncio.sslproto._SSLProtocolTransport,
+    "_start_tls_compatible", True
+)
+
+class GoogleBase(FlowComponent):
+    """
+    GoogleBase.
+
+        Overview: A base class for Google API components.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self._type: str = kwargs.pop('type', None)
+        self.api_key: str = kwargs.pop('api_key', GOOGLE_API_KEY)
+        self.use_proxies: bool = kwargs.pop('use_proxies', False)
+        self.paid_proxy: bool = kwargs.pop('paid_proxy', False)
+        super(GoogleBase, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+        self.semaphore = asyncio.Semaphore(10)  # Adjust the limit as needed
+
+    async def close(self):
+        pass
+
+    def _evaluate_input(self):
+        if self.previous:
+            self.data = self.input
+        elif self.input is not None:
+            self.data = self.input
+
+    async def start(self, **kwargs):
+        self._counter: int = 0
+        self._evaluate_input()
+        if not self._type:
+            raise RuntimeError(
+                'Google requires a Type Function'
+            )
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError(
+                "Incompatible Pandas Dataframe", status=404
+            )
+        if not self.api_key:
+            self.api_key = GOOGLE_PLACES_API_KEY
+            if not self.api_key:
+                raise ComponentError(
+                    "Google API Key is missing", status=404
+                )
+        return True
+
+    def _get_session_args(self) -> dict:
+        """Get aiohttp Session arguments."""
+        # Total timeout for the request
+        timeout = aiohttp.ClientTimeout(total=20)
+        resolver = AsyncResolver(
+            nameservers=["1.1.1.1", "8.8.8.8"]
+        )
+        connector = aiohttp.TCPConnector(
+            limit=100,
+            resolver=resolver
+        )
+        return {
+            "connector": connector,
+            "timeout": timeout,
+            "json_serialize": json_encoder,
+            "trust_env": True
+        }
+
+    async def get_proxies(self):
+
+        if self.paid_proxy is True:
+            proxies = await Oxylabs().get_proxy_list()
+            return proxies.get('https')
+        else:
+            p = []
+            proxies = await FreeProxy().get_list()
+            for address in proxies:
+                host, port = address.split(':')
+                if await check_address(host=host, port=port) is True:
+                    p.append(f"http://{address}")
+        return p[0]
+
+    async def _google_session(
+        self,
+        url: str,
+        session_args: dict,
+        params: dict = None,
+        method: str = 'GET',
+        use_json: bool = False,
+        as_json: bool = True,
+        use_proxies: bool = False,
+        google_search: bool = False,
+        **kwargs
+    ) -> Union[aiohttp.ClientResponse, dict]:
+        """Make a Google API request using aiohttp Session."""
+        _proxies = None
+        if use_proxies is True or self.use_proxies is True:
+            _proxies = await self.get_proxies()
+
+        ssl_context = ssl.create_default_context()
+        # Ensure at least TLS 1.2 is used
+        ssl_context.minimum_version = ssl.TLSVersion.TLSv1_2
+        ssl_context.check_hostname = False
+        ssl_context.verify_mode = ssl.CERT_NONE
+        async with aiohttp.ClientSession(**session_args) as session:
+            if method.upper() == 'GET':
+                request = partial(
+                    session.request,
+                    method.upper(),
+                    url,
+                    params=params,
+                    proxy=_proxies,
+                    ssl=ssl_context,
+                    **kwargs
+                )
+            else:
+                if use_json is True:
+                    request = partial(
+                        session.request,
+                        method.upper(),
+                        url,
+                        json=params,
+                        proxy=_proxies,
+                        ssl=ssl_context,
+                        **kwargs
+                    )
+                else:
+                    request = partial(
+                        session.request,
+                        method.upper(),
+                        url, data=params,
+                        proxy=_proxies,
+                        ssl=ssl_context,
+                        **kwargs
+                    )
+            async with request() as response:
+                if response.status == 200:
+                    if as_json is True:
+                        result = await response.json()
+                        if result['status'] == 'OK':
+                            # TODO: Check if it's a premise or subpremise
+                            return result
+                    else:
+                        if google_search is True:
+                            return await response.read()
+                        else:
+                            return await response.text()
+                else:
+                    if google_search is True:
+                        await self.check_response_search(response)
+                    else:
+                        await self.google_response_code(response)
+                    return None
+
+    async def check_response_search(self, response: aiohttp.ClientResponse):
+        if response.status == 429:
+            error = await response.text()
+            self._logger.error(
+                "Google Search: Too many requests"
+            )
+            return None
+        elif response.status > 299:
+            error = await response.text()
+            self._logger.error(
+                f"Raw response Error: {error}"
+            )
+            raise ComponentError(
+                f"Google Places Error {response.status}",
+                f"Error: {error}"
+            )
+
+    async def google_response_code(self, response: aiohttp.ClientResponse):
+        """
+        check if query quota has been surpassed or other errors that can happen.
+        :param resp: json response
+        :return:
+        """
+        if response.status == 429:
+            error = await response.text()
+            self._logger.error(
+                "Google Search: Too many requests"
+            )
+            return None
+        else:
+            result = await response.json()
+            status = result.get('status', 'Unknown')
+            if status == "OK" or status == "ZERO_RESULTS":
+                return
+            # Error:
+            error = result.get('error', result)
+            status = error.get('status', 'Unknown')
+            message = error.get('message', error)
+
+            self._logger.error(
+                f"{status}: {message}: {error}"
+            )
+
+            if status == "REQUEST_DENIED":
+                raise ComponentError(
+                    (
+                        f"Google Places {status}: "
+                        "Request was denied, maybe the API key is invalid."
+                    )
+                )
+
+            if status == "OVER_QUERY_LIMIT":
+                raise ComponentError(
+                    (
+                        f"Google Places {status}: "
+                        "You exceeded your Query Limit for Google Places API Web Service, "
+                        "check https://developers.google.com/places/web-service/usage "
+                        "to upgrade your quota."
+                    )
+                )
+
+            if status == "INVALID_REQUEST":
+                raise ComponentError(
+                    (
+                        f"Google Places {status}: "
+                        "Invalid Request: "
+                        "The query string is malformed, "
+                        "check if your formatting for lat/lng and radius is correct."
+                        f"Error: {error}"
+                    )
+                )
+
+            if status == "NOT_FOUND":
+                raise ComponentError(
+                    (
+                        f"Google Places {status}: "
+                        "The place ID was not found and either does not exist or was retired."
+                    )
+                )
+
+            raise ComponentError(
+                (
+                    f"Google Places {status}: "
+                    "Unidentified error with the Places API, please check the response code"
+                    f"error: {error}"
+                )
+            )
+
+    def column_exists(self, column: str):
+        """Returns True if the column exists in the DataFrame."""
+        if column not in self.data.columns:
+            self._logger.warning(
+                f"Column {column} does not exist in the dataframe"
+            )
+            self.data[column] = None
+            return False
+        return True
+
+    def chunkify(self, lst, n):
+        """Split list lst into chunks of size n."""
+        for i in range(0, len(lst), n):
+            yield lst[i:i + n]
+
+    async def _processing_tasks(self, tasks: list) -> pd.DataFrame:
+        """Process tasks concurrently."""
+        results = []
+        for chunk in self.chunkify(tasks, self.chunk_size):
+            result = await asyncio.gather(*chunk, return_exceptions=True)
+            if result:
+                for res in result:
+                    if isinstance(res, Exception):
+                        # Handle the exception
+                        self._logger.error(
+                            f"Task failed with exception: {res}. Type: {type(res)}"
+                        )
+                        self._logger.error(
+                            f"Exception type: {type(res)}, Task input types: {type(chunk)}"
+                        )
+                        continue
+                    results.append(res)
+        results_list = []
+        for idx, result in results:
+            if result:
+                result['idx'] = idx  # Add the index to the result dictionary
+                results_list.append(result)
+        if results_list:
+            results_df = pd.DataFrame(results_list)
+            results_df.set_index('idx', inplace=True)
+            # If necessary, reindex results_df to match self.data
+            results_df = results_df.reindex(self.data.index)
+            # Directly assign columns from results_df to self.data
+            for column in results_df.columns:
+                mask = results_df[column].notnull()
+                indices = results_df.index[mask]
+                self.data.loc[indices, column] = results_df.loc[indices, column]
+        return self.data
+
+    async def run(self):
+        """Run the Google Places API."""
+        tasks = []
+        fn = getattr(self, self._type)
+        tasks = [
+            fn(
+                idx,
+                row,
+            ) for idx, row in self.data.iterrows()
+        ]
+        # Execute tasks concurrently
+        df = await self._processing_tasks(tasks)
+        if self._debug is True:
+            print(df)
+            print("::: Printing Column Information === ")
+            for column, t in df.dtypes.items():
+                print(column, "->", t, "->", df[column].iloc[0])
+        self.add_metric("GOOGLE_PLACES_DOWNLOADED", self._counter)
+        self._result = df
+        return self._result

flowtask/components/group.py

@@ -0,0 +1,221 @@
+import asyncio
+import copy
+from collections.abc import Callable
+from navconfig.logging import logging
+from asyncdb.exceptions import NoDataFound, ProviderError
+from ..utils.stats import StepMonitor
+from ..interfaces.log import SkipErrors
+from ..exceptions import DataNotFound, NotSupported, ComponentError
+from ..utils import cPrint
+from .flow import FlowComponent
+
+
+class GroupComponent(FlowComponent):
+    """
+    GroupComponent
+
+    Overview
+
+        This component executes a group of other FlowTask components sequentially as a single unit.
+        It allows chaining multiple tasks together and provides error handling for various scenarios.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------------+----------+----------------------------------------------------------------------------------------------------------------+
+        | Name                   | Required | Description                                                                                                    |
+        +------------------------+----------+----------------------------------------------------------------------------------------------------------------+
+        | component_list (list)  |   Yes    | List of dictionaries defining the components to be executed in the group. Each dictionary                      |
+        |                        |          | should contain the following keys:                                                                             |
+        |                        |          |   - "component": The FlowTask component class to be used.                                                      |
+        |                        |          |   - "params": A dictionary containing parameters to be passed to the component.                                |
+        |                        |          | (Optional)                                                                                                     |
+        |                        |          |   - "conditions": A dictionary containing conditions that must be met before running the component. (Optional) |
+        +------------------------+----------+----------------------------------------------------------------------------------------------------------------+
+        | stat (Callable)        |    No    | Optional callback function for step-level monitoring and statistics collection.                                |
+        +------------------------+----------+----------------------------------------------------------------------------------------------------------------+
+        | skipError              |    No    | Defines the behavior when a component within the group raises an error.                                        |
+        |                        |          | Valid options are:                                                                                             |
+        |                        |          |   SkipErrors: Skip This makes the component continue his execution.                                            |
+        |                        |          |   SkipErrors: Raise This Raise the error and interrupt execution.                                              |
+        +------------------------+----------+----------------------------------------------------------------------------------------------------------------+
+
+    Return
+
+        The component modifies the data received from the previous component and returns the final output after
+        all components in the group have been executed.
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        component_list: list = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._params = {}
+        self._components = component_list
+        self._conditions: dict = {}
+        super(GroupComponent, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        steps = []
+        prev = self.previous
+        result = None
+        for step in self._components:
+            step = copy.deepcopy(step)
+            step_name = step.name
+            try:
+                _prev = prev
+                component = self.get_component(step=step, previous=prev)
+                prev = component
+            except Exception as e:
+                raise ComponentError(f"{e!s}") from e
+            # calling start method for component
+            start = getattr(component, "start", None)
+            if callable(start):
+                try:
+                    if asyncio.iscoroutinefunction(start):
+                        st = await component.start()
+                    else:
+                        st = component.start()
+                    logging.debug(f"{step_name} STARTED: {st}")
+                except (NoDataFound, DataNotFound) as err:
+                    if component.skipError == SkipErrors.SKIP:
+                        self._logger.warning(
+                            f"::: SKIPPING Error on {step_name} :::: "
+                        )
+                        prev = _prev
+                        continue
+                    raise DataNotFound(
+                        f'Data Not Found over {step_name}'
+                    ) from err
+                except (ProviderError, ComponentError, NotSupported) as err:
+                    raise ComponentError(
+                        f"Group Error: calling Start on {step.name}, error: {err}"
+                    ) from err
+            else:
+                raise ComponentError(f"Group Error: missing Start on {step.name}")
+            # then, calling the run method:
+            try:
+                run = getattr(component, "run", None)
+                if asyncio.iscoroutinefunction(run):
+                    result = await run()
+                else:
+                    result = run()
+            except (NoDataFound, DataNotFound) as err:
+                if component.skipError == SkipErrors.SKIP:
+                    self._logger.warning(
+                        f"::: SKIPPING Error on {step_name} :::: "
+                    )
+                    prev = _prev
+                    continue
+                raise DataNotFound(
+                    f'Data Not Found over {step_name}'
+                ) from err
+            except (ProviderError, ComponentError, NotSupported) as err:
+                if component.skipError == SkipErrors.SKIP:
+                    self._logger.warning(
+                        f"::: SKIPPING Error on {step_name} :::: "
+                    )
+                    prev = _prev
+                    continue
+                raise NotSupported(
+                    f"Group Error: Not Supported on {step.name}, error: {err}"
+                ) from err
+            except Exception as err:
+                if component.skipError == SkipErrors.SKIP:
+                    self._logger.warning(
+                        f"::: SKIPPING Error on {step_name} :::: "
+                    )
+                    prev = _prev
+                    continue
+                raise ComponentError(
+                    f"Group Error: Calling Start on {step.name}, error: {err}"
+                ) from err
+            finally:
+                steps.append(step_name)
+                try:
+                    close = getattr(component, "close", None)
+                    if asyncio.iscoroutinefunction(close):
+                        await close()
+                    else:
+                        close()
+                except Exception as e:  # pylint: disable=W0703
+                    logging.warning(e)
+        self._result = result
+        return self._result
+
+    def get_component(self, step, previous):
+        if self.stat:
+            parent_stat = self.stat.parent()
+            stat = StepMonitor(name=step.name, parent=parent_stat)
+            parent_stat.add_step(stat)
+        else:
+            stat = None
+        params = step.params
+        try:
+            if params["conditions"]:
+                self._conditions[step.name] = params["conditions"]
+        except KeyError:
+            pass
+        params["ENV"] = self._environment
+        # params
+        if self._params:
+            try:
+                params["params"] = {**params["params"], **self._params}
+            except (KeyError, TypeError):
+                pass
+        # parameters
+        if self._parameters:
+            parameters = params.get("parameters", {})
+            params["parameters"] = {**parameters, **self._parameters}
+        if hasattr(self, "_program"):
+            params["_program"] = self._program
+        # useful to change variables in set var components
+        params["_vars"] = self._vars
+        # variables dictionary
+        params["variables"] = self._variables
+        params["_args"] = self._args
+        # argument list for components (or tasks) that need argument lists
+        params["arguments"] = self._arguments
+        # for components with conditions, we can add more conditions
+        conditions = params.get("conditions", {})
+        step_conds = self._conditions.get(step.name, {})
+        if self.conditions is not None:
+            step_conds = {**self.conditions, **step_conds}
+        params["conditions"] = {**conditions, **step_conds}
+        # attributes only usable component-only
+        params["attributes"] = self._attributes
+        # the current Pile of components
+        params["TaskPile"] = self._TaskPile
+        # params['TaskName'] = step_name
+        params["debug"] = self._debug
+        params["argparser"] = self._argparser
+        # the current in-memory connector
+        params["memory"] = self._memory
+        target = step.component
+        job = None
+        try:
+            job = target(job=previous, loop=self._loop, stat=stat, **params)
+            job.SetPile(self._TaskPile)
+            cPrint(
+                f"LOADED STEP: {step.name}",
+                level="DEBUG"
+            )
+            return job
+        except Exception as err:
+            raise ComponentError(
+                f"Component Error on {target}, error: {err}"
+            ) from err

flowtask/components/reviewscrap.py

@@ -0,0 +1,132 @@
+from typing import Any
+from collections.abc import Callable
+import asyncio
+import httpx
+from pandas import DataFrame
+from seleniumwire import webdriver
+from navconfig.logging import logging
+from ..exceptions import (
+    ConfigError,
+    ComponentError,
+    NotSupported,
+)
+
+from .flow import FlowComponent
+from ..interfaces import SeleniumService
+from ..interfaces import HTTPService
+
+logging.getLogger(name='selenium.webdriver').setLevel(logging.WARNING)
+logging.getLogger(name='WDM').setLevel(logging.WARNING)
+logging.getLogger(name='hpack').setLevel(logging.WARNING)
+logging.getLogger(name='seleniumwire').setLevel(logging.WARNING)
+
+def on_backoff(details):
+    logging.warning(
+        f"Backing off {details['wait']:0.1f} seconds after {details['tries']} tries due to error: {details['exception']}"
+    )
+
+def bad_gateway_exception(exc):
+    """Check if the exception is a 502 Bad Gateway error."""
+    return isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 502
+
+
+class ReviewScrapper(FlowComponent, SeleniumService, HTTPService):
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._fn = kwargs.pop('type', None)
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.task_parts: int = kwargs.get('task_parts', 10)
+        if not self._fn:
+            raise ConfigError(
+                f"{self.__name__}: require a `type` Function to be called, ex: availability"
+            )
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def get_cookies(self, url: str) -> dict:
+        options = webdriver.ChromeOptions()
+        options.add_argument('--headless')  # Run in headless mode
+        driver = webdriver.Chrome(options=options)
+        driver.get(url)
+        cookies = driver.get_cookies()
+        driver.quit()
+        return {cookie['name']: cookie['value'] for cookie in cookies}
+
+    def chunkify(self, lst, n):
+        """Split list lst into chunks of size n."""
+        for i in range(0, len(lst), n):
+            yield lst[i:i + n]
+
+    def column_exists(self, column: str, default_val: Any = None):
+        if column not in self.data.columns:
+            self._logger.warning(
+                f"Column {column} does not exist in the Dataframe"
+            )
+            self.data[column] = default_val
+            return False
+        return True
+
+    def split_parts(self, task_list, num_parts: int = 5) -> list:
+        part_size, remainder = divmod(len(task_list), num_parts)
+        parts = []
+        start = 0
+        for i in range(num_parts):
+            # Distribute the remainder across the first `remainder` parts
+            end = start + part_size + (1 if i < remainder else 0)
+            parts.append(task_list[start:end])
+            start = end
+        return parts
+
+    async def _processing_tasks(self, tasks: list) -> DataFrame:
+        """Process tasks concurrently."""
+        results = []
+        for chunk in self.split_parts(tasks, self.task_parts):
+            result = await asyncio.gather(*chunk, return_exceptions=False)
+            results.extend(result)
+        return results
+
+    async def run(self):
+        # we need to call the "function" for Services.
+        fn = getattr(self, self._fn)
+        result = None
+        if not callable(fn):
+            raise ComponentError(
+                f"{self.__name__}: Function {self._fn} doesn't exists."
+            )
+        try:
+            result = await fn()
+        except (ComponentError, TimeoutError, NotSupported):
+            raise
+        except Exception as exc:
+            raise ComponentError(
+                f"{self.__name__}: Unknown Error: {exc}"
+            ) from exc
+        # Print results
+        print(result)
+        print("::: Printing Column Information === ")
+        for column, t in result.dtypes.items():
+            print(column, "->", t, "->", result[column].iloc[0])
+        self._result = result
+        return self._result
+
+    async def close(self, **kwargs) -> bool:
+        self.close_driver()
+        return True
+
+    async def start(self, **kwargs) -> bool:
+        await super(ReviewScrapper, self).start(**kwargs)
+        if self.previous:
+            self.data = self.input
+            if not isinstance(self.data, DataFrame):
+                raise ComponentError(
+                    "Incompatible Pandas Dataframe"
+                )
+        self.api_token = self.get_env_value(self.api_token) if hasattr(self, 'api_token') else self.get_env_value('TARGET_API_KEY')  # noqa
+        if not hasattr(self, self._fn):
+            raise ConfigError(
+                f"{self.__name__}: Unable to found Function {self._fn} in Component."
+            )