flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/Loop.py

@@ -0,0 +1,253 @@
+from collections.abc import Callable, Awaitable
+import asyncio
+from typing import Any
+from pandas import DataFrame
+from asyncdb.exceptions import NoDataFound, ProviderError, DriverError
+from .flow import FlowComponent
+from ..exceptions import (
+    ComponentError,
+    DataNotFound,
+    NotSupported,
+    FileNotFound
+)
+from ..utils.stats import StepMonitor
+from ..interfaces.log import SkipErrors
+
+
+class Loop(FlowComponent):
+    """Loop.
+
+    Overview:
+        The Loop class is a FlowComponent that is used to iterate over the next Component and execute them in a
+        sequential order.
+        It extends the FlowComponent class and provides methods for starting tasks, retrieving steps,
+        and executing jobs asynchronously.
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._conditions: dict = {}
+        self._generator = kwargs.get("generator", None)  # Predefined generator
+        self._iterate: bool = kwargs.get('iterate', False)
+        self._iterable = kwargs.get("iterable", None)    # Custom iterable
+        self._done: bool = False  # Flag to indicate if iteration is complete
+        # Component to be executed when finished.
+        self._ondone: str = kwargs.get("onDone", None)
+        super(Loop, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """
+        start.
+
+            Initialize (if needed) a task
+        """
+        if self.previous:
+            self.data = self.input
+
+        # check if previous data is an iterable:
+        if isinstance(self.data, DataFrame):
+            self._iterable = self.data.iterrows()
+        elif self._iterate is True or hasattr(self.data, '__iter__'):
+            self._iterable = self.data
+        elif self._generator:
+            self._iterable = self._resolve_generator(self._generator)
+        elif self._iterable and not hasattr(self._iterable, "__iter__"):
+            raise ComponentError(
+                "'iterable' must be an iterable object."
+            )
+
+        self._iterator = iter(self._iterable)
+        return True
+
+    def _resolve_generator(self, generator_name):
+        """
+        Resolve predefined generators like 'days_of_week' or 'days_of_month'.
+        """
+        # Define predefined generators here.
+        # Each generator should return a list of values.
+        # For example, 'days_of_week' could return a list of weekdays,
+        # and 'days_of_month' could return a list of days of the month.
+        predefined_generators = {
+            "days_of_week": lambda: ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"],
+            "days_of_month": lambda: [f"Day {i}" for i in range(1, 32)],
+        }
+        if generator_name not in predefined_generators:
+            raise ComponentError(
+                f"Unknown generator: {generator_name}"
+            )
+        return predefined_generators[generator_name]()
+
+    async def close(self):
+        pass
+
+    def get_component(self, step, idx):
+        params = None
+        try:
+            if not self._TaskPile:
+                raise ComponentError(
+                    "No Components in TaskPile"
+                )
+            params = step.params()
+            try:
+                if params["conditions"]:
+                    self._conditions[step.name] = params["conditions"]
+            except KeyError:
+                pass
+            params["ENV"] = self._environment
+            # params
+            params["params"] = self._params
+            # parameters
+            params["parameters"] = self._parameters
+            # useful to change variables in set var components
+            params["_vars"] = self._vars
+            # variables dictionary
+            params["variables"] = self._variables
+            params["_args"] = self._args
+            # argument list for components (or tasks) that need argument lists
+            params["arguments"] = self._arguments
+            # for components with conditions, we can add more conditions
+            conditions = params.get("conditions", {})
+            step_conds = self._conditions.get(step.name, {})
+            if self.conditions is not None:
+                step_conds = {**self.conditions, **step_conds}
+            params["conditions"] = {**conditions, **step_conds}
+            # attributes only usable component-only
+            params["attributes"] = self._attributes
+            # the current Pile of components
+            params["TaskPile"] = self._TaskPile
+            # params['TaskName'] = step_name
+            params["debug"] = self._debug
+            params["argparser"] = self._argparser
+            # the current in-memory connector
+            params["memory"] = self._memory
+            target = step.component
+            # remove this element from tasks, doesn't need to run again
+            self._TaskPile.delStep(idx)
+            # return target and params
+            return [target, params]
+        finally:
+            pass
+
+    def create_component(self, target, value: Any = None, **params):
+        """get_component.
+
+        Create a new component instance.
+        """
+        try:
+            return target(
+                job=self,
+                loop=self._loop,
+                stat=self.stat,
+                input_result=value,
+                **params
+            )
+        except Exception as err:
+            raise ComponentError(
+                f"Component Error on {target}: {err}"
+            ) from err
+
+    async def exec_component(self, job, step_name):
+        start = getattr(job, "start", None)
+        if callable(start):
+            try:
+                if asyncio.iscoroutinefunction(start):
+                    st = await job.start()
+                else:
+                    st = job.start()
+                self._logger.debug(f"STARTED: {st}")
+            except (NoDataFound, DataNotFound) as err:
+                raise DataNotFound(f"{err!s}") from err
+            except (ProviderError, ComponentError, NotSupported) as err:
+                raise ComponentError(
+                    f"Error running Start Function on {step_name}, error: {err}"
+                ) from err
+        else:
+            raise ComponentError(
+                f"Error running Function on {step_name}"
+            )
+        try:
+            run = getattr(job, "run", None)
+            if asyncio.iscoroutinefunction(run):
+                result = await job.run()
+            else:
+                result = job.run()
+            self._result = result
+            return self._result
+        except (NoDataFound, DataNotFound, FileNotFound) as err:
+            try:
+                if job.skipError == SkipErrors.SKIP:
+                    self._logger.warning(
+                        f"Component {job!s} was Skipped, error: {err}"
+                    )
+                    self._result = self.data
+                    return self._result
+                elif job.skipError == SkipErrors.ENFORCE:
+                    # Enforcing to Raise Error:
+                    raise DataNotFound(f"{err!s}") from err
+                else:
+                    # Log Only
+                    self._logger.error(
+                        f"Component {job!s} was Skipped, error: {err}"
+                    )
+            except AttributeError:
+                raise DataNotFound(f"{err!s}") from err
+        except (ProviderError, ComponentError, NotSupported) as err:
+            raise NotSupported(
+                f"Error running Component {step_name}, error: {err}"
+            ) from err
+        except Exception as err:
+            self._logger.exception(err, exc_info=True)
+            raise ComponentError(
+                f"Iterator Error on {step_name}, error: {err}"
+            ) from err
+        finally:
+            try:
+                close = getattr(job, "close", None)
+                if asyncio.iscoroutinefunction(close):
+                    await job.close()
+                else:
+                    job.close()
+            except Exception:
+                pass
+
+    async def run(self):
+        """Async Run Method."""
+        # iterate over next Component
+        step, idx = self._TaskPile.nextStep(self.StepName)
+        target, params = self.get_component(step, idx)
+        step_name = step.name
+        i = 0
+        results = []
+        while True:
+            try:
+                # Get the next item from the iterator
+                item = next(self._iterator)
+                self.setTaskVar('value', item)
+                cp = self.create_component(target, item, **params)
+                try:
+                    result = await self.exec_component(cp, step_name)
+                    results.append(result)
+                    i += 1
+                except (NoDataFound, DataNotFound) as err:
+                    # its a data component a no data was found
+                    self._logger.notice(
+                        f"Data not Found over {step_name} at {i} iteration, got: {err}"
+                    )
+                    i += 1
+                    continue
+            except StopIteration:
+                self._done = True
+                break  # Exit loop when iteration is complete
+        # when iteration is finished, return the results collected:
+        if self.stat:
+            parent_stat = self.stat.parent()
+            stat = StepMonitor(name=step_name, parent=parent_stat)
+            parent_stat.add_step(stat)
+            stat.add_metric('ITERATIONS', i)
+        self._result = results
+        return self._result

flowtask/components/Lowes.py

@@ -0,0 +1,334 @@
+"""
+Scrapping a Web Page Using Selenium + ChromeDriver + BeautifulSoup.
+
+
+        Example:
+
+        ```yaml
+        Lowes:
+          type: reviews
+          use_proxies: true
+          paid_proxy: true
+          api_token: xxx
+        ```
+
+    """
+import asyncio
+from collections.abc import Callable
+import random
+import httpx
+import pandas as pd
+import backoff
+# Internals
+from ..exceptions import (
+    ComponentError,
+    ConfigError,
+    NotSupported
+)
+from ..interfaces.http import ua
+from .reviewscrap import ReviewScrapper, bad_gateway_exception
+
+
+class Lowes(ReviewScrapper):
+    """Lowes.
+
+    Combining API Key and Web Scrapping, this component will be able to extract
+    Lowes Information (reviews, etc).
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        super(Lowes, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        # Always use proxies:
+        self.use_proxy: bool = True
+        self._free_proxy: bool = False
+        self.cookies = {
+            "dbidv2": "956aa8ea-87f3-4068-96a8-3e2bdf4e84ec",
+            "al_sess": "FuA4EWsuT07UWryyq/3foLUcOGRVVGi7yYKO2imCjWnuWxkaJXwqJRDEw8CjJaWJ",
+            # Add other necessary cookies here
+            # Ensure tokens are valid and not expired
+        }
+        self.headers: dict = {
+            "Accept": "application/json",
+            "Accept-Encoding": "gzip, deflate, br, zstd",
+            "Accept-Language": "es-US,es;q=0.9,en-US;q=0.8,en;q=0.7,es-419;q=0.6",
+            "Cache-Control": "no-cache",
+            "Connection": "keep-alive",
+            "Content-Type": "application/json",
+            "Host": "www.lowes.com",
+            "Pragma": "no-cache",
+            "Origin": "https://www.lowes.com",
+            "Referer": "https://www.lowes.com/pd/",
+            "Sec-CH-UA": '"Not A(Brand";v="8", "Chromium";v="132", "Google Chrome";v="132"',
+            "Sec-CH-UA-Mobile": "?0",
+            "Sec-CH-UA-Platform": '"Linux"',
+            'sec-fetch-site': 'none',
+            'sec-fetch-mode': 'navigate',
+            'sec-fetch-dest': 'document',
+            "Sec-Fetch-Site": "same-origin",
+            "User-Agent": random.choice(ua)
+        }
+        self.semaphore = asyncio.Semaphore(10)
+
+    async def close(self, **kwargs) -> bool:
+        self.close_driver()
+        return True
+
+    async def start(self, **kwargs) -> bool:
+        await super(Lowes, self).start(**kwargs)
+        if self.previous:
+            self.data = self.input
+            if not isinstance(self.data, pd.DataFrame):
+                raise ComponentError(
+                    "Incompatible Pandas Dataframe"
+                )
+        self.api_token = self.get_env_value(self.api_token) if hasattr(self, 'api_token') else self.get_env_value('TARGET_API_KEY')  # noqa
+        if not hasattr(self, self._fn):
+            raise ConfigError(
+                f"BestBuy: Unable to found Function {self._fn} in Target Component."
+            )
+
+    @backoff.on_exception(
+        backoff.expo,
+        (httpx.ConnectTimeout, httpx.HTTPStatusError),
+        max_tries=2,
+        giveup=lambda e: not bad_gateway_exception(e) and not isinstance(e, httpx.ConnectTimeout)
+    )
+    async def _product_reviews(self, idx, row, cookies):
+        async with self.semaphore:
+            # Prepare payload for the API request
+            sku = row['sku']
+            page_size = 10  # fixed size
+            current_offset = 0
+            max_pages = 20  # Maximum number of pages to fetch
+            all_reviews = []
+            total_reviews = 0
+            try:
+                while current_offset < max_pages * page_size:
+                    if current_offset == 0:
+                        payload = {
+                            "sortBy": "newestFirst"
+                        }
+                    else:
+                        payload = {
+                            "sortBy": "newestFirst",
+                            "offset": current_offset
+                        }
+                    url = f"https://www.lowes.com/rnr/r/get-by-product/{sku}"
+                    result = await self.api_get(
+                        url=url,
+                        cookies=cookies,
+                        params=payload,
+                        headers=self.headers
+                    )
+                    if not result:
+                        self._logger.warning(
+                            f"No Product Reviews found for {sku}."
+                        )
+                        break
+                    # Extract the reviews data from the API response
+                    reviews_section = result.get('results', [])
+                    total_reviews = result.get('totalResults', 0)
+                    if not reviews_section:
+                        self._logger.info(f"No more reviews found for SKU {sku} at offset {current_offset}.")
+                        break
+                    if len(reviews_section) == 0:
+                        break
+                    all_reviews.extend(reviews_section)
+                    # Update total_reviews
+
+                    # Check if we've fetched all reviews
+                    if len(all_reviews) >= total_reviews:
+                        self._logger.info(f"Fetched all reviews for SKU {sku}.")
+                        break
+                    current_offset += page_size  # Move to the next page
+            except (httpx.TimeoutException, httpx.HTTPError) as ex:
+                self._logger.warning(f"Request failed: {ex}")
+                return []
+            except Exception as ex:
+                self._logger.error(f"An error occurred: {ex}")
+                return []
+
+            # Extract the reviews data from the API response
+            reviews = []
+            for item in all_reviews:
+                # Exclude certain keys
+                # Extract relevant fields
+                # Combine with original row data
+                review_data = row.to_dict()
+                review = {
+                    **review_data,
+                    "id": item.get("id"),
+                    "legacyId": item.get("legacyId"),
+                    "title": item.get("title"),
+                    "review": item.get("reviewText"),
+                    "rating": item.get("rating"),
+                    "isRecommended": item.get("isRecommended"),
+                    "userNickname": item.get("userNickname"),
+                    "submissionTime": item.get("submissionTime"),
+                    "verifiedPurchaser": item.get("verifiedPurchaser"),
+                    "helpfulVoteCount": item.get("helpfulVoteCount"),
+                    "notHelpfulVoteCount": item.get("notHelpfulVoteCount"),
+                    "clientResponses": item.get("clientResponses"),
+                    "relevancyScore": item.get("relevancyScore"),
+                    "productId": item.get("productId"),
+                }
+                review['total_reviews'] = total_reviews
+                # Optionally, handle client responses
+                if review["clientResponses"]:
+                    # For simplicity, concatenate all responses into a single string
+                    responses = []
+                    for response in review["clientResponses"]:
+                        response_text = response.get("response", "")
+                        responses.append(response_text.strip())
+                    review["clientResponses"] = " | ".join(responses)
+                reviews.append(review)
+            self._logger.info(
+                f"Fetched {len(reviews)} reviews for SKU {sku}."
+            )
+            return reviews
+
+    async def reviews(self):
+        """reviews.
+
+        Target Product Reviews.
+        """
+        httpx_cookies = httpx.Cookies()
+        for key, value in self.cookies.items():
+            httpx_cookies.set(
+                key, value,
+                domain='.lowes.com',
+                path='/'
+            )
+
+        # Iterate over each row in the DataFrame
+        print('starting ...')
+
+        tasks = [
+            self._product_reviews(
+                idx,
+                row,
+                httpx_cookies
+            ) for idx, row in self.data.iterrows()
+        ]
+        # Gather results concurrently
+        all_reviews_nested = await self._processing_tasks(tasks)
+
+        # Flatten the list of lists
+        all_reviews = [review for reviews in all_reviews_nested for review in reviews]
+
+        # Convert to DataFrame
+        reviews_df = pd.DataFrame(all_reviews)
+
+        # at the end, adding a column for origin of reviews:
+        reviews_df['origin'] = 'lowes'
+
+        # show the num of rows in final dataframe:
+        self._logger.notice(
+            f"Ending Product Reviews: {len(reviews_df)}"
+        )
+
+        # Override previous dataframe:
+        self.data = reviews_df
+
+        # return existing data
+        return self.data
+
+    @backoff.on_exception(
+        backoff.expo,
+        (httpx.ConnectTimeout, httpx.HTTPStatusError),
+        max_tries=2,
+        giveup=lambda e: not bad_gateway_exception(e) and not isinstance(e, httpx.ConnectTimeout)
+    )
+    async def _product_details(self, idx, row, cookies):
+        async with self.semaphore:
+            # Prepare payload for the API request
+            sku = row['sku']
+            storeid = row['store_id']
+            zipcode = row['zipcode'],
+            state_code = row['state_code']
+            payload = {
+                "nearByStore": storeid,
+                "zipState": state_code,
+                "quantity": 1
+            }
+            try:
+                # url = "https://www.lowes.com/lowes-proxy/wpd/1000379005/productdetail/1845/Guest/60639?nearByStore=1845&zipState=IL&quantity=1"
+                # url = f"https://www.lowes.com/wpd/{sku}/productdetail/{storeid}/Guest/{zipcode}"
+                url = f"https://www.lowes.com/lowes-proxy/wpd/{sku}/productdetail/{storeid}/Guest/{zipcode}"
+                result = await self.api_get(
+                    url=url,
+                    # cookies=cookies,
+                    # params=payload,
+                    headers=self.headers
+                )
+                if not result:
+                    self._logger.warning(
+                        f"No Product Details found for {sku}."
+                    )
+                    return []
+                # Extract the product details data from the API response
+                print('RESULT > ', result)
+            except (httpx.TimeoutException, httpx.HTTPError) as ex:
+                self._logger.warning(f"Request failed: {ex}")
+                return []
+            except Exception as ex:
+                print(ex)
+                self._logger.error(f"An error occurred: {ex}")
+                return []
+
+    async def product_details(self):
+        """product_details.
+
+        Get Product Details from Lowes URL.
+        """
+        self.cookies = {}
+        httpx_cookies = httpx.Cookies()
+        for key, value in self.cookies.items():
+            httpx_cookies.set(
+                key, value,
+                domain='.lowes.com',
+                path='/'
+            )
+
+        # Iterate over each row in the DataFrame
+        print('starting ...')
+
+        tasks = [
+            self._product_details(
+                idx,
+                row,
+                httpx_cookies
+            ) for idx, row in self.data.iterrows()
+        ]
+        # Gather results concurrently
+        all_products_nested = await self._processing_tasks(tasks)
+
+        # Flatten the list of lists
+        all_products = [product for products in all_products_nested for product in products]
+
+        # Convert to DataFrame
+        _df = pd.DataFrame(all_products)
+
+        # at the end, adding a column for origin of reviews:
+        _df['origin'] = 'lowes'
+
+        # show the num of rows in final dataframe:
+        self._logger.notice(
+            f"Ending Product Details: {len(_df)}"
+        )
+
+        # Override previous dataframe:
+        self.data = _df
+
+        # return existing data
+        return self.data