flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/ServiceScrapper/parsers/__init__.py

@@ -0,0 +1 @@
+from .costco import CostcoScrapper

flowtask/components/ServiceScrapper/parsers/base.py

@@ -0,0 +1,94 @@
+import asyncio
+from typing import Any, List, Dict
+from abc import abstractmethod
+import pandas as pd
+from bs4 import BeautifulSoup as bs
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.common.exceptions import (
+    TimeoutException,
+    NoSuchElementException,
+    WebDriverException
+)
+from ....interfaces import SeleniumService, HTTPService
+import re
+import logging
+
+class ScrapperBase(SeleniumService, HTTPService):
+    """
+    ScrapperBase Model.
+
+
+    Define how scrappers should be work.-
+    """
+    url: str = ''
+    domain: str = ''
+    cookies: Any
+    expected_columns: List[str] = []
+
+    def __init__(self, *args, **kwargs):
+        self._driver = None
+        self.cookies = kwargs.get('cookies', None)
+        self._logger = logging.getLogger(self.__class__.__name__)
+        self._counter: int = 0
+        self.search_term_used: str = ''
+        self.url = kwargs.get('url', self.url)
+        self.use_proxy = True
+        self._free_proxy = False
+        super().__init__(*args, **kwargs)
+
+    def get_bs(self, response: object) -> Any:
+        if isinstance(response, str):
+            return bs(response, 'html.parser')
+        return bs(response.text, 'html.parser')
+
+    async def get(self, url: str, headers: dict = None):
+        try:
+            try:
+                await asyncio.sleep(1)
+                await self.get_page(url=url)
+                # Driver Wait until body is available:
+                # WebDriverWait(self._driver, 10).until(
+                #     EC.presence_of_element_located((By.TAG_NAME, 'body'))
+                # )
+                return self._driver.page_source
+            except TimeoutException:
+                return None
+        except Exception as err:
+            self._logger.error(f'Error getting page: {err}')
+            return None
+
+    async def get_http(self, url, headers: dict = None):
+        return await self._get(url, headers=headers, use_proxy=True)
+
+    async def start(self):
+        """Starts de Navigation to Main Site.
+        """
+        if self.url:
+            response = await self.get(self.url)
+            return response
+        return True
+
+    def set_columns(self, df: pd.DataFrame) -> None:
+        for col in self.expected_columns:
+            if col not in df.columns:
+                df[col] = None
+
+    @abstractmethod
+    async def connect(self):
+        """Creates the Driver and Connects to the Site.
+        """
+
+    @abstractmethod
+    async def disconnect(self):
+        """Disconnects the Driver and closes the Connection.
+        """
+
+    # Context Manager:
+    async def __aenter__(self):
+        await self.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        await self.disconnect()

flowtask/components/ServiceScrapper/parsers/costco.py

@@ -0,0 +1,93 @@
+from .base import ScrapperBase
+
+
+class CostcoScrapper(ScrapperBase):
+    domain: str = 'costco.com'
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.expected_columns = [
+            'brand_name',
+            'brand_image',
+            'short_name',
+            'brand_description',
+        ]
+        # self.url: str = 'https://www.costco.com/special-events.html'
+        self.url: str = None
+        self.use_proxy = True
+        self.us_proxy = True
+        self._free_proxy = False
+        # self.use_edge = True
+        # self.use_firefox = True
+        self.headless = False
+        self.use_wire = False
+        # self._browser_binary = '/opt/google/chrome/chrome'
+        # self._driver_binary = '/home/jesuslara/.cache/selenium/geckodriver/linux64/0.33.0/geckodriver'
+
+    async def connect(self):
+        """Creates the Driver and Connects to the Site.
+        """
+        self._driver = await self.get_driver()
+        await self.start()
+
+    async def disconnect(self):
+        """Disconnects the Driver and closes the Connection.
+        """
+        if self._driver:
+            self.close_driver()
+
+    async def special_events(self, response: object, idx: int, row: dict) -> tuple:
+        """
+        Get the special events from Costco.
+        """
+        try:
+            document = self.get_bs(response)
+            category_header = document.find('div', {'id': 'category-name-header'})
+            print('category_header > ', category_header)
+            return idx, row
+        except Exception as err:
+            self._logger.error(f'Error getting special events from Costco: {err}')
+            return None
+
+    async def product_information(self, response: object, idx: int, row: dict) -> tuple:
+        """
+        Get the product information from Costco.
+        """
+        try:
+            document = self.get_bs(response)
+            search_results_div = document.find('div', {'id': 'search-results'})
+            if search_results_div:
+                # 1. Get the brand name: find the <div> with class "search-results-tile", then the first <h1>
+                search_results_tile = search_results_div.find('div', class_="search-results-tile")
+                brand_name = None
+                if search_results_tile:
+                    h1_tag = search_results_tile.find('h1')
+                    if h1_tag:
+                        brand_name = h1_tag.get_text(strip=True)
+                        row['brand_name'] = brand_name
+                # 2. Get the brand image: find the <div> with class "dual-row", then find the <img> with class "img-responsive"
+                dual_row_div = search_results_div.find('div', class_="dual-row")
+                brand_image = None
+                if dual_row_div:
+                    img_tag = dual_row_div.find('img', class_="img-responsive")
+                    if img_tag and img_tag.has_attr('src'):
+                        brand_image = img_tag['src']
+                        row['brand_image'] = brand_image
+                # 3. Get the brand description: find the <div> with class "sp-event-product-copy" then its first <p>
+                copy_div = search_results_div.find('div', class_="sp-event-product-copy")
+                brand_description = None
+                if copy_div:
+                    p_tag = copy_div.find('p')
+                    if p_tag:
+                        brand_description = p_tag.get_text(strip=True)
+                        row['brand_description'] = brand_description
+                # 4. Get the short name: find the <div> with class "search-results-tile",
+                _div = search_results_div.find('div', class_="sp-event-product-title")
+                short_name = None
+                if _div:
+                    short_name = _div.get_text(strip=True)
+                    row['short_name'] = short_name
+            return idx, row
+        except Exception as err:
+            self._logger.error(f'Error getting product information from Costco: {err}')
+            return None

flowtask/components/ServiceScrapper/scrapper.py

@@ -0,0 +1,199 @@
+from collections.abc import Callable
+import asyncio
+import random
+import httpx
+import pandas as pd
+from tqdm.asyncio import tqdm
+from bs4 import BeautifulSoup
+from ...exceptions import ComponentError, ConfigError
+from ...interfaces import HTTPService, SeleniumService
+from ...interfaces.http import ua
+from ..flow import FlowComponent
+from .parsers import (
+    CostcoScrapper,
+)
+
+
+class ServiceScrapper(FlowComponent, SeleniumService, HTTPService):
+    """
+    Service Scraper Component
+
+    Overview:
+
+    Pluggable component for scrapping several services and sites using different scrapers.
+
+    .. table:: Properties
+    :widths: auto
+
+    +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+    | Name                  | Required | Description                                                                                          |
+    +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+    | url_column (str)      |   Yes    | Name of the column containing URLs to scrape (default: 'search_url')                                |
+    +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+    | wait_for (tuple)      |   No     | Element to wait for before scraping (default: ('class', 'company-overview'))                        |
+    +-----------------------+----------+------------------------------------------------------------------------------------------------------+
+
+    Return:
+    - DataFrame with company information
+    """  # noqa: E501
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.info_column: str = kwargs.get('column_name', 'url')
+        self.scrapper_name: str = kwargs.get('scrapper', ['costco'])
+        self.scrapper_class: Callable = None
+        self._scrapper_func: str = kwargs.get('function', 'special_events')
+        self.wait_for: tuple = kwargs.get('wait_for', ('class', 'company-overview'))
+        self._counter: int = 0
+        self.use_proxy: bool = True
+        self._free_proxy: bool = False
+        self.paid_proxy: bool = True
+        self.chunk_size: int = kwargs.get('chunk_size', 100)
+        self.concurrently: bool = kwargs.get('concurrently', False)
+        self.task_parts: int = kwargs.get('task_parts', 10)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        # Headers configuration
+        self.headers: dict = {
+            "Accept": self.accept,
+            "TE": "trailers",
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": random.choice(ua),
+            **kwargs.get('headers', {})
+        }
+        self._free_proxy = False
+
+    def split_parts(self, task_list, num_parts: int = 5) -> list:
+        part_size, remainder = divmod(len(task_list), num_parts)
+        parts = []
+        start = 0
+        for i in range(num_parts):
+            # Distribute the remainder across the first `remainder` parts
+            end = start + part_size + (1 if i < remainder else 0)
+            parts.append(task_list[start:end])
+            start = end
+        return parts
+
+    async def _processing_tasks(self, tasks: list) -> pd.DataFrame:
+        """Process tasks concurrently."""
+        results = []
+        total_tasks = len(tasks)
+        # Overall progress bar
+        with tqdm(total=total_tasks, desc="Scraping Progress", unit="task") as pbar_total:
+            if self.concurrently is False:
+                # run every task in a sequential manner:
+                for task in tasks:
+                    try:
+                        idx, row = await task
+                        results.append((idx, row))  # Append as tuple (idx, row)
+                        await asyncio.sleep(
+                            random.uniform(0.25, 1.5)
+                        )
+                    except Exception as e:
+                        self._logger.error(f"Task error: {str(e)}")
+                        results.append((idx, row))  # Store the failure result
+                    finally:
+                        pbar_total.update(1)
+            else:
+                for chunk in self.split_parts(tasks, self.task_parts):
+                    result = await asyncio.gather(*chunk, return_exceptions=False)
+                    results.extend(result)
+                # Convert results to DataFrame
+        if not results:
+            return pd.DataFrame()
+
+        indices, data_dicts = zip(*results) if results else ([], [])
+        df = pd.DataFrame(data_dicts, index=indices)
+        return df
+
+    async def start(self, **kwargs) -> bool:
+        """Initialize the component and validate required parameters."""
+        if self.previous:
+            self.data = self.input
+
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError(
+                "Input must be a DataFrame", status=404
+            )
+
+        if self.info_column not in self.data.columns:
+            raise ConfigError(
+                f"Column {self.info_column} not found in DataFrame"
+            )
+
+        return True
+
+    async def _start_scrapping(self, idx, row):
+        try:
+            async with self._semaphore:
+                url = row[self.info_column]
+                self._logger.debug(
+                    f"Scraping URL: {url}"
+                )
+                async with self.scrapper_class as scrapper:
+                    response = await scrapper.get(url, headers=self.headers)
+                    if response:
+                        try:
+                            fn = getattr(scrapper, self._scrapper_func, 'scrapping')
+                            idx, row = await fn(response, idx, row)
+                        except Exception as err:
+                            print(' Exception caught > ', err)
+                            return idx, row
+            return idx, row
+        except Exception as err:
+            print(' Exception caught > ', err)
+            raise ComponentError(
+                f"Error while scrapping: {err}"
+            )
+
+    def _get_scrapper(self):
+        """Get the scrapper class."""
+        try:
+            if self.scrapper_name == 'costco':
+                return CostcoScrapper()
+            else:
+                return None
+        except Exception as err:
+            print(err)
+            raise ConfigError(
+                f"Error while getting scrapper: {err}"
+            )
+
+    async def run(self):
+        """Execute scraping for requested URL in the DataFrame."""
+        # function that import the specified scrapper
+        try:
+            self.scrapper_class = self._get_scrapper()
+        except Exception as err:
+            raise ConfigError(
+                f"Error while getting scrapper: {err}"
+            )
+        httpx_cookies = self.get_httpx_cookies(
+            domain=self.scrapper_class.domain, cookies=self.cookies
+        )
+        self.scrapper_class.cookies = httpx_cookies
+        self.scrapper_class.set_columns(self.data)
+        if not self.scrapper_class:
+            raise ConfigError(
+                "No valid scrapper were found or provided in configuration"
+            )
+        tasks = [
+            self._start_scrapping(
+                idx, row
+            ) for idx, row in self.data.iterrows()
+        ]
+        df = await self._processing_tasks(tasks)
+        self._result = df
+        self._print_data_(self._result, 'Scrapper Results')
+        return self._result
+
+    async def close(self):
+        """Clean up resources."""
+        return True

flowtask/components/SetVariables.py

@@ -0,0 +1,156 @@
+import asyncio
+from typing import Callable
+from dateutil import parser
+import pandas as pd
+from ..exceptions import ComponentError
+from .flow import FlowComponent
+from ..utils.executor import getFunction
+
+
+class SetVariables(FlowComponent):
+    """
+    SetVariables
+
+        Overview
+
+            The SetVariables class is a component for extracting values from data and setting them as variables
+            for use in other components. This component can set variables based on specific column values in
+            a DataFrame or by executing functions, with support for date formatting and value aggregation.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | vars           |   Yes    | Dictionary defining variables to set with options for format,             |
+            |                |          | row selection, and data sources.                                          |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns the original data after setting variables based on the `vars` dictionary.
+            Each variable is created from a specified column or function, and supports formatting options
+            such as date, timestamp, epoch, or custom string formatting. Metrics are recorded for each variable
+            set, and any issues with variable definitions or data retrieval raise a descriptive `ComponentError`.
+    
+
+        Example:
+
+        ```yaml
+        SetVariables:
+          vars:
+            max_date:
+            - order_date
+            - row: max
+            min_date:
+            - order_date
+            - row: min
+        ```
+
+    """
+
+    data = None
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        super(SetVariables, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+
+    def close(self):
+        pass
+
+    async def run(self):
+        if hasattr(self, "vars"):
+            for var, params in self.vars.items():
+                variable = ""
+                fname = ""
+                try:
+                    fname = params[0]
+                except (KeyError, IndexError, ValueError) as err:
+                    raise ComponentError(
+                        f"Error Getting the variable definition: {err}"
+                    ) from err
+                try:
+                    fmt = params[1]["format"]
+                except (KeyError, IndexError, ValueError):
+                    fmt = None
+                # Si existe una columna llamada fname en el dataframe se saca de ahí
+                if isinstance(self.data, pd.DataFrame) and fname in self.data.columns:
+                    try:
+                        try:
+                            row = params[1]["row"]
+                        except Exception:
+                            row = 0
+                        if isinstance(row, int):
+                            variable = self.data.iloc[row][fname]
+                        elif row == "min":
+                            variable = self.data[fname].min()
+                        elif row == "max":
+                            variable = self.data[fname].max()
+                        elif row == "array":
+                            variable = self.data[fname].unique().tolist()
+                        if fmt is not None:
+                            # apply Format
+                            if fmt == "date":
+                                # convert to a date:
+                                _var = parser.parse(str(variable))
+                                variable = _var.strftime("%Y-%m-%d")
+                            elif fmt == "timestamp":
+                                _var = parser.parse(str(variable))
+                                variable = _var.strftime("%Y-%m-%d %H:%M:%S")
+                            elif fmt == "epoch":
+                                _var = parser.parse(str(variable))
+                                variable = _var.strftime("%s")
+                            else:
+                                try:
+                                    _var = parser.parse(variable)
+                                    variable = _var.strftime(fmt)
+                                except (parser.ParserError, Exception):
+                                    # f-string formatting:
+                                    variable = f"{variable:fmt}"
+                    except Exception as err:
+                        print('E ', err)
+                        raise ComponentError(
+                            f"Error Getting the variable definition: {err}"
+                        ) from err
+                # Si no existe se saca de una función
+                else:
+                    try:
+                        func = getFunction(fname)
+                        if callable(func):
+                            try:
+                                args = params[1]
+                                variable = func(**args)
+                            except (KeyError, IndexError, ValueError):
+                                variable = func()
+                            if fmt is not None:
+                                print("VAR ", variable)
+                    except Exception as err:
+                        raise ComponentError(
+                            f"Error Getting the variable definition: {err}"
+                        ) from err
+                self.add_metric(
+                    f"{self.StepName}_{var!s}", variable
+                )
+                self._variables[f"{self.StepName}_{var}"] = variable
+                self._logger.debug(
+                    f"Set Variable: {self.StepName}_{var} to {variable!s}"
+                )
+                print("VAR: ", f"{self.StepName}_{var}", variable)
+        self._result = self.data
+        return self._result