flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/ScrapPage.py

@@ -0,0 +1,216 @@
+"""
+Scrapping a Web Page Using Selenium + ChromeDriver + BeautifulSoup.
+
+
+        Example:
+
+        ```yaml
+        ScrapPage:
+          url: https://www.consumeraffairs.com/insurance/assurant-phone-insurance.html?page=2#sort=recent
+          rotate_ua: true
+          use_selenium: true
+          use_proxy: true
+          paid_proxy: true
+          as_mobile: false
+          timeout: 60
+          wait_until:
+          - - class_name
+            - js-rvw
+          screenshot:
+            filename: reviews_page.png
+            directory: /home/ubuntu/symbits/ejemplo/screenshots/
+          outputs:
+          - scraped_content
+          - screenshot_path
+        ```
+
+    """
+import asyncio
+from collections.abc import Callable
+from pathlib import Path, PurePath
+from typing import Optional
+from navconfig import BASE_DIR
+# Internals
+from ..exceptions import (
+    ComponentError,
+    DataNotFound,
+    TimeOutError
+)
+from .flow import FlowComponent
+from ..interfaces import HTTPService, SeleniumService
+
+
+class ScrapPage(SeleniumService, HTTPService, FlowComponent):
+    """ScrapPage.
+        Scrapping a Web Page using Selenium.
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.url: str = kwargs.get("url", None)
+        self.urls: list = kwargs.get('urls', [])
+        self.find_elements: list = kwargs.get('find_elements', [])
+        self.rotate_ua: bool = True
+        kwargs['rotate_ua'] = True  # Forcing UA Rotation.
+        self.use_selenium: bool = kwargs.pop(
+            "use_selenium",
+            False
+        )
+        # URL Function: generate the URL based on a function:
+        self.url_function: str = kwargs.pop('url_function', None)
+        # Return the Driver (avoid closing the Driver at the end of the process).
+        self.return_driver: bool = kwargs.pop('return_driver', False)
+        super(ScrapPage, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        # Fix the Headers for Scrapping:
+        self.headers: dict = {
+            "Host": self.extract_host(self.url),
+            **self.headers
+        }
+
+    async def start(self, **kwargs) -> bool:
+        await super(ScrapPage, self).start(**kwargs)
+        if self.use_selenium is True:
+            await self.get_driver()
+        # Generate a URL based on a URL Function:
+        if self.url_function:
+            fn = getattr(self, self.url_function, None)
+            if fn:
+                self.url = await fn()
+        return True
+
+    async def close(self, **kwargs) -> bool:
+        if self.use_selenium is True:
+            if self.return_driver is False:
+                self.close_driver()
+        return True
+
+    async def run_http(self):
+        """Run the Scrapping Tool Using HTTPx."""
+        result, error = await self.session(
+            url=self.url,
+            method=self.method,
+            headers=self.headers,
+            cookies=self.cookies,
+            follow_redirects=True,
+            use_proxy=self.use_proxy
+        )
+        if error:
+            raise ComponentError(
+                f"Error running HTTP: {error}"
+            )
+        if not result:
+            raise DataNotFound(
+                f"No content on URL {self.url}"
+            )
+        return result
+
+    async def run_selenium(self):
+        """Run the Scrapping Tool Using Selenium."""
+        try:
+            await self.get_page(self.url, self.cookies)
+            found_elements = None
+            file = None
+            content = None
+            if self.inner_tag:
+                content = self.driver().find_element(*self.inner_tag).get_attribute('innerHTML')
+            else:
+                content = self.driver().page_source
+            if hasattr(self, 'screenshot'):
+                # capture an screenshot from the page and save it (and returning as binary as well)
+                filename = self.screenshot.get('filename', 'screenshot.png')
+                directory = Path(self.screenshot.get(
+                    'directory', BASE_DIR.joinpath('static', 'images', 'screenshots')
+                ))
+                if not directory.is_absolute():
+                    directory = BASE_DIR.joinpath('static', 'images', directory)
+                if directory.exists() is False:
+                    directory.mkdir(parents=True, exist_ok=True)
+                # Take the screenshot
+                file = directory.joinpath(filename)
+                self.save_screenshot(str(file))
+            if self.find_elements:
+                # For the main content, fallback to page_source:
+                content = self.driver().page_source
+                try:
+                    print('FINDING > ', self.find_elements)
+                    elements = self.driver().find_elements(*self.find_elements)
+                    found_elements = elements
+                except Exception as exc:
+                    # Log the exception as a warning instead of raising it
+                    self.logger.warning(
+                        f"Unable to find elements with locator {self.find_elements}: {exc}"
+                    )
+                    found_elements = []
+            # Return the content of the page.
+            return content, file, found_elements
+        except (TimeOutError, ComponentError):
+            raise
+        except Exception as exc:
+            raise ComponentError(
+                f"Error running Selenium: {exc}"
+            )
+
+    def _build_result_content(self, content: str, screenshot: PurePath, found_elements: Optional[list] = None) -> dict:
+        """Build the Result Content."""
+        soup = self.get_soup(content)
+        _xml, _html = self.get_etree(content)
+        result = {
+            "raw": content,
+            "soup": soup,
+            "html": _html,
+            "xml": _xml,
+            "screenshot": screenshot
+        }
+        if found_elements is not None:
+            result["found_elements"] = found_elements
+        return result
+
+    async def run(self):
+        """Run the Scrapping Tool."""
+        self._result = None
+        screenshot = None
+        found_elements = None
+        try:
+            if self.use_selenium is True:
+                if self.urls:
+                    results = []
+                    for url in self.urls:
+                        self.url = url
+                        self.headers['Host'] = self.extract_host(url)
+                        self.headers['Referer'] = url
+                        content, screenshot, found_elements = await self.run_selenium()
+                        result = self._build_result_content(content, screenshot, found_elements)
+                        results.append(
+                            {"url": url, "content": result}
+                        )
+                    self._result = results
+                    return results
+                else:
+                    content, screenshot, found_elements = await self.run_selenium()
+            else:
+                content = await self.run_http()
+            if not content:
+                raise DataNotFound(
+                    f"No content on URL {self.url}"
+                )
+        except ComponentError:
+            raise
+        except Exception as exc:
+            raise ComponentError(
+                f"Error running ScrapPage: {exc}"
+            )
+        if self.return_driver is True:
+            self._result = self.driver()
+        else:
+            self._result = self._build_result_content(content, screenshot, found_elements)
+        return self._result

flowtask/components/ScrapSearch.py

@@ -0,0 +1,79 @@
+from bs4 import BeautifulSoup
+from .ScrapPage import ScrapPage
+from ..exceptions import ComponentError
+
+
+class ScrapSearch(ScrapPage):
+    """Search by a Product, retrieve the URL (based on rules) and scrap the page."""
+    # TODO: Idea is for making the search functions pluggable.
+    def __init__(self, **kwargs):
+        self.find_element: tuple = kwargs.pop('find_element', ('li', {'class': ['sku-item']}))
+        super().__init__(**kwargs)
+        self.product_sku: str = kwargs.pop('product_sku', None)
+        self.brand: str = kwargs.pop('brand', None)
+
+    async def _bby_products(self):
+        front_url = "https://www.bestbuy.com/site/searchpage.jsp?cp="
+        middle_url = "&searchType=search&st="
+        page_count = 1
+        # TODO: Get the Brand and Model from the Component.
+        model = self.product_sku
+        brand = self.brand
+        search_term = f'{brand}%20{model}'
+        end_url = "&_dyncharset=UTF-8&id=pcat17071&type=page&sc=Global&nrp=&sp=&qp=&list=n&af=true&iht=y&usc=All%20Categories&ks=960&keys=keys"  # noqa
+        url = front_url + str(page_count) + middle_url + search_term + end_url
+        print('SEARCH URL: ', url)
+        return url
+
+    async def _search_bby_products(self, content: str) -> str:
+        soup = BeautifulSoup(content, 'html.parser')
+        # Find all elements with class "sku-item"
+        product_items = soup.find_all(*self.find_element)
+        # Iterate over each product item
+        url = None
+        for item in product_items:
+            # Get the "data-sku-id" attribute
+            sku_id = item.get("data-sku-id")
+            # Check if the SKU ID matches your target SKU ID
+            if sku_id == self.product_sku:
+                print(f"Found matching SKU ID: {sku_id}")
+                # Now look for the child with class "sku-title"
+                pd = item.find('h4', {'class': ['sku-title']})
+                anchor = pd.a
+                url = "https://www.bestbuy.com{url}".format(
+                    url=anchor['href']
+                )
+                print('Product URL: ', url)
+        return url
+
+    async def run(self):
+        # Run works differently for ScrapPage:
+        self._result = None
+        screenshot = None
+        # 1. Get the Product List URL
+        fn = getattr(self, f"_{self.url_function}", None)
+        if not fn:
+            raise ComponentError(
+                f"Function {self.url_function} not found."
+            )
+        url = await fn()
+        if self.use_selenium is True:
+            await self.get_page(url, self.cookies)
+            content = self.driver().page_source
+            # 2. Search the Product
+            search_fn = getattr(self, f"_search_{self.url_function}", None)
+            if search_fn:
+                url = await search_fn(content)
+                if url:
+                    # 3. Get the URL
+                    self.url = url
+                    # 4. Run the Scrapping Tool to extract the Product page.
+                    content, screenshot = await self.run_selenium()
+                    if self.return_driver is True:
+                        self._result = self.driver()
+                    else:
+                        self._result = self._build_result_content(
+                            content,
+                            screenshot
+                        )
+        return self._result

flowtask/components/SendNotify.py

@@ -0,0 +1,257 @@
+import os
+import asyncio
+from typing import Dict, List
+from collections.abc import Callable, Iterable
+from pathlib import Path
+from navconfig.logging import logging
+from notify import Notify
+from notify.models import Actor
+from ..exceptions import ComponentError, FileNotFound
+from .flow import FlowComponent
+from ..interfaces import DBSupport
+
+
+def expand_path(filename: str) -> Iterable[Path]:
+    p = Path(filename)
+    return Path(p.parent).expanduser().glob(p.name)
+
+
+class SendNotify(DBSupport, FlowComponent):
+    """
+    SendNotify
+
+        Overview
+
+            The SendNotify class is a component for sending notifications to a list of recipients via various
+            channels (e.g., email) using the Notify component. It supports adding attachments, templating messages
+            with masked variables, and utilizing custom credentials for authentication.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | Name           | Required | Summary                                                                          |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | via            |   Yes    | The method for sending the notification, e.g., "email".                          |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | account        |   Yes    | A dictionary with server credentials, including `host`, `port`,                  |
+            |                |          | `username`, and `password`.                                                      |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | recipients     |   Yes    | List of dictionaries with target user details for notification.                  |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | list           |   No     | Optional mailing list name for retrieving recipients from the database.          |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | attachments    |   No     | List of file paths for attachments to include in the notification.               |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+            | message        |   Yes    | Dictionary with the notification message content, supporting template variables. |
+            +----------------+----------+-----------+----------------------------------------------------------------------+
+
+        Returns
+
+            This component returns the input data after sending the notification. Metrics are recorded for each
+            successful send, with details on recipients and the message content. If any specified attachment file
+            is missing, a `FileNotFound` exception is raised. If there are errors in setting up or sending the
+            notification, a `ComponentError` is raised with descriptive messages.
+    
+
+        Example:
+
+        ```yaml
+        SendNotify:
+          via: email
+          account:
+            hostname: NAVIGATOR_ALERT_EMAIL_HOSTNAME
+            port: NAVIGATOR_ALERT_EMAIL_PORT
+            password: NAVIGATOR_ALERT_EMAIL_PASSWORD
+            username: NAVIGATOR_ALERT_EMAIL_USERNAME
+          attachments:
+          - /home/ubuntu/symbits/bose/files/report/troc_open_tickets_{today}.csv
+          masks:
+            '{today}':
+            - today
+            - mask: '%Y-%m-%d'
+            '{yesterday}':
+            - yesterday
+            - mask: '%Y-%m-%d'
+            '{human-today}':
+            - today
+            - mask: '%m/%d/%Y'
+            '{human-yesterday}':
+            - yesterday
+            - mask: '%m/%d/%Y'
+            '#today-timestamp#':
+            - current_timestamp
+            - tz: America/New_York
+          recipients:
+          - name: Carlos Rivero
+            account:
+              address: crivero@trocglobal.com
+          - name: Arturo Martinez
+            account:
+              address: amartinez@trocglobal.com
+          - name: Jhoanir Torres
+            account:
+              address: jhtorres@trocglobal.com
+          - name: Steven Greenstein
+            account:
+              address: sgreenstein@trocglobal.com
+          - name: Sabra Pierre
+            account:
+              address: spierre1@trocglobal.com
+          - name: Daniel McGee
+            account:
+              address: dmcgee@trocglobal.com
+          - name: Kile Harris
+            account:
+              address: kharris10@trocglobal.com
+          - name: Gerardo Espinoza
+            account:
+              address: gespinoza@trocglobal.com
+          - name: Christopher Harmon
+            account:
+              address: charmon@trocglobal.com
+          message:
+            subject: T-ROC BOSE Break&Fix Open Tickets ({human-today})
+            message_content: Please find attached the report generated on {human-today}.This
+              is an automated message - please do not reply directly to this email.
+            template: email_custom_report.html
+            clientName: Bose
+            dateGenerated: '{human-today}'
+            created_at: '#today-timestamp#'
+        ```
+
+    """  # noqa
+
+    _credentials: dict = {
+        "hostname": str,
+        "port": int,
+        "username": str,
+        "password": str,
+    }
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.attachments: List = []
+        self.list_attachment: List = []
+        self.notify: Callable = None
+        self.recipients: List = []
+        self._recipients: List = []
+        self.via: str = "email"
+        self.message: Dict = {}
+        # renaming account to credentials in kwargs:
+        if "account" in kwargs:
+            kwargs["credentials"] = kwargs.pop("account")
+        super(SendNotify, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+
+    def status_sent(self, recipient, message, result, *args, **kwargs):
+        print(
+            f"Notification with status {result!s} to {recipient.account!s}"
+        )
+        logging.info(f"Notification with status {result!s} to {recipient.account!s}")
+        status = {"recipient": recipient, "result": result}
+        self.add_metric("Sent", status)
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        await super().start(**kwargs)
+        self.processing_credentials()
+        # TODO: generate file from dataset (dataframe)
+        # using mailing list:
+        if hasattr(self, "list"):
+            # getting the mailing list:
+            lst = self.list
+            sql = f"SELECT * FROM troc.get_mailing_list('{lst!s}')"
+            try:
+                connection = self.get_connection()
+                async with await connection.connection() as conn:
+                    result, error = await conn.query(sql)
+                    if error:
+                        raise ComponentError(
+                            f"CreateReport: Error on Recipients: {error!s}."
+                        )
+                    for r in result:
+                        actor = Actor(**dict(r))
+                        self._recipients.append(actor)
+            except Exception as err:
+                logging.exception(err)
+        else:
+            # determine the recipients:
+            try:
+                self._recipients = [Actor(**user) for user in self.recipients]
+            except Exception as err:
+                raise ComponentError(f"Error formatting Recipients: {err}") from err
+        if not self._recipients:
+            raise ComponentError("SendNotify: Invalid Number of Recipients.")
+        if hasattr(self, "masks"):
+            for _, attach in enumerate(self.attachments):
+                attachment = self.mask_replacement(attach)
+                # resolve filenames:
+                files = expand_path(attachment)
+                for file in files:
+                    self.list_attachment.append(file)
+            # Mask transform of message
+            for key, value in self.message.items():
+                self.message[key] = self.mask_replacement(value)
+                self._logger.notice(
+                    f"Variable: {key} = {self.message[key]}"
+                )
+        # Verify if file exists
+        for file in self.list_attachment:
+            if not file.exists():
+                raise FileNotFound(
+                    f"File doesn't exists: {file}"
+                )
+        return True
+
+    async def close(self):
+        if self.notify:
+            try:
+                await self.notify.close()
+            except Exception as err:
+                print(err)
+
+    async def run(self):
+        """
+        Running the Notification over all recipients.
+        """
+        self._result = self.data  # by-pass override data (pass-through)
+        if self.data is not None:
+            if isinstance(self.data, list):
+                self.message['filenames'] = self.data
+            elif isinstance(self.data, dict):
+                self.message.update(self.data)
+
+        # create the notify component
+        account = {**self.credentials}
+
+        try:
+            self.notify = Notify(self.via, loop=self._loop, **account)
+            self.notify.sent = self.status_sent
+        except Exception as err:
+            raise ComponentError(f"Error Creating Notification App: {err}") from err
+        try:
+            result = await self.notify.send(
+                recipient=self._recipients,
+                attachments=self.list_attachment,
+                **self.message,
+            )
+            logging.debug(f"Notification Status: {result}")
+            # add metric:
+            self.add_metric("Notification", self.message)
+        except Exception as err:
+            raise ComponentError(f"SendNotify Error: {err}") from err
+        if self.data is None:
+            return True
+        return self._result