flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/ExecuteSQL.py

@@ -0,0 +1,323 @@
+import asyncio
+from collections.abc import Callable
+from pathlib import PurePath
+from concurrent.futures import ThreadPoolExecutor
+from functools import partial
+import aiofiles
+from asyncdb.drivers.pg import pg
+from asyncdb.exceptions import StatementError, DataError
+from querysource.conf import (
+    default_dsn,
+    DB_TIMEOUT,
+    DB_STATEMENT_TIMEOUT,
+    DB_SESSION_TIMEOUT,
+    DB_KEEPALIVE_IDLE
+)
+from navconfig.logging import logging
+from ..exceptions import ComponentError, FileError
+from ..utils import SafeDict
+# TODO: migrate to FileStore component
+from .flow import FlowComponent
+from ..interfaces import TemplateSupport
+from ..conf import TASK_PATH
+from ..interfaces.qs import QSSupport
+
+
+class ExecuteSQL(QSSupport, FlowComponent, TemplateSupport):
+    """
+    ExecuteSQL
+
+    Overview
+
+        Executes one or more SQL queries against a PostgreSQL database, also can execute SQL's in a file.
+
+    **Properties** (inherited from FlowComponent)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | Name         | Required | Summary                                                           |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | skipError    |   No     | The name of the database schema to use (default: "").             |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | sql          |   No     | A raw SQL query string to execute.                                |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | file_sql     |   No     | A path (string) or list of paths (strings) to SQL files           |
+        |              |          | containing the queries to execute.                                |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | pattern      |   No     | A dictionary mapping variable names to functions that return      |
+        |              |          | the corresponding values to be used in the SQL query.             |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | use_template |   No     | Whether to treat the SQL string as a template and use the         |
+        |              |          | `_templateparser` component to render it (default: False).        |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | multi        |   No     | Whether to treat the `sql` property as a list of multiple         |
+        |              |          | queries to execute sequentially (default: False).                 |
+        +--------------+----------+-----------+-------------------------------------------------------+
+        | exec_timeout |   No     | The timeout value for executing a single SQL query                |
+        |              |          | (default: 3600 seconds).                                          |
+        +--------------+----------+-----------+-------------------------------------------------------+
+    **Methods**
+
+    * open_sqlfile(self, file: PurePath, **kwargs) -> str: Opens an SQL file and returns its content.
+    * get_connection(self, event_loop: asyncio.AbstractEventLoop): Creates a connection pool to the PostgreSQL database.
+    * _execute(self, query, event_loop): Executes a single SQL query asynchronously.
+    * execute_sql(self, query: str, event_loop: asyncio.AbstractEventLoop) -> str: Executes an SQL query and returns the result.
+
+    **Notes**
+
+    * This component uses asynchronous functions for non-blocking I/O operations.
+    * Error handling is implemented to catch exceptions during database connection, SQL execution, and file operations.
+    * Supports loading SQL queries from files.
+    * Supports using templates for dynamic SQL generation.
+    * Supports executing multiple queries sequentially.
+
+
+        Example:
+
+    ```yaml
+    ExecuteSQL:
+        file_sql: fill_employees.sql
+    ```
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.tablename: str = ""
+        self.schema: str = ""
+        self._connection: Callable = None
+        self._queries = []
+        self.exec_timeout: float = kwargs.pop(
+            "exec_timeout", 3600000.0
+        )
+        self._driver: str = kwargs.pop('driver', 'pg')
+        self.multi = bool(kwargs.pop('multi', False))
+        self.credentials = kwargs.pop('credentials', {})
+        self.use_template: bool = bool(kwargs.get('use_template', False))
+        self.use_dataframe: bool = bool(kwargs.get('use_dataframe', False))
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+        # set the program:
+        if hasattr(self, "program"):
+            self._program = self.program
+
+    async def close(self):
+        """Closing Database Connection."""
+        pass
+
+    async def open_sqlfile(self, file: PurePath, **kwargs) -> str:
+        content = None
+        self._logger.info(f"Open SQL File: {file}")
+        if file.exists() and file.is_file():
+            # open SQL File:
+            async with aiofiles.open(file, "r+") as afp:
+                content = await afp.read()
+                # check if we need to replace masks
+                if "{" in content:
+                    content = self.mask_replacement(content)
+            if self.use_template is True:
+                content = self._templateparser.from_string(content, kwargs)
+            return content
+        else:
+            raise FileError(f"ExecuteSQL: Missing SQL File: {file}")
+
+    async def start(self, **kwargs):
+        """Start Component"""
+        self.processing_credentials()
+        if self.previous:
+            self.data = self.input
+        # check if sql comes from a filename:
+        if hasattr(self, "file_sql"):
+            self._logger.debug(f"SQL File: {self.file_sql}")
+            self._queries = []
+            qs = []
+            if isinstance(self.file_sql, str):
+                qs.append(self.file_sql)
+            elif isinstance(self.file_sql, list):
+                qs = self.file_sql
+            else:
+                raise ComponentError(
+                    "ExecuteSQL: Unknown type for *file_sql* attribute."
+                )
+            for fs in qs:
+                self._logger.debug(f"Execute SQL File: {fs!s}")
+                file_path = TASK_PATH.joinpath(self._program, "sql", fs)
+                try:
+                    sql = await self.open_sqlfile(file_path)
+                    self._queries.append(sql)
+                except Exception as err:
+                    raise ComponentError(f"{err}") from err
+        if hasattr(self, "pattern"):
+            # need to parse variables in SQL
+            pattern = self.pattern
+            self._queries = []
+            try:
+                variables = {}
+                for field, val in pattern.items():
+                    variables[field] = self.getFunc(val)
+            except (TypeError, AttributeError) as err:
+                self._logger.error(err)
+            # replace all ocurrences on SQL
+            try:
+                # TODO: capture when sql is a list of queries
+                sql = self.sql.format_map(SafeDict(**variables))
+                # Replace variables
+                for val in self._variables:
+                    if isinstance(self._variables[val], list):
+                        if isinstance(self._variables[val], int):
+                            self._variables[val] = ", ".join(self._variables[val])
+                        else:
+                            self._variables[val] = ", ".join(
+                                "'{}'".format(v) for v in self._variables[val]
+                            )
+                    sql = sql.replace(
+                        "{{{}}}".format(str(val)), str(self._variables[val])
+                    )
+                self._queries.append(sql)
+            except Exception as err:
+                logging.exception(err, stack_info=True)
+        if hasattr(self, "sql"):
+            if isinstance(self.sql, str):
+                self._queries = [self.sql]
+            elif isinstance(self.sql, list):
+                self._queries = self.sql
+        # Replace variables
+        for val in self._variables:
+            sqls = []
+            for sql in self._queries:
+                if isinstance(self._variables[val], list):
+                    if isinstance(self._variables[val], int):
+                        self._variables[val] = ", ".join(self._variables[val])
+                    else:
+                        self._variables[val] = ", ".join(
+                            "'{}'".format(v) for v in self._variables[val]
+                        )
+                sql = sql.replace("{{{}}}".format(str(val)), str(self._variables[val]))
+                if hasattr(self, 'masks'):
+                    sql = self.mask_replacement(sql)
+                sqls.append(sql)
+            self._queries = sqls
+        return True
+
+    async def _execute(self, query, event_loop):
+        try:
+            connection = await self.create_connection(
+                driver=self._driver
+            )
+            async with await connection.connection() as conn:
+                if hasattr(self, "background"):
+                    future = asyncio.create_task(conn.execute(query))
+                    # query will be executed in background
+                    _, pending = await asyncio.wait(
+                        [future], timeout=self.exec_timeout, return_when="ALL_COMPLETED"
+                    )
+                    if future in pending:
+                        ## task reachs timeout
+                        for t in pending:
+                            t.cancel()
+                        raise asyncio.TimeoutError(
+                            f"Query {query!s} was cancelled due timeout."
+                        )
+                    result, error = future.result()
+                else:
+                    try:
+                        result, error = await conn.execute(query)
+                    except asyncio.TimeoutError as exc:
+                        raise asyncio.TimeoutError(
+                            f"Query {query!s} was cancelled due Timeout."
+                        ) from exc
+                    except Exception as exc:
+                        raise ComponentError(f"ExecuteSQL Error: {exc!s}") from exc
+                if error:
+                    raise ComponentError(
+                        f"Execute SQL error: {result!s} err: {error!s}"
+                    )
+                else:
+                    if self._driver == 'bigquery':
+                        return next(iter(result))
+                    return result
+        except StatementError as err:
+            raise StatementError(f"Statement error: {err}") from err
+        except DataError as err:
+            raise DataError(f"Data error: {err}") from err
+        except ComponentError:
+            raise
+        except Exception as err:
+            raise ComponentError(f"ExecuteSQL error: {err}") from err
+        finally:
+            connection = None
+
+    def execute_sql(self, query: str, event_loop: asyncio.AbstractEventLoop) -> str:
+        asyncio.set_event_loop(event_loop)
+        if self._debug:
+            self._logger.verbose(f"::: Exec SQL: {query}")
+        future = event_loop.create_task(self._execute(query, event_loop))
+        try:
+            result = event_loop.run_until_complete(future)
+            st = {"sql": query, "result": result}
+            self.add_metric("EXECUTED", st)
+            return result
+        except Exception as err:
+            self.add_metric("QUERY_ERROR", str(err))
+            self._logger.error(f"{err}")
+
+    async def run(self):
+        """Run Raw SQL functionality."""
+        try:
+            _new = True
+            event_loop = asyncio.new_event_loop()
+        except RuntimeError:
+            event_loop = asyncio.get_running_loop()
+            _new = False
+        ct = len(self._queries)
+        if ct <= 0:
+            ct = 1
+        result = []
+        try:
+            loop = asyncio.get_event_loop()
+            asyncio.set_event_loop(loop)
+            with ThreadPoolExecutor(max_workers=ct) as executor:
+                for query in self._queries:
+                    if self.use_dataframe is True:
+                        # Execute the Query for every row in dataframe:
+                        if self.data is not None:
+                            for _, row in self.data.iterrows():
+                                # Replace variables in SQL with values from dataframe
+                                # row to dict:
+                                data = row.to_dict()
+                                sql = query.format_map(SafeDict(**data))
+                                # Execute the SQL
+                                fn = partial(self.execute_sql, sql, event_loop)
+                                try:
+                                    res = await loop.run_in_executor(executor, fn)
+                                    result.append(res)
+                                except Exception as err:
+                                    self._logger.error(
+                                        f"ExecuteSQL error on query {sql!s}: {err!s}"
+                                    )
+                    else:
+                        # Execute the SQL
+                        fn = partial(self.execute_sql, query, event_loop)
+                        res = await loop.run_in_executor(executor, fn)
+                        result.append(res)
+        except ComponentError:
+            raise
+        except Exception as err:
+            raise ComponentError(f"{err}") from err
+        finally:
+            try:
+                if _new is True:
+                    event_loop.close()
+            except Exception:
+                pass
+        # returning the previous data:
+        if self.data is not None:
+            self._result = self.data
+        else:
+            self._result = result
+        return self._result

flowtask/components/ExtractHTML.py

@@ -0,0 +1,178 @@
+
+
+import asyncio
+from datetime import datetime
+from typing import Optional
+from collections.abc import Callable
+from bs4 import BeautifulSoup
+from lxml import etree
+from .flow import FlowComponent
+from ..interfaces.dataframes import PandasDataframe
+
+
+class ExtractHTML(FlowComponent, PandasDataframe):
+    """
+    ExtractHTML
+
+        Overview:
+        Extract HTML using XPATH or BS CSS Selectors.
+
+    
+
+        Example:
+
+        ```yaml
+        ExtractHTML:
+          custom_parser: trustpilot_reviews
+          as_dataframe: true
+        ```
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._xpath: Optional[str] = kwargs.get('xpath', None)
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self._input = self.input
+
+        if not self._input:
+            raise ValueError("No input provided.")
+
+        if not isinstance(self._input, dict):
+            raise TypeError("Input must be a dictionary.")
+
+    async def close(self):
+        pass
+
+    def get_soup(self, content: str, parser: str = 'html.parser'):
+        """Get a BeautifulSoup Object."""
+        return BeautifulSoup(content, parser)
+
+    def trustpilot_reviews(self, xml_obj, **kwargs):
+        xpath = '//article[@data-service-review-card-paper="true"]'
+        # Extract using XPATH
+        elements = xml_obj.xpath(xpath)
+        results = []
+        date_formats = [
+            "%b. %d, %Y",
+            "%B %d, %Y",
+            "%b %d, %Y",
+        ]
+        for el in elements:
+            soup = self.get_soup(etree.tostring(el))
+            # Extract using BS CSS Selectors
+            consumer_profile = soup.find('a', {"name": "consumer-profile"})
+            username = consumer_profile.find('span').get_text(strip=True)
+            user_url = consumer_profile.get('href')
+            rating_div = soup.find('div', {"data-service-review-rating": True})
+            # Extract title
+            title_h2 = soup.find("h2", {"data-service-review-title-typography": "true"})
+            review_title = title_h2.get_text(strip=True) if title_h2 else None
+            rating = rating_div.get('data-service-review-rating')
+            review_body = soup.find('p', {"data-service-review-text-typography": "true"})
+            review = review_body.get_text(strip=True) if review_body else None
+            review_date_p = soup.find('p', {"data-service-review-date-of-experience-typography": "true"})
+            review_date_str = review_date_p.get_text(strip=True) if review_date_p else None
+            review_date = None
+            if review_date_str:
+                if ":" in review_date_str:
+                    review_date_str = review_date_str.split(":", 1)[1].strip()
+                for date_format in date_formats:
+                    try:
+                        review_date = datetime.strptime(review_date_str, date_format)
+                        break 
+                    except ValueError:
+                        continue
+
+                if not review_date:
+                    print(f"Error to convert the date '{review_date_str}'")
+            if not review:
+                continue
+            results.append(
+                {
+                    "origin": "trustpilot",
+                    "username": username,
+                    #"user_url": user_url,
+                    "review_date": review_date,
+                    "rating": rating,
+                    #"title": review_title,
+                    "review": review
+                }
+            )
+        return results
+
+    def consumeraffairs_reviews(self, xml_obj, **kwargs):
+        xpath = '//div[@id="reviews-container"]//div[@itemprop="reviews"]'
+        elements = xml_obj.xpath(xpath)
+        results = []
+        date_formats = [
+            "%b. %d, %Y",
+            "%B %d, %Y",
+            "%b %d, %Y",
+        ]
+        for el in elements:
+            soup = self.get_soup(etree.tostring(el))
+            consumer_profile = soup.find('span', class_='rvw__inf-nm')
+            username = consumer_profile.get_text(strip=True) if consumer_profile else None
+            rating_div = soup.find('div', class_='rvw__hdr-stat')
+            rating_meta = rating_div.find('meta', itemprop='ratingValue') if rating_div else None
+            rating = rating_meta.get('content') if rating_meta else None
+            review_body_tag = soup.find('div', class_='rvw__top-text')
+            review = review_body_tag.get_text(strip=True) if review_body_tag else None
+            date_tag = soup.find('p', class_='rvw__rvd-dt')
+            review_date_str = date_tag.get_text(strip=True).replace("Reviewed ", "") if date_tag else None
+
+            review_date = None
+            if review_date_str:
+                review_date_str = review_date_str.replace("Reviewed ", "").replace("Updated review: ", "").replace("Original Review: ", "")
+                review_date_str = review_date_str.replace("Sept.", "Sep.")
+                for date_format in date_formats:
+                    try:
+                        review_date = datetime.strptime(review_date_str, date_format)
+                        break
+                    except ValueError:
+                        continue
+
+                if not review_date:
+                    print(f"Error to convert the date '{review_date_str}'")
+
+            if not review:
+                continue
+            results.append(
+                {
+                    "origin": "consumeraffairs",
+                    "username": username,
+                    "review_date": review_date,
+                    "rating": rating,
+                    "review": review
+                }
+            )
+        return results
+
+    async def run(self):
+        results = []
+        parser = None
+        if hasattr(self, 'custom_parser'):
+            parser = getattr(self, self.custom_parser, None)
+        for filename, result in self._input.items():
+            html_obj = result.get('html', None)
+            if html_obj is None:
+                raise ValueError("No HTML object found.")
+            if parser:
+                results += parser(html_obj)
+            else:
+                # Extract using BS CSS Selectors
+                pass
+        if getattr(self, 'as_dataframe', False) is True:
+            df = await self.create_dataframe(results)
+            self._result = df
+        else:
+            self._result = results
+        return self._result

flowtask/components/FileBase.py

@@ -0,0 +1,178 @@
+from abc import abstractmethod
+import logging
+import asyncio
+import glob
+from pathlib import PurePath, Path
+from collections.abc import Callable
+from ..exceptions import FileNotFound, ComponentError
+from ..utils.mail import MailMessage
+from ..utils import check_empty
+from .flow import FlowComponent
+
+
+class FileBase(FlowComponent):
+    """
+    FileBase
+
+    **Overview**
+
+        Abstract base class for file-based components.
+
+    **Properties** (inherited from FlowComponent)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+------------------------------------------------------------+
+        | Name               | Required | Summary                                                                |
+        +--------------------+----------+-----------+------------------------------------------------------------+
+        | create_destination |   No     | Boolean flag indicating whether to create the destination              |
+        |                    |          | directory if it doesn't exist (default: True).                         |
+        +--------------------+----------+-----------+------------------------------------------------------------+
+        | directory          |   Yes    | The path to the directory containing the files.                        |
+        +--------------------+----------+-----------+------------------------------------------------------------+
+        | filename           |   No     | A filename (string), list of filenames (strings), or a                 |
+        |                    |          | glob pattern (string) to identify files.                               |
+        +--------------------+----------+-----------+------------------------------------------------------------+
+        | file               |   No     | A pattern or filename (string) to identify files.                      |
+        |                    |          | (This property takes precedence over 'filename' if both are specified.)|
+        +--------------------+----------+-----------+------------------------------------------------------------+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        # self.directory: str = None
+        self._filenames: list[PurePath] = []
+        self._path: str = None
+        super(FileBase, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Check for File and Directory information."""
+        await super().start(**kwargs)
+        try:
+            if hasattr(self, "directory"):
+                try:
+                    directory = Path(self.directory)
+                    if not directory.is_absolute():
+                        self.directory = self._filestore.get_directory(self.directory)
+                    else:
+                        self.directory = directory
+                except TypeError:
+                    self.directory = self._filestore.default_directory("/")
+            # check for filename:
+            if self.previous and not check_empty(self.input):
+                if (
+                    not hasattr(self, "ignore_previous")
+                    or self.ignore_previous is False
+                ):
+                    if not isinstance(self.previous, FileBase):
+                        # avoid chaining components
+                        if isinstance(self.input, list):
+                            if isinstance(self.input[0], MailMessage):
+                                self._filenames = []
+                                for file in self.input:
+                                    for f in file.attachments:
+                                        fname = f["filename"]
+                                        logging.debug(
+                                            f"File: Detected attachment: {fname}"
+                                        )
+                                        self._filenames.append(fname)
+                            elif "files" in self.input:
+                                self._filenames = self.input["files"]
+                            else:
+                                self._filenames = self.input
+                            return True
+                        elif isinstance(self.input, dict):
+                            if "files" in self.input:
+                                # there is a "files" attribute in dictionary:
+                                self._filenames = self.input["files"]
+                        else:
+                            self._filenames = [self.input]
+                        return True
+            if hasattr(self, "filename"):
+                if isinstance(self.filename, list):
+                    for file in self.filename:
+                        self._filenames.append(self.directory.joinpath(file))
+                elif isinstance(self.filename, PurePath):
+                    self._filenames.append(self.filename)
+                elif isinstance(self.filename, str):
+                    if "*" in self.filename:
+                        # is a glob list of files
+                        path = self.directory.joinpath(self.filename)
+                        listing = glob.glob(str(path))  # TODO using glob from pathlib
+                        for fname in listing:
+                            logging.debug(f"Filename > {fname}")
+                            self._filenames.append(fname)
+                    else:
+                        self.filename = self.mask_replacement(self.filename)
+                        self._path = self.directory.joinpath(self.filename)
+                        self._filenames.append(self._path)
+                return True
+            elif hasattr(self, "file"):
+                filename = self.process_pattern("file")
+                if hasattr(self, "masks"):
+                    filename = self.mask_replacement(filename)
+                # path for file
+                self._path = self.directory.joinpath(filename)
+                listing = glob.glob(str(self._path))
+                if not listing:
+                    raise FileNotFound(
+                        f"FileExists: There are no files in {self._path}"
+                    )
+                for fname in listing:
+                    logging.debug(f"Filename > {fname}")
+                    self._filenames.append(fname)
+                    logging.debug(f" ::: Checking for Files: {self._filenames}")
+                return True
+        except (FileNotFound, ComponentError):
+            raise
+        except Exception as err:
+            raise ComponentError(
+                f"File: Invalid Arguments: {err!s}"
+            ) from err
+
+    def get_filelist(self) -> list[PurePath]:
+        """
+        Retrieves a list of files based on the component's configuration.
+
+        This method determines the list of files to process based on the component's
+        attributes such as 'pattern', 'file', or 'filename'. It applies any masks or
+        variables to the file patterns if specified.
+
+        Returns:
+            list[PurePath]: A list of PurePath objects representing the files to be processed.
+                            If no specific pattern or filename is set, it returns all files
+                            in the component's directory.
+        """
+        if hasattr(self, "pattern"):
+            value = self.pattern
+            if hasattr(self, "masks"):
+                for mask, replace in self._mask.items():
+                    value = str(value).replace(mask, replace)
+            if self._variables:
+                value = value.format(**self._variables)
+            files = (f for f in self.directory.glob(value))
+        elif hasattr(self, "file"):
+            # using pattern/file version
+            value = self.get_filepattern()
+            files = (f for f in self.directory.glob(value))
+        elif hasattr(self, "filename"):
+            # already discovered by start:
+            files = self._filenames
+        else:
+            files = (f for f in self.directory.iterdir() if f.is_file())
+        return files
+
+    @abstractmethod
+    async def run(self):
+        """Run File checking."""
+
+    @abstractmethod
+    async def close(self):
+        """Method."""