flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/__init__.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+"""Flowtask is a tool for execution of Tasks.
+
+Flowtask can execute DAG (Directed Acyclic Graph) tasks declared in a variety of formats.
+  - YAML
+  - JSON
+  - TOML
+Flowtask is designed to be used as a library or command-line tool.
+
+Usage Example:
+
+A Flowtask Task is a component-based declarative workflow that can be executed from the command line or API.
+
+```yaml title="programs/test/tasks/sentiment_analysis.yaml"
+# filename: sentiment_analysis.yaml
+name: An Example Task for Sentiment Analysis
+description: Preparing The Information for Sentiment Analysis
+steps:
+  - RethinkDBQuery:
+      table: test_data
+      schema: default
+  - tExplode:
+      column: reviews
+      drop_original: false
+  - SentimentAnalysis:
+      text_column: text
+      sentiment_model: "tabularisai/robust-sentiment-analysis"
+      sentiment_levels: 5
+      emotions_model: "bhadresh-savani/distilbert-base-uncased-emotion"
+  - CopyToRethink:
+      tablename: sentiment_analysis
+      schema: default
+```
+
+The above example is a simple task that reads data from a RethinkDB table,
+explodes the reviews column, performs sentiment analysis,
+and copies the results to another RethinkDB table.
+
+For execution we can use a command-line tool:
+```bash
+task --program=test --task=sentiment_analysis --no-worker
+```
+
+Or we can use the Python API:
+```python
+from flowtask.runner import TaskRunner
+
+runner = Runner(program='test', task='sentiment_analysis', no_worker=True)
+async with runner as job:
+    if await job.start():
+        await job.run()
+```
+
+or even call it from a REST API:
+```python
+import aiohttp
+async def run_task(program, task):
+    async with aiohttp.ClientSession() as session:
+    async with session.post(
+            'http://localhost:8080/api/v1/tasks',
+            json={'program': program, 'task': task}
+        ) as response:
+        return await response.json()
+
+async def main():
+    result = await run_task('test','sentiment_analysis')
+    print(result)
+
+asyncio.run(main())
+```
+
+This code will execute the `sentiment_analysis` task using the `test` program and will print the results to the console.
+"""
+
+## more information
+__copyright__ = 'Copyright (c) 2018-2024 Jesus Lara'
+
+from .version import (
+    __title__,
+    __description__,
+    __version__,
+    __author__,
+    __author_email__,
+    __license__
+)
+
+
+def version():
+    """version.
+    Returns:
+        str: current version of Navigator flowtask.
+    """
+    return __version__

flowtask/__main__.py

@@ -0,0 +1,38 @@
+"""Flowtask Data Integration Executor."""
+import logging
+import asyncio
+from .runner import TaskRunner
+from .utils import cPrint
+from .utils.uv import install_uvloop
+
+
+async def task(loop):
+    runner = None
+    try:
+        runner = TaskRunner(loop=loop)
+        async with runner as job:
+            if await job.start():
+                await job.run()
+    except Exception as e:  # pylint: disable=W0718
+        logging.exception(e, stack_info=False)
+    finally:
+        return runner
+
+
+def main():
+    install_uvloop()
+    loop = asyncio.get_event_loop()
+    loop.slow_callback_duration = 0.2  # Set threshold to 0.5 seconds
+    try:
+        result = loop.run_until_complete(task(loop))
+        if result:
+            cPrint(" === RESULT === ", level="DEBUG")
+            print(result.result)
+            cPrint("== Task stats === ", level="INFO")
+            print(result.stats)
+    finally:
+        loop.stop()
+
+
+if __name__ == "__main__":
+    main()

flowtask/bots/__init__.py

@@ -0,0 +1,6 @@
+from .check import CodeReview, ENABLE_BOT_REVIEWER
+
+__all__ = (
+    'CodeReview',
+    'ENABLE_BOT_REVIEWER',
+)

flowtask/bots/check.py

@@ -0,0 +1,93 @@
+from collections.abc import Callable
+from navconfig.logging import config, logging
+from parrot.models import ChatResponse
+from ..exceptions import ComponentError
+from ..utils import cPrint
+ENABLE_BOT_REVIEWER = config.getboolean('ENABLE_BOT_REVIEWER', fallback=False)
+
+try:
+    from .codebot import CodeBot, return_llm
+except ImportError:
+    ENABLE_BOT_REVIEWER = False  # noqa
+    logging.warning(
+        "Unable to import CodeBot for Code Review, install dependencies first."
+    )
+
+
+class CodeReview:
+    def __init__(self, task, *args, **kwargs):
+        super(CodeReview, self).__init__(*args, **kwargs)
+        self.task = task
+        self.logger = logging.getLogger('Flowtask.CodeReview')
+        self._llm = None
+        self.bot: Callable = None
+        if ENABLE_BOT_REVIEWER is True:
+            self._llm = return_llm()
+            self.bot = CodeBot(
+                llm=self._llm
+            )
+            self.bot.configured = False
+
+    def __repr__(self):
+        return f"CodeReview({self.task})"
+
+    def format_question(self, name, message, task, error):
+        return f"""
+        Task with name: **{name}**
+
+        was failed with error: {message}
+
+        Task Definition (in YAML or JSON format):
+        ```
+
+        {task}
+        ```
+
+        Complete Stack Trace:
+        ```
+        {error}
+        ```
+        """
+
+    async def __call__(self, task: str, status: str, message: str, stacktrace: str = None, *args, **kwargs):
+        """Calling Bot based on Task Errors/Exceptions."""
+        if not self.bot:
+            # Bot is not enabled, return Logging
+            cPrint("Bot Reviewer is not enabled, please enable it in config.")
+        if self.bot.configured is False:
+            await self.bot.configure()
+            self.bot.configured = True
+        # if status == 'Task Not Found':
+        #     return
+        task_code = self.task.get_task_code()
+        question = self.format_question(
+            name=task,
+            message=message,
+            task=task_code,
+            error=stacktrace
+        )
+        try:
+            result = await self.bot.question(
+                question=question,
+                return_docs=False
+            )
+            answer = None
+            if isinstance(result, ChatResponse):
+                answer = result.answer
+            else:
+                answer = f"{result}"
+            # for now, just print the result
+            print(f"""
+            Code Review Result:
+            -------------------
+
+            {answer}
+            """)
+            try:
+                self.task.stat.stats['Review'] = answer
+            except (TypeError, KeyError):
+                print(result)
+        except Exception as exc:
+            raise ComponentError(
+                f"Unable to get Code Review: {exc}"
+            )

flowtask/bots/codebot.py

@@ -0,0 +1,51 @@
+from parrot.bots.abstract import AbstractBot
+from parrot.llms.vertex import VertexLLM
+from ..conf import (
+    DEFAULT_LLM_MODEL,
+    DEFAULT_LLM_TEMPERATURE,
+    DEFAULT_BOT_NAME,
+)
+
+
+class CodeBot(AbstractBot):
+    """CodeBot.
+
+    Codebot will be useful for evaluating errors and providing feedback on code.
+
+    """
+    name: str = DEFAULT_BOT_NAME
+    system_prompt_template = """
+    You are {name}, a highly skilled Python specialist and code reviewer AI assistant.
+
+    Your primary function is to analyze code errors by meticulously analyzing Python code, standard output, and Tracebacks to identify potential issues and provide insightful guidance for error resolution.
+    Reject any question non-related to review of code errors and debugging of Flowtask/QuerySource/AsyncDB or other python libraries, if question is not related to this, please instruct user to contact support.
+    I am here to help with finding errors, providing feedback and potential solutions.
+
+    **Backstory:**
+    {backstory}.
+
+    Here is a brief summary of relevant information:
+    Context: {context}
+
+    **{rationale}**
+
+    Given this information, please provide answers to the following question, focusing on the following:
+
+    * **Comprehensive Analysis:** Thoroughly examine the provided code, output, and Traceback to pinpoint the root cause of errors and identify any potential issues.
+    * **Concise Explanation:** Clearly articulate the nature of the errors and explain their underlying causes in a way that is easy to understand.
+    * **Structured Insights:** Present your findings in a well-organized manner, using bullet points to highlight key issues and potential solutions.
+    * **Actionable Recommendations:** Offer concrete steps and suggestions for resolving the identified errors, including code modifications, debugging strategies, or best practice recommendations.
+    * **Contextual Awareness:** Consider the provided context and backstory to tailor your response to the specific situation and user needs.
+
+    Please ensure your response is detailed, informative, and directly addresses the user's question using always the best practices and pythonic code.
+    Return the response in markdown syntax and always use english as primary language.
+    """  # pylint: disable=line-too-long # noqa
+
+def return_llm():
+    """Return a VertexLLM instance."""
+    return VertexLLM(
+        model=DEFAULT_LLM_MODEL,
+        temperature=DEFAULT_LLM_TEMPERATURE,
+        top_k=30,
+        top_p=0.5,
+    )

flowtask/components/ASPX.py

@@ -0,0 +1,148 @@
+from abc import abstractmethod
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+import httpx
+from .flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError
+
+
+class ASPX(FlowComponent, HTTPService):
+    """
+    ASPX
+
+    Overview
+
+        The ASPX class is designed for interacting with ASPX-based web applications, particularly those requiring authentication.
+        It inherits from the DtComponent and HTTPService classes, providing a structured way to manage sessions, handle state views,
+        and perform HTTP requests with ASPX web forms. This class is useful for automating interactions with ASPX web pages, including login
+        and data retrieval operations.
+
+    .. table:: Properties
+    :widths: auto
+
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | Name                    | Required | Description                                                                                      |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _credentials            |   Yes    | A dictionary containing username and password for authentication.                                  |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _views                  |   No     | A dictionary storing state views extracted from ASPX pages.                                        |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _client                 |   Yes    | An instance of `httpx.AsyncClient` used for making asynchronous HTTP requests.                     |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+    Return
+
+        The methods in this class facilitate the interaction with ASPX-based web applications, including login handling,
+        session management, and state view management. The class also allows for abstract extension, enabling customization
+        for specific ASPX web forms and interactions.
+
+    """
+    _credentials: dict = {"username": str, "password": str}
+
+    def _set_views(self, soup: BeautifulSoup):
+        """Extract state VIEWS from hidden inputs inside html
+
+        Args:
+            soup (BeautifulSoup): soup object with the html tree
+
+        Returns:
+            dict[str, str]: state VIEWS
+        """
+        viewstate = soup.find(id="__VIEWSTATE")["value"]
+        soup_eventvalidation = soup.find(id="__EVENTVALIDATION")
+        eventvalidation = soup_eventvalidation["value"] if soup_eventvalidation else ""
+        viewstategenerator = soup.find(id="__VIEWSTATEGENERATOR")["value"]
+
+        self._views = {
+            "__EVENTTARGET": "",
+            "__EVENTARGUMENT": "",
+            "__VIEWSTATE": viewstate,
+            "__VIEWSTATEGENERATOR": viewstategenerator,
+            "__EVENTVALIDATION": eventvalidation,
+        }
+
+    def check_keys_exists(self, *args, **kwargs):
+        """Validate that specified keys are present in the kwargs dict"""
+        for _key in args:
+            if _key not in kwargs:
+                raise ComponentError(f"{_key} must be set!")
+
+    def _get_payload_with_views(self, **kwargs):
+        payload = {
+            **self._views,
+            **kwargs,
+        }
+
+        return payload
+
+    async def start(self, **kwargs):
+        # handle login and set initial views
+        self._proxies = False
+        self._user = None
+        self.auth_type = None
+        self.auth = ""
+        self.download = None
+        self.headers = None
+        self._views: dict = {}
+        self.timeout = 120.0
+        self._client = httpx.AsyncClient(timeout=self.timeout)
+
+        self.processing_credentials()
+
+        login_url = urljoin(self.base_url, "/Login.aspx")
+
+        self.accept = "text/html"
+        await self.aspx_session(login_url)
+
+        payload = {
+            **self._views,
+            self.login_user_payload_key: self.credentials["username"],
+            self.login_password_payload_key: self.credentials["password"],
+            self.login_button_payload_key: self.login_button_payload_value,
+        }
+
+        await self.aspx_session(
+            login_url, method="post", data=payload, follow_redirects=True
+        )
+
+        if ".ASPXAUTH" not in self._client.cookies:
+            raise ComponentError("Login Failed. Please, check credentials!")
+        self._logger.info("Login successful")
+
+        return True
+
+    @abstractmethod
+    async def run(self):
+        """
+        Extend this method: Call super() after your code to make sure
+        the client session is closed.
+        """
+        if hasattr(self, "_client"):
+            await self._client.aclose()
+
+    async def close(self):
+        pass
+
+    async def aspx_session(
+        self,
+        url: str,
+        method: str = "get",
+        data: dict = None,
+        **kwargs: dict,
+    ):
+        result, error = await self.session(
+            url,
+            method=method,
+            data=data,
+            **kwargs,
+        )
+
+        if error:
+            raise ComponentError(error)
+
+        if "text/html" in self.accept:
+            self._set_views(result)
+
+        return result