flowtask 5.11.11__tar.gz → 5.12.1__tar.gz

flowtask-5.12.1/PKG-INFO

@@ -0,0 +1,229 @@
+Metadata-Version: 2.4
+Name: flowtask
+Version: 5.12.1
+Summary: Framework for Task orchestration
+Author-email: Jesus Lara <jesuslarag@gmail.com>
+License: Apache-2.0
+Project-URL: Homepage, https://github.com/phenobarbital/flowtask
+Project-URL: Source, https://github.com/phenobarbital/flowtask
+Project-URL: Funding, https://paypal.me/phenobarbital
+Project-URL: Say Thanks!, https://saythanks.io/to/phenobarbital
+Keywords: Flowtask,Data Integration,Task Orchestration,Task-Runner,Pipelines
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: Topic :: Software Development :: Libraries :: Application Frameworks
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Classifier: Topic :: Software Development :: Build Tools
+Classifier: Topic :: Utilities
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Framework :: AsyncIO
+Requires-Python: <3.14,>=3.11
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: borax==3.5.0
+Requires-Dist: PyDrive==1.3.1
+Requires-Dist: chardet==5.2.0
+Requires-Dist: aiohttp-jinja2==1.6
+Requires-Dist: asyncssh[bcrypt,fido2,pkcs11,pyopenssl]==2.23.0
+Requires-Dist: pyxlsb==1.0.10
+Requires-Dist: python-calamine==0.2.3
+Requires-Dist: pyecharts==2.0.8
+Requires-Dist: selenium>=4.35.0
+Requires-Dist: snapshot-selenium==0.0.2
+Requires-Dist: httpx[http2,socks]>=0.26.0
+Requires-Dist: urllib3[socks]>=2.3.0
+Requires-Dist: requests==2.32.4
+Requires-Dist: h2>=4.2.0
+Requires-Dist: blinker>=1.9.0
+Requires-Dist: webdriver-manager==4.0.2
+Requires-Dist: aioimaplib==1.1.0
+Requires-Dist: adal==1.2.7
+Requires-Dist: xlrd>=2.0.2
+Requires-Dist: xmltodict<1.0.0,>=0.14.2
+Requires-Dist: zeep[async]==4.3.2
+Requires-Dist: nltk>=3.9.3
+Requires-Dist: jdcal==1.4.1
+Requires-Dist: html5lib==1.1
+Requires-Dist: shapely==2.0.6
+Requires-Dist: tzwhere==3.0.3
+Requires-Dist: tabulate==0.9.0
+Requires-Dist: python-magic==0.4.27
+Requires-Dist: pytomlpp==1.0.13
+Requires-Dist: psutil==7.2.2
+Requires-Dist: networkx>=3.4.1
+Requires-Dist: watchdog>=6.0.0
+Requires-Dist: hachiko==0.4.0
+Requires-Dist: paramiko>=4.0.0
+Requires-Dist: jira==3.10.5
+Requires-Dist: datatable==1.1.0
+Requires-Dist: imageio==2.37.0
+Requires-Dist: decorator>=5.1.1
+Requires-Dist: emoji==2.14.1
+Requires-Dist: caio==0.9.24
+Requires-Dist: Wand==0.6.13
+Requires-Dist: pylibdmtx==0.1.10
+Requires-Dist: aiofile>=3.9.0
+Requires-Dist: filetype==1.2.0
+Requires-Dist: proxylists>=0.14.0
+Requires-Dist: aioftp==0.23.1
+Requires-Dist: py7zr==0.22.0
+Requires-Dist: rarfile==4.2
+Requires-Dist: python-pptx==1.0.2
+Requires-Dist: gspread==6.2.0
+Requires-Dist: oauth2client>=4.1.3
+Requires-Dist: google-auth>=2.35.0
+Requires-Dist: google-cloud-language>=2.15.0
+Requires-Dist: googlemaps==4.10.0
+Requires-Dist: fastavro==1.9.7
+Requires-Dist: pgpy==0.6.0
+Requires-Dist: dropbox==12.0.2
+Requires-Dist: dask-expr==1.1.13
+Requires-Dist: jsonpath_ng==1.7.0
+Requires-Dist: ortools<=9.14.6206,>=9.11.4210
+Requires-Dist: gmqtt==0.7.0
+Requires-Dist: selectorlib==0.16.0
+Requires-Dist: playwright==1.52.0
+Requires-Dist: fuzzywuzzy==0.18.0
+Requires-Dist: praw==7.8.1
+Requires-Dist: prawcore==2.4.0
+Requires-Dist: osmnx==2.0.1
+Requires-Dist: osmium==4.0.2
+Requires-Dist: python-Levenshtein==0.26.1
+Requires-Dist: undetected_chromedriver==3.5.5
+Requires-Dist: duckduckgo-search==8.1.1
+Requires-Dist: numba>=0.62.1
+Requires-Dist: APScheduler<3.12.0,>=3.11.0
+Requires-Dist: google-cloud-bigquery>=3.30.0
+Requires-Dist: pyheif==0.8.0
+Requires-Dist: filetype==1.2.0
+Requires-Dist: exif==1.6.1
+Requires-Dist: pillow-avif-plugin==1.5.2
+Requires-Dist: pillow-heif==0.22.0
+Requires-Dist: pgvector==0.4.1
+Requires-Dist: proxylists>=0.14.0
+Requires-Dist: async-notify[all]>=1.4.0
+Requires-Dist: querysource>=3.17.0
+Requires-Dist: asyncdb[all]>=2.11.7
+Requires-Dist: navconfig[default,uvloop]>=1.7.13
+Requires-Dist: navigator-auth>=0.15.8
+Requires-Dist: navigator-api[locale,uvloop]>=2.12.22
+Requires-Dist: ai-parrot[llms]>=0.17.4
+Requires-Dist: msal>=1.30.0
+Requires-Dist: azure-core==1.38.0
+Requires-Dist: azure-identity==1.20.0
+Requires-Dist: microsoft-kiota-authentication-azure==1.9.6
+Requires-Dist: msgraph-sdk==1.22.0
+Requires-Dist: zendriver>=0.15.2
+Requires-Dist: pyarrow>=14.0
+Provides-Extra: docker
+Requires-Dist: aiodocker>=0.21.0; extra == "docker"
+Provides-Extra: k8s
+Requires-Dist: kr8s>=0.16.0; extra == "k8s"
+Provides-Extra: git
+Requires-Dist: gitpython>=3.1.50; extra == "git"
+Provides-Extra: ai
+Requires-Dist: ai-parrot[agents,embeddings,images,integrations,llms,scheduler]>=0.20.4; extra == "ai"
+Requires-Dist: ai-parrot-loaders[audio,document,ebook,ml,pdf,scraping,video,web,youtube]>=0.1.0; extra == "ai"
+Requires-Dist: ai-parrot-tools[analysis,arxiv,aws,codeinterpreter,db,docker,excel,finance,flowtask,git,google,jira,messaging,pdf,pulumi,sandbox,scraping,sitesearch,slack,weather,wikipedia]>=0.1.0; extra == "ai"
+Requires-Dist: ai-parrot-pipelines>=0.1.0; extra == "ai"
+Provides-Extra: dev
+Requires-Dist: pytest>=5.4.0; extra == "dev"
+Requires-Dist: coverage; extra == "dev"
+Requires-Dist: pytest-asyncio==0.23.8; extra == "dev"
+Requires-Dist: pytest-xdist==3.3.1; extra == "dev"
+Requires-Dist: pytest-assume==2.4.3; extra == "dev"
+Requires-Dist: ruff; extra == "dev"
+Requires-Dist: aioresponses>=0.7.8; extra == "dev"
+Dynamic: license-file
+
+# FlowTask #
+
+FlowTask is a plugin-based, component-driven task execution framework for create complex Tasks.
+
+FlowTask runs Tasks defined in JSON, YAML or TOML files, any Task is a combination of Components,
+and every component in the Task run sequentially or depend of others, like a DAG.
+
+Can create a Task combining Commands, Shell scripts and other specific Components (as TableInput: Open a Table using a datasource, DownloadFromIMAP: Download a File from a IMAP Folder, and so on), any Python Callable can be a Component inside a Task, or can extends UserComponent to build your own componets.
+
+Every designed Task can run from CLI, programmatically, via RESTful API (using our aioHTTP-based Handler), called by WebHooks or even dispatched to a external Worker using our built-in Scheduler.
+
+## Quickstart ##
+
+```console
+pip install flowtask
+```
+
+Tasks can organizated into directory structure like this:
+
+tasks /
+    ├── programs /
+      ├── test /
+           ├── tasks /
+
+The main reason of this structure, is maintain organized several tasks by tenant/program, avoiding filling a directory with several task files.
+
+FlowTask support "TaskStorage", a Task Storage is the main repository for tasks, main Task Storage is a directory in any filesystem path (optionally you can syncronize that path using git), but Tasks can be saved onto a Database or a S3 bucket.
+
+## Dependencies ##
+
+ * aiohttp (Asyncio Web Framework and Server) (required by navigator)
+ * AsyncDB
+ * QuerySource
+ * Navigator-api
+ * (Optional) Qworker (for distributing asyncio Tasks on distributed workers).
+
+## Features ##
+
+* Component-based Task execution framework with several components covering several actions (download files, create pandas dataframes from files, mapping dataframe columns to a json-dictionary, etc)
+* Built-in API for execution of Tasks.
+
+### How I run a Task? ###
+
+Can run a Task from CLI:
+```console
+task --program=test --task=example
+```
+
+on CLI, you can pass an ENV (enviroment) to change the environment file on task execution.
+```console
+ENV=dev task --program=test --task=example
+```
+
+or Programmatically:
+```python
+from flowtask import Task
+import asyncio
+
+task = Task(program='test', task='example')
+results = asyncio.run(task.run())
+# we can alternatively, using the execution mode of task object:
+results = asyncio.run(task())
+```
+
+### Requirements ###
+
+* Python >= 3.9
+* asyncio (https://pypi.python.org/pypi/asyncio/)
+* aiohttp >= 3.6.2
+
+### Contribution guidelines ###
+
+Please have a look at the Contribution Guide
+
+* Writing tests
+* Code review
+* Other guidelines
+
+### Who do I talk to? ###
+
+* Repo owner or admin
+* Other community or team contact
+
+### License ###
+
+Navigator is licensed under Apache 2.0 License. See the LICENSE file for more details.

flowtask-5.12.1/flowtask/components/ASPX.py

@@ -0,0 +1,159 @@
+from abc import abstractmethod
+from urllib.parse import urljoin
+
+from bs4 import BeautifulSoup
+import httpx
+from ..interfaces.flow import FlowComponent
+from ..interfaces.http import HTTPService
+from ..exceptions import ComponentError
+
+
+class ASPX(FlowComponent, HTTPService):
+    """
+    ASPX
+
+    Overview
+
+        The ASPX class is designed for interacting with ASPX-based web applications, particularly those requiring authentication.
+        It inherits from the DtComponent and HTTPService classes, providing a structured way to manage sessions, handle state views,
+        and perform HTTP requests with ASPX web forms. This class is useful for automating interactions with ASPX web pages, including login
+        and data retrieval operations.
+
+    :widths: auto
+
+        | _credentials            |   Yes    | A dictionary containing username and password for authentication.                                  |
+        | _views                  |   No     | A dictionary storing state views extracted from ASPX pages.                                        |
+        | _client                 |   Yes    | An instance of `httpx.AsyncClient` used for making asynchronous HTTP requests.                     |
+
+    Return
+
+        The methods in this class facilitate the interaction with ASPX-based web applications, including login handling,
+        session management, and state view management. The class also allows for abstract extension, enabling customization
+        for specific ASPX web forms and interactions.
+
+    |---|---|---|
+    | version | No | version of component |
+
+
+        Example:
+
+        | Name | Required | Summary |
+    |---|---|---|
+    | version | No | version of component |
+
+
+        Example:
+
+        ```yaml
+          ASPX:
+          # attributes here
+        ```
+    """
+    _version = "1.0.0"
+    _credentials: dict = {"username": str, "password": str}
+
+    def _set_views(self, soup: BeautifulSoup):
+        """Extract state VIEWS from hidden inputs inside html
+
+        Args:
+            soup (BeautifulSoup): soup object with the html tree
+
+        Returns:
+            dict[str, str]: state VIEWS
+        """
+        viewstate = soup.find(id="__VIEWSTATE")["value"]
+        soup_eventvalidation = soup.find(id="__EVENTVALIDATION")
+        eventvalidation = soup_eventvalidation["value"] if soup_eventvalidation else ""
+        viewstategenerator = soup.find(id="__VIEWSTATEGENERATOR")["value"]
+
+        self._views = {
+            "__EVENTTARGET": "",
+            "__EVENTARGUMENT": "",
+            "__VIEWSTATE": viewstate,
+            "__VIEWSTATEGENERATOR": viewstategenerator,
+            "__EVENTVALIDATION": eventvalidation,
+        }
+
+    def check_keys_exists(self, *args, **kwargs):
+        """Validate that specified keys are present in the kwargs dict"""
+        for _key in args:
+            if _key not in kwargs:
+                raise ComponentError(f"{_key} must be set!")
+
+    def _get_payload_with_views(self, **kwargs):
+        payload = {
+            **self._views,
+            **kwargs,
+        }
+
+        return payload
+
+    async def start(self, **kwargs):
+        # handle login and set initial views
+        self._proxies = False
+        self._user = None
+        self.auth_type = None
+        self.auth = ""
+        self.download = None
+        self.headers = None
+        self._views: dict = {}
+        self.timeout = 120.0
+        self._client = httpx.AsyncClient(timeout=self.timeout)
+
+        self.processing_credentials()
+
+        login_url = urljoin(self.base_url, "/Login.aspx")
+
+        self.accept = "text/html"
+        await self.aspx_session(login_url)
+
+        payload = {
+            **self._views,
+            self.login_user_payload_key: self.credentials["username"],
+            self.login_password_payload_key: self.credentials["password"],
+            self.login_button_payload_key: self.login_button_payload_value,
+        }
+
+        await self.aspx_session(
+            login_url, method="post", data=payload, follow_redirects=True
+        )
+
+        if ".ASPXAUTH" not in self._client.cookies:
+            raise ComponentError("Login Failed. Please, check credentials!")
+        self._logger.info("Login successful")
+
+        return True
+
+    @abstractmethod
+    async def run(self):
+        """
+        Extend this method: Call super() after your code to make sure
+        the client session is closed.
+        """
+        if hasattr(self, "_client"):
+            await self._client.aclose()
+
+    async def close(self):
+        pass
+
+    async def aspx_session(
+        self,
+        url: str,
+        method: str = "get",
+        data: dict = None,
+        **kwargs: dict,
+    ):
+        result, error = await self.session(
+            url,
+            method=method,
+            data=data,
+            **kwargs,
+        )
+
+        if error:
+            raise ComponentError(error)
+
+        if "text/html" in self.accept:
+            self._set_views(result)
+
+        return result