flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/services/tasks/task_manager.py

@@ -0,0 +1,376 @@
+"""
+TaskManager.
+
+Work with managing tasks, creation, deleting, listing, etc.
+"""
+import asyncio
+import json
+import datetime
+import uuid
+from functools import partial
+from uuid import UUID
+import logging
+import ast
+from aiohttp import web
+from navigator.views import DataView
+from asyncdb.utils.encoders import BaseEncoder, DefaultEncoder
+from querysource.utils.functions import format_date
+from .tasks import task_state, execute_task, launch_task
+from ...exceptions import (
+    NotSupported,
+    ComponentError,
+    TaskNotFound,
+    TaskDefinition,
+    TaskFailed,
+    FileError,
+    FileNotFound,
+)
+
+
+def is_boolean(value: str) -> bool:
+    if isinstance(value, bool):
+        return value
+    try:
+        return ast.literal_eval(value)
+    except ValueError:
+        return False
+
+
+class TaskManager(DataView):
+    async def get(self):
+        """
+        GET Method.
+        ---
+        description: get all tasks, or a task by ID (or get status of execution)
+        tags:
+        - tasks
+        - DataIntegration
+        consumes:
+        - application/json
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was retrieved.
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Task(s) were found
+            "406":
+                description: Query Error
+        """
+        try:
+            sql = "SELECT * FROM {program}.tasks"
+            await self.connect(self.request)
+            params = self.get_args()
+            print(params, sql)
+            # try:
+            #     session = self.request['session']
+            # except KeyError:
+            #     return self.error(request=self.request, response="Unknown User", state=403)
+            try:
+                program = params["program"]
+            except KeyError:
+                program = "troc"
+            try:
+                task = params["task_id"]
+            except KeyError:
+                task = None
+            sql = sql.format(program=program)
+            print(sql)
+            if task is not None:
+                sql = "{sql} WHERE task = '{id}' AND enabled = true".format(
+                    sql=sql, id=task
+                )
+            print("SQL IS ", sql)
+            try:
+                result = await self.query(sql)
+            except Exception as e:
+                msg = {"error": str(e), "exception": str(e.__class__)}
+                return self.error(
+                    request=self.request, response=msg, exception=e, state=400
+                )
+            if result:
+                data = []
+                for row in result:
+                    data.append(dict(row))
+                for elements in data:
+                    for key, value in elements.items():
+                        elements[key] = (
+                            str(value)
+                            if isinstance(
+                                value, (UUID, datetime.time, datetime.datetime)
+                            )
+                            else value
+                        )
+                headers = {"X-STATUS": "OK", "X-MESSAGE": "Tasks List"}
+                return self.json_response(response=data, headers=headers)
+            else:
+                if self._lasterr:
+                    msg = {"error": str(self._lasterr)}
+                    return self.error(
+                        request=self.request,
+                        response=msg,
+                        exception=self._lasterr,
+                        state=400,
+                    )
+                headers = {"X-STATUS": "EMPTY", "X-MESSAGE": "Data not Found"}
+                return self.no_content(headers=headers)
+        except Exception as e:
+            print(e)
+            return self.critical(self.request, e)
+        finally:
+            await self.close()
+
+    async def post(self):
+        """
+        POST Method.
+        description: inserting or updating a Task or executing a Task
+        tags:
+        - tasks
+        - DataIntegration
+        consumes:
+        - application/json
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated or executed.
+            "201":
+                description: New Task was inserted
+            "202":
+                description: Task was accepted to run
+            "400":
+                description: Task Failed to execute
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+        # using LOCATION header to return the URL of the API
+        await self.connect(self.request)
+        # get the URL parameters
+        params = self.get_args()
+        # get post Data
+        data = await self.post_data()
+        # long
+        longrunner = False
+        try:
+            program = params["program"]
+            del params["program"]
+        except KeyError:
+            # we need a program
+            try:
+                program = data["program"]
+            except KeyError:
+                headers = {
+                    "X-STATUS": "Error",
+                    "X-MESSAGE": "Resource Error: We need a Program Name",
+                }
+                msg = {
+                    "state": "Failed",
+                    "message": "Error: we need a program name",
+                    "status": 400,
+                }
+                return self.error(
+                    request=self.request, response=msg, headers=headers, state=400
+                )
+        try:
+            task_id = params["task_id"]
+            del params["task_id"]
+        except KeyError:
+            # we need a Task ID
+            try:
+                task_id = data["task_id"]
+            except KeyError:
+                headers = {
+                    "X-STATUS": "Error",
+                    "X-MESSAGE": "Resource Error: We need a Task Name",
+                }
+                msg = {
+                    "state": "Failed",
+                    "message": "Error: we need a Task ID",
+                    "status": 400,
+                }
+                return self.error(
+                    request=self.request, response=msg, headers=headers, state=400
+                )
+        try:
+            try:
+                longrunner = is_boolean(data["long_running"])
+                del data["long_running"]
+            except KeyError:
+                try:
+                    longrunner = is_boolean(params["long_running"])
+                    del params["long_running"]
+                except KeyError:
+                    longrunner = False
+            try:
+                no_worker = is_boolean(data["no_worker"])
+                del data["no_worker"]
+            except KeyError:
+                no_worker = False
+            logging.debug(f"Long Runner: {longrunner}, No Worker: {no_worker}")
+            # cannot update or insert a Task, we need to Run that task
+            status = None
+            result = {}
+            try:
+                # TODO: passing arguments via URL to the task
+                args = {}
+                if isinstance(params, dict):
+                    args = params
+                if data:
+                    args = {**args, **data}
+                task_uuid = uuid.uuid4()
+                status, action, result = await launch_task(
+                    program_slug=program,
+                    task_id=task_id,
+                    loop=self._loop,
+                    task_uuid=task_uuid,
+                    queued=longrunner,
+                    no_worker=no_worker,
+                    **args,
+                )
+                result = {"task": f"{program}.{task_id}", "task_execution": task_uuid}
+                if action == "Executed":
+                    state = 200
+                else:
+                    state = 202
+                response = {
+                    "state": state,
+                    "message": "Task {}.{} was {}".format(program, task_id, action),
+                    **result,
+                }
+                headers = {
+                    "X-STATUS": "Task OK",
+                    "X-MESSAGE": "Execution of Task {}.{} on {}".format(
+                        program, task_id, action
+                    ),
+                }
+                return self.json_response(
+                    response=response,
+                    headers=headers,
+                    state=response["state"],
+                    cls=BaseEncoder,
+                )
+            except TaskNotFound as err:
+                error = "Error: on Task {}.{}".format(program, task_id)
+                msg = {"message": error}
+                return self.error(
+                    request=self.request, response=msg, exception=err, state=401
+                )
+            except TaskFailed as err:
+                print(err)
+                error = "Error: Task {} Failed".format(task_id)
+                msg = {"message": error}
+                return self.error(
+                    request=self.request, response=msg, exception=err, state=400
+                )
+            except (FileNotFound, FileError) as err:
+                print(err)
+                error = "Error on Task {} File Not Found: {}".format(task_id, str(err))
+                msg = {"message": error}
+                return self.error(
+                    request=self.request, response=msg, exception=err, state=404
+                )
+            except Exception as err:
+                print(err)
+                return self.critical(request=self.request, exception=err, state=500)
+            headers = {"X-STATUS": "EMPTY", "X-MESSAGE": "Data not Found"}
+            return self.no_content(headers=headers)
+        except Exception as e:
+            print(f"Generic Error on POST Method: {e}")
+            return self.critical(self.request, e)
+        finally:
+            await self.close()
+
+    async def put(self):
+        """
+        PUT Method.
+        description: inserting or updating a Task
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        consumes:
+        - application/merge-patch+json
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "204":
+                description: success execution but no content on return (resource was deleted)
+            "400":
+                description: Invalid resource according data schema
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+        pass
+
+    async def delete(self):
+        """
+        DELETE Method.
+        description: remove resource.
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+        pass
+
+    async def patch(self):
+        """
+        PATCH Method.
+        description: updating partially info about a Task
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "304":
+                description: Task not modified, its currently the actual version of Task
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+        pass
+
+    async def head(self):
+        pass

flowtask/services/tasks/tasks.py

@@ -0,0 +1,155 @@
+from typing import Union
+import asyncio
+from concurrent.futures import ThreadPoolExecutor
+import jsonpickle
+import pandas as pd
+from navconfig import config as ENV
+from navconfig.logging import logging
+
+# Queue Worker Client:
+from qw.client import QClient
+from qw.wrappers import TaskWrapper
+from ...conf import DEBUG, WORKER_LIST, WORKER_HIGH_LIST
+from ...exceptions import (
+    NotSupported,
+    TaskNotFound,
+    TaskFailed,
+    FileError,
+    FileNotFound,
+    DataNotFound,
+)
+from ...tasks.task import Task
+
+
+if WORKER_LIST:
+    QW = QClient(worker_list=WORKER_LIST)
+    QW_high = QClient(worker_list=WORKER_HIGH_LIST)
+else:
+    QW = QClient()  # auto-discovering of workers
+    QW_high = QW
+
+
+async def launch_task(
+    program_slug: str,
+    task_id: str,
+    loop: asyncio.AbstractEventLoop = None,
+    task_uuid: str = None,
+    queued: bool = False,
+    no_worker: bool = False,
+    priority: str = "low",
+    userid: Union[int, str] = None,
+    **kwargs,
+):
+    """launch_task.
+    Runs (or queued) a Task from Task Monitor.
+    """
+    state = None
+    result = {}
+
+    if no_worker is True:
+        # Running task in Local
+        print(f"RUNNING TASK {program_slug}.{task_id}")
+        task = Task(
+            task=task_id,
+            program=program_slug,
+            loop=loop,
+            ignore_results=False,
+            ENV=ENV,
+            debug=DEBUG,
+            userid=userid,
+            **kwargs,
+        )
+        try:
+            state = await task.start()
+            if not state:
+                logging.warning(
+                    f"Task {program_slug}.{task_id} return False on Start Time."
+                )
+        except Exception as err:
+            logging.error(err)
+            raise
+        try:
+            state = await task.run()
+            try:
+                result["stats"] = task.stats.stats
+            except Exception as err:
+                result["stats"] = None
+                result["error"] = err
+            ### gettting the result of Task execution
+            if isinstance(state, pd.DataFrame):
+                numrows = len(state.index)
+                columns = list(state.columns)
+                num_cols = len(columns)
+                state = {
+                    "type": "Dataframe",
+                    "numrows": numrows,
+                    "columns": columns,
+                    "num_cols": num_cols,
+                }
+            else:
+                state = f"{state!r}"
+            result["result"] = state
+            return (state, "Executed", result)
+        except DataNotFound as err:
+            raise
+        except NotSupported as err:
+            raise
+        except TaskNotFound as err:
+            raise TaskNotFound(f"Task: {task_id}: {err}") from err
+        except TaskFailed as err:
+            raise TaskFailed(f"Task {task_id} failed: {err}") from err
+        except FileNotFound:
+            raise
+        except FileError as err:
+            raise FileError(f"Task {task_id}, File Not Found error: {err}") from err
+        except Exception as err:
+            raise TaskFailed(f"Error: Task {task_id} failed: {err}") from err
+        finally:
+            await task.close()
+    else:
+        if queued:
+            action = "Queued"
+        else:
+            action = "Dispatched"
+    # Task:
+    task = TaskWrapper(
+        program=program_slug, task=task_id, ignore_results=True, userid=userid, **kwargs
+    )
+    ### TODO: Add Publish into Broker
+    if action == "Queued":
+        if priority == "high":
+            result = await asyncio.wait_for(QW_high.queue(task), timeout=10)
+        else:
+            result = await asyncio.wait_for(QW.queue(task), timeout=10)
+        try:
+            result["message"] = result["message"].decode("utf-8")
+        except (TypeError, KeyError):
+            result["message"] = None
+        print(f"Task Queued: {result!s}")
+        return (task_uuid, action, result)
+    else:
+        try:
+            if priority == "high":
+                result = await QW_high.run(task)
+            else:
+                result = await QW.run(task)
+            if isinstance(result, str):
+                # check if can we unserialize the jsonpickle
+                try:
+                    result = jsonpickle.decode(result)
+                except (TypeError, KeyError, ValueError, AttributeError) as ex:
+                    result = {"message": result, "error": ex}
+            logging.debug(f"Executed Task: {result!s}")
+            try:
+                result["message"] = result["message"].decode("utf-8")
+            except (TypeError, KeyError):
+                result["message"] = None
+            except AttributeError:
+                pass
+            return (task_uuid, action, result)
+        except (NotSupported, TaskNotFound, TaskFailed, FileNotFound, FileError) as ex:
+            logging.error(ex)
+            raise
+        except Exception as exc:
+            logging.error(exc)
+            raise TaskFailed(f"Error: Task {task_id} failed: {exc}") from exc

flowtask/storages/__init__.py

@@ -0,0 +1,16 @@
+"""
+Storages.
+
+Storages covering the ways to store the data processed by the tasks and the task/hooks definitions.
+
+FileStore: how Flowtask can store files in the filesystem.
+TaskStorage: how Flowtask can store the task definitions (yaml, json files).
+"""
+from .files import FileStore
+from .tasks import (
+    FileTaskStorage,
+    RowTaskStorage,
+    MemoryTaskStorage,
+    DatabaseTaskStorage,
+    GitTaskStorage
+)

flowtask/storages/exceptions.py

@@ -0,0 +1,12 @@
+"""
+Storage Exceptions.
+
+This module provides the exceptions for the Storages.
+"""
+
+
+class StoreError(Exception):
+    """
+    Base Exception Triggered for all Storages.
+    """
+    pass

flowtask/storages/files/__init__.py

@@ -0,0 +1,8 @@
+"""File Store.
+
+Used to save files downloaded/created by tasks to a File Store.
+"""
+from .filesystem import FileStore
+
+
+__all__ = ("FileStore",)

flowtask/storages/files/abstract.py

@@ -0,0 +1,29 @@
+from typing import Any
+from abc import ABC, abstractmethod
+from navconfig import DEBUG
+from navconfig.logging import logging
+from ..exceptions import StoreError
+
+
+class AbstractStore(ABC):
+    """Abstract class for File Store."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        self._name = self.__class__.__name__
+        self._program: Any = kwargs.pop("program", None)
+        self.logger = logging.getLogger(f"FlowTask.Files.{self._name}")
+        if DEBUG is True:
+            self.logger.notice(f":: Starting Store {self._name}")
+        self.kwargs = kwargs
+        self.args = args
+
+    def set_program(self, program: str) -> None:
+        self._program = program
+
+    @abstractmethod
+    def default_directory(self, directory: str):
+        pass
+
+    @abstractmethod
+    def get_directory(self, directory: str):
+        pass

flowtask/storages/files/filesystem.py

@@ -0,0 +1,66 @@
+from typing import Union, Optional
+from pathlib import Path, PurePath
+from .abstract import AbstractStore
+from ..exceptions import StoreError
+from ...exceptions import FileNotFound
+
+
+class FileStore(AbstractStore):
+    """
+    FileStore: Saving files in local filesystem.
+    """
+
+    def __init__(
+        self, path: Union[str, PurePath], prefix: str, *args, **kwargs
+    ) -> None:
+        if isinstance(path, str):
+            path = Path(path)
+        if path.exists() and path.is_dir():
+            self.path: PurePath = path
+        else:
+            raise StoreError(
+                f"FileStore: directory doesn't exists: {path}"
+            )
+        self._prefix: str = prefix
+        super().__init__(*args, **kwargs)
+
+    def default_directory(self, directory: str):
+        return self.path.joinpath(self._program, self._prefix, directory)
+
+    def get_directory(self, directory: str, program: Optional[str] = None):
+        if not program:
+            program = self._program
+        if isinstance(directory, PurePath):
+            _dir = directory
+        if ":" in directory:
+            # relative directory to Storage:
+            _dir = self.path.joinpath(
+                program, self._prefix, directory.rsplit(":", 1)[1]
+            )
+        elif ".." in directory:
+            # relative directory to base directory:
+            _dir = self.path.joinpath(directory)
+        else:
+            _dir = Path(directory)
+            # If not relative, try to use as is:
+            if not _dir.is_absolute() or not _dir.exists():
+                # Try to use instead base directory:
+                _dir = self.path.joinpath(
+                    program,
+                    self._prefix,
+                    directory
+                )
+            elif not _dir.exists():
+                _dir = Path(directory).resolve()
+        if _dir.exists() and _dir.is_dir():
+            self.logger.debug(
+                f"Directory: {_dir}"
+            )
+            return _dir
+        else:
+            self.logger.error(
+                f"Path doesn't exists: {_dir}"
+            )
+            raise FileNotFound(
+                f"Path doesn't exists: {_dir}"
+            )

flowtask/storages/tasks/__init__.py

@@ -0,0 +1,19 @@
+"""
+Task Storage.
+
+Saving Tasks on different Storages (Filesystem, S3 buckets, databases, etc)
+"""
+
+from .filesystem import FileTaskStorage
+from .database import DatabaseTaskStorage
+from .row import RowTaskStorage
+from .github import GitTaskStorage
+from .memory import MemoryTaskStorage
+
+__all__ = (
+    "FileTaskStorage",
+    "DatabaseTaskStorage",
+    "RowTaskStorage",
+    "GitTaskStorage",
+    "MemoryTaskStorage",
+)