flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/services/tasks/manager.py

@@ -0,0 +1,323 @@
+"""
+TaskManager.
+
+Task Management: creation, deleting, listing, etc.
+"""
+import re
+import uuid
+import io
+import traceback
+from contextlib import redirect_stdout
+import pandas as pd
+from asyncdb.exceptions import NoDataFound
+from navigator_session import get_session
+from navigator.views import BaseView
+from navigator.responses import JSONResponse
+from ...models import TaskModel
+# Getting Task
+from ...tasks.task import Task
+from ...storages import MemoryTaskStorage
+from ...utils.stats import TaskMonitor
+
+def is_uuid(s):
+    try:
+        uuid.UUID(s)
+        return True
+    except (TypeError, ValueError):
+        return False
+
+
+class TaskManager(BaseView):
+    """TaskManager.
+
+    description: API Endpoint for Task Management (creation of Tasks).
+    """
+    def post_init(self, *args, **kwargs):
+        super().post_init(*args, **kwargs)
+        self.taskstorage = MemoryTaskStorage()
+        self.ansi_escape = re.compile(r'\x1B\[[0-?]*[ -/]*[@-~]')
+
+    model: TaskModel = TaskModel
+    _fields = [
+        "task",
+        "task_id",
+        "task_path",
+        "task_definition",
+        "url",
+        "attributes",
+        "params",
+        "enabled",
+        "last_done_time",
+        "last_exec_time",
+        "program_id",
+        "program_slug",
+        "task_state",
+        "traceback",
+    ]
+
+    async def session(self):
+        session = None
+        try:
+            session = await get_session(self.request)
+        except (ValueError, RuntimeError) as err:
+            return self.critical(
+                reason="Error Decoding Session", request=self.request, exception=err
+            )
+        return session
+
+    async def get(self):
+        """
+        GET Method.
+        ---
+        description: get all tasks, or a task by ID (or get status of execution)
+        tags:
+        - tasks
+        - DataIntegration
+        consumes:
+        - application/json
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was retrieved.
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Task(s) were found
+            "406":
+                description: Query Error
+        """
+        qp = self.query_parameters(self.request)
+        args = self.match_parameters(self.request)
+        try:
+            task_id = args["task_id"]
+        except KeyError:
+            task_id = None
+        if is_uuid(task_id):
+            slug = None
+        else:
+            slug = task_id
+            task_id = None
+        try:
+            program_slug = args["program"]
+        except KeyError:
+            program_slug = None
+        try:
+            meta = args["meta"]
+        except KeyError:
+            meta = None
+        try:
+            if meta == ":meta":
+                # returning JSON schema of Model:
+                response = TaskModel.schema(as_dict=True)
+                return JSONResponse(response, status=200)
+        except (ValueError, TypeError) as ex:
+            self.logger.warning(str(ex))
+        #### getting all tasks:
+        if "fields" in qp:
+            args = {"fields": qp["fields"]}
+        else:
+            args = {"fields": self._fields}
+        try:
+            del qp["fields"]
+        except KeyError:
+            pass
+        try:
+            db = self.request.app["qs_connection"]
+            async with await db.acquire() as conn:
+                TaskModel.Meta.connection = conn
+                if program_slug is not None:
+                    TaskModel.Meta.schema = program_slug
+                if task_id is not None:
+                    result = await TaskModel.get(**{"task_id": task_id})
+                elif slug and program_slug:
+                    result = await TaskModel.get(
+                        **{"task": slug, "program_slug": program_slug}
+                    )
+                elif len(qp) > 0:
+                    result = await TaskModel.filter(**qp)
+                else:
+                    tasks = await TaskModel.all(fields=self._fields)
+                    result = [row.dict() for row in tasks]
+                return self.json_response(result)
+        except NoDataFound as err:
+            headers = {
+                "X-STATUS": "EMPTY",
+                "X-ERROR": str(err),
+                "X-MESSAGE": f"Task {task_id}:{slug} not Found",
+            }
+            return self.no_content(headers=headers)
+        except Exception as err:
+            return self.error(reason=f"Error getting Task: {err}", exception=err)
+
+    async def put(self):
+        """
+        PUT Method.
+        description: Send a Task as Code directly to Task Executor.
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        consumes:
+        - application/merge-patch+json
+        - application/json
+        responses:
+            "200":
+                description: Task Executed
+            "204":
+                description: success execution but no content on return (resource was deleted)
+            "400":
+                description: Invalid resource according data schema
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+        try:
+            body_task = await self.request.text()
+            if not body_task:
+                return self.no_content()
+        except Exception as e:
+            return self.error(
+                response={
+                    "message": f"Error reading body: {e}",
+                    "error": e
+                },
+                status=400
+            )
+        # Then, try to validate With Task
+        # generate an uuid as name:
+        task_name = uuid.uuid4()
+        task = Task(
+            task=task_name
+        )
+        async with task as t:
+            # Assign the Memory Storage
+            t.taskstore = self.taskstorage
+            stdout = io.StringIO()
+            stacktrace = None
+            error = None
+            result = None
+            stats = None
+            captured_stdout = None
+            if await t.start(payload=body_task):
+                try:
+                    with redirect_stdout(stdout):
+                        # can we run the task
+                        result = await t.run()
+                except Exception as e:
+                    result = None
+                    stacktrace = traceback.format_exc()
+                    error = str(e)
+                captured_stdout = stdout.getvalue()
+                stats = t.get_stats()
+            if isinstance(result, pd.DataFrame):
+                resultset = []
+                resultset.append(
+                    {"result": str(result)}
+                )
+                for column, t in result.dtypes.items():
+                    resultset.append(
+                        f"{column}->{t}->{result[column].iloc[0]}"
+                    )
+                result = resultset
+            result = {
+                "result": f"{result!s}",
+                "error": error,
+                "stacktrace": stacktrace,
+                "stdout": self.ansi_escape.sub('', captured_stdout),
+                "stat": stats
+            }
+            return JSONResponse(
+                result
+            )
+        return self.error(
+            response={
+                "message": "Error reading Task body"
+            },
+            status=400
+        )
+
+    async def delete(self):
+        """
+        DELETE Method.
+        description: remove resource.
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+
+    async def patch(self):
+        """
+        PATCH Method.
+        description: updating partially info about a Task
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "304":
+                description: Task not modified, its currently the actual version of Task
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+
+    async def post(self):
+        """
+        PATCH Method.
+        description: updating or creating tasks
+        tags:
+        - tasks
+        - DataIntegration
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was updated.
+            "201":
+                description: New Task was inserted
+            "304":
+                description: Task not modified, its currently the actual version of Task
+            "403":
+                description: Forbidden Call
+            "404":
+                description: No Data was found
+            "406":
+                description: Query Error
+            "409":
+                description: Conflict, a constraint was violated
+        """
+
+    async def head(self):
+        pass

flowtask/services/tasks/service.py

@@ -0,0 +1,275 @@
+"""
+TaskService.
+
+Service for running tasks.
+"""
+from typing import Any
+import uuid
+import random
+import logging
+from aiohttp import web
+from aiohttp.web import StreamResponse
+import pandas as pd
+from navigator.views import BaseView
+from navigator.conf import AUTH_SESSION_OBJECT
+from navigator_session import get_session
+from ...exceptions import (
+    TaskException,
+    TaskDefinition,
+    TaskParseError,
+    TaskNotFound,
+    DataNotFound,
+    FileNotFound,
+)
+from ...tasks.task import Task
+
+
+class TaskService(BaseView):
+    """TaskService.
+
+    Task As a Service: launch data-based tasks and returning the resultset in useful formats.
+    Args:
+        BaseView (_type_): _description_
+    """
+
+    def get_user(self, session, idx: str = "user_id") -> int:
+        if not session:
+            self.error(response={"error": "Unauthorized"}, status=403)
+        try:
+            if AUTH_SESSION_OBJECT in session:
+                return session[AUTH_SESSION_OBJECT][idx]
+            else:
+                return session[idx]
+        except KeyError:
+            self.error(
+                response={
+                    "error": "Unauthorized",
+                    "message": "Hint: maybe you need to pass an Authorization token.",
+                },
+                status=403,
+            )
+
+    async def check_task(self):
+        # get the URL parameters
+        params = self.get_args()
+        try:
+            program = params["program"]
+            del params["program"]
+        except (ValueError, KeyError) as ex:
+            # we need a program
+            raise TaskDefinition("Resource Error: We need a Program Name") from ex
+        try:
+            task_id = params["task_id"]
+            del params["task_id"]
+        except (ValueError, KeyError) as ex:
+            # we need a Task ID
+            raise TaskDefinition("Resource Error: We need a Task Name") from ex
+        return task_id, program
+
+    async def get_task(self, task: str, program: str, **kwargs) -> Task:
+        # Task ID:
+        task_id = uuid.uuid1(node=random.getrandbits(48) | 0x010000000000)
+        try:
+            usr = await get_session(self.request)
+            user = self.get_user(usr)
+        except (TypeError, ValueError):
+            user = None
+
+        try:
+            task = Task(
+                task_id=task_id,
+                task=task,
+                program=program,
+                loop=self._loop,
+                userid=user,
+                **kwargs,
+            )
+            # disable statistics
+            task.enable_stat = False
+        except Exception as err:
+            raise TaskException(err) from err
+        # then, try to "start" the task:
+        logging.debug(f"::::: DI: Running Task: {task!r}")
+        return task
+
+    async def launch_task(self, task: Task) -> Any:
+        try:
+            await task.start()
+        except (TaskParseError, TaskDefinition, TaskNotFound) as ex:
+            raise TaskDefinition(str(ex)) from ex
+        except Exception as err:
+            logging.error(err)
+            raise TaskException(f"Unhandled Task Error: {err}") from err
+        # run task:
+        try:
+            return await task.run()
+        except Exception as err:
+            logging.exception(err)
+            raise
+        finally:
+            await task.close()
+
+    async def get(self):
+        """
+        GET Method.
+        description: Executing a Task and returning the result
+        tags:
+        - tasks
+        - DataIntegration
+        consumes:
+        - application/json
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was executed.
+            "202":
+                description: Task was accepted to queue (no return)
+            "204":
+                description: No data was found
+            "400":
+                description: Task Failed to execute
+            "403":
+                description: Forbidden Call
+            "404":
+                description: no Task was found.
+            "406":
+                description: Query Error
+            "409":
+                description: Task Conflict, a constraint was violated
+        """
+        # get query parameters
+        query = self.query_parameters(request=self.request)
+        # Default launch a task on Queue
+        # no_worker = True
+        # TODO: check for user permissions
+        try:
+            task, program = await self.check_task()
+        except TaskDefinition as ex:
+            headers = {"X-STATUS": "Error", "X-MESSAGE": ex}
+            msg = {"state": "Failed", "message": f"Error: {ex}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+        try:
+            task = await self.get_task(task, program, **query)
+        except TaskException as ex:
+            headers = {"X-STATUS": "Error", "X-MESSAGE": ex}
+            msg = {"state": "Failed", "message": f"Error: {ex}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+        try:
+            result = await self.launch_task(task)
+            return await self.task_output(result)
+        except Exception as err:  # pylint: disable=W0703
+            headers = {"X-STATUS": "Error", "X-MESSAGE": err}
+            msg = {"state": "Failed", "message": f"Error: {err}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+
+    async def post(self):
+        """
+        GET Method.
+        description: Executing a Task and returning the result
+        tags:
+        - tasks
+        - DataIntegration
+        consumes:
+        - application/json
+        produces:
+        - application/json
+        responses:
+            "200":
+                description: Existing Task was executed.
+            "202":
+                description: Task was accepted to queue (no return)
+            "204":
+                description: No data was found
+            "400":
+                description: Task Failed to execute
+            "403":
+                description: Forbidden Call
+            "404":
+                description: no Task was found.
+            "406":
+                description: Query Error
+            "409":
+                description: Task Conflict, a constraint was violated
+        """
+        # get post Data
+        data = await self.post_data()
+        # get query parameters
+        query = self.query_parameters(request=self.request)
+        # Default launch a task on Queue
+        # TODO: check for user permissions
+        try:
+            params = {**query, **data}
+        except (KeyError, TypeError):
+            params = data
+        try:
+            task, program = await self.check_task()
+        except TaskDefinition as ex:
+            headers = {"X-STATUS": "Error", "X-MESSAGE": ex}
+            msg = {"state": "Failed", "message": f"Error: {ex}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+        try:
+            task = await self.get_task(task, program, **params)
+        except TaskException as ex:
+            headers = {"X-STATUS": "Error", "X-MESSAGE": ex}
+            msg = {"state": "Failed", "message": f"Error: {ex}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+        try:
+            result = await self.launch_task(task)
+            return await self.task_output(result)
+        except (DataNotFound, FileNotFound) as ex:
+            headers = {"X-STATUS": "Not Found", "X-MESSAGE": f"Data not Found: {ex}"}
+            return self.no_content(headers=headers)
+        except Exception as err:  # pylint: disable=W0703
+            headers = {"X-STATUS": "Error", "X-MESSAGE": err}
+            msg = {"state": "Failed", "message": f"Error: {err}", "status": 400}
+            return self.error(response=msg, headers=headers, status=400)
+
+    async def task_output(self, resultset: Any) -> web.Response:
+        ## TODO: making output pluggable like QuerySource
+        response = None
+        buffer = None
+        if isinstance(resultset, pd.DataFrame):
+            buffer = resultset.to_json(orient="records")
+            response = self.get_response()
+        return await self.stream_response(response, bytes(buffer, "utf-8"))
+
+    def get_response(self) -> web.Response:
+        response = StreamResponse(
+            status=200,
+            reason="OK",
+            headers={
+                "Pragma": "public",  # required,
+                "Expires": "0",
+                "Connection": "keep-alive",
+                "Cache-Control": "must-revalidate, post-check=0, pre-check=0",
+                "Content-Type": "application/json",
+                "X-APPLICATION": "QuerySource",
+            },
+        )
+        response.enable_compression(force=web.ContentCoding.gzip)
+        return response
+
+    async def stream_response(
+        self, response: web.StreamResponse, data: Any
+    ) -> web.StreamResponse:
+        content_length = len(data)
+        response.content_length = content_length
+        chunk_size = 16384
+        response.headers["Content-Range"] = f"bytes 0-{chunk_size}/{content_length}"
+        try:
+            i = 0
+            await response.prepare(self.request)
+            while True:
+                chunk = data[i: i + chunk_size]
+                i += chunk_size
+                if not chunk:
+                    break
+                await response.write(chunk)
+                await response.drain()
+            await response.write_eof()
+            return response
+        except Exception as ex:  # pylint: disable=W0703
+            return self.error(
+                message="Error Starting Stream Transmision", exception=ex, status=500
+            )