flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/storages/tasks/abstract.py

@@ -0,0 +1,26 @@
+from typing import Any
+from pathlib import PurePath
+from abc import ABC, abstractmethod
+from navconfig.logging import logging
+
+
+class AbstractTaskStorage(ABC):
+    """
+    Abstract Base class for all Task Storages.
+    """
+    _name_: str = "TaskStorage"
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.path: PurePath = None
+        self.logger = logging.getLogger(
+            f"FlowTask.Storage.Task.{self._name_}"
+        )
+
+    def get_path(self) -> PurePath:
+        return self.path
+
+    @abstractmethod
+    async def open_task(self, task: str, program: str = None, **kwargs) -> Any:
+        """open_task.
+        Open A Task from Task Storage, support JSON, YAML and TOML formats.
+        """

flowtask/storages/tasks/database.py

@@ -0,0 +1,33 @@
+from typing import Any
+import asyncio
+from asyncdb import AsyncDB
+from .abstract import AbstractTaskStorage
+
+
+class DatabaseTaskStorage(AbstractTaskStorage):
+    """Saving Tasks on an postgreSQL Database."""
+    _name_: str = "Database"
+
+    def __init__(
+        self,
+        driver: str = "pg",
+        dsn: str = None,
+        credentials: dict = None,
+        *args,
+        **kwargs,
+    ) -> None:
+        super().__init__(*args, **kwargs)
+        self.loop = asyncio.get_event_loop()
+        asyncio.set_event_loop(self.loop)
+        self._connection = AsyncDB(
+            driver, dsn=dsn, params=credentials, loop=self.loop, **kwargs
+        )
+
+    async def open_task(
+        self,
+        task: str = None,
+        program: str = None,
+        **kwargs,
+    ) -> Any:
+        ## TODO: getting tasks from a external Task Storage (like a database).
+        return self._task

flowtask/storages/tasks/filesystem.py

@@ -0,0 +1,108 @@
+from typing import Union
+from pathlib import Path, PurePath
+from ...exceptions import (
+    FlowTaskError,
+    TaskNotFound,
+    TaskParseError,
+    TaskDefinition,
+    FileNotFound,
+)
+from ...parsers import JSONParser, TOMLParser, YAMLParser
+from .abstract import AbstractTaskStorage
+
+
+class FileTaskStorage(AbstractTaskStorage):
+    """Saving Tasks on the Filesystem."""
+    _name_: str = "Filesystem"
+
+    def __init__(self, path: PurePath, *args, **kwargs):
+        super(FileTaskStorage, self).__init__(*args, **kwargs)
+        if not path:
+            ## Default Task Path
+            raise FlowTaskError(
+                "Required Task Path for Filesystem Task Storage"
+            )
+        else:
+            self.path = path
+            if isinstance(path, str):
+                self.path = Path(path)
+
+    async def open_task(
+        self, task: str = None, program: str = None, **kwargs
+    ) -> Union[dict, str]:
+        """open_task.
+        Open A Task from FileSystem, support json, yaml and toml formats.
+        """
+        if not program:
+            program = "navigator"
+        taskpath = self.path.joinpath(program, "tasks")
+        self.logger.notice(f"Program Task Path: {taskpath}")
+        if not taskpath.exists():
+            raise TaskNotFound(
+                f"FlowTask: Task Path not found: {taskpath}"
+            )
+        ext = None
+        for f in (
+            "json",
+            "yaml",
+            "toml",
+        ):
+            filename = taskpath.joinpath(f"{task}.{f}")
+            if filename.exists():
+                self.logger.info(f"Task File: {filename}")
+                ext = f
+                break
+        else:
+            if ext is None:
+                raise TaskNotFound(
+                    f"FlowTask: Task {program}.{task} Not Found on file > {filename}"
+                )
+        try:
+            if f == "json":
+                parse = JSONParser(file=str(filename))
+            elif f == "yaml":
+                parse = YAMLParser(file=str(filename))
+            elif f == "toml":
+                parse = TOMLParser(file=str(filename))
+            else:
+                raise FlowTaskError(
+                    f"Invalid File format for Task: {filename.name}"
+                )
+            return await parse.run()
+        except TaskParseError as err:
+            raise TaskParseError(f"Task Parse Error for {filename}: {err}") from err
+        except Exception as err:
+            raise TaskDefinition(
+                f"DI: Error Parsing {f} Task in {task} \
+                    for filename: {filename.name}: {err}"
+            ) from err
+
+    async def open_hook(self, filename: Union[str, PurePath]) -> Union[dict, str]:
+        """open_hook.
+        Open A Hook from FileSystem, support json, yaml and toml formats.
+        """
+        if filename.exists():
+            f = filename.suffix
+            try:
+                if f == ".json":
+                    parse = JSONParser(str(filename))
+                elif f == ".yaml":
+                    parse = YAMLParser(str(filename))
+                elif f == ".toml":
+                    parse = TOMLParser(str(filename))
+                else:
+                    raise FlowTaskError(
+                        f"Invalid File format for Hook: {filename.name}"
+                    )
+                self.logger.notice(f"Hook File: {filename}")
+                return await parse.run()
+            except TaskParseError as err:
+                raise TaskParseError(
+                    f"Hook Parsing Error for {filename}: {err}"
+                ) from err
+            except Exception as err:
+                raise FlowTaskError(
+                    f"Error loading Hook {filename.name}: {err}"
+                ) from err
+        else:
+            raise FileNotFound(f"Hook Not Found on file > {filename}")

flowtask/storages/tasks/github.py

@@ -0,0 +1,119 @@
+import os
+from pathlib import PurePath
+import tempfile
+from git import Repo, GitCommandError
+from navconfig.logging import logging
+from ...exceptions import FlowTaskError, ConfigError
+from .filesystem import FileTaskStorage
+
+logging.getLogger("git").setLevel(logging.WARNING)
+
+
+class GitTaskStorage(FileTaskStorage):
+    """Getting Tasks on Filesystem with Github Support."""
+    _name_: str = "Git"
+    use_ssh: bool = True
+
+    def __init__(
+        self,
+        path: PurePath,
+        git_url: str,
+        *args,
+        git_user: str = None,
+        password: str = None,
+        git_private_key: str = None,
+        **kwargs,
+    ):
+        super(GitTaskStorage, self).__init__(path, *args, **kwargs)
+
+        if git_url:
+            self.git_url = git_url
+            if git_url.startswith("http"):
+                # using HTTP instead SSH for cloning
+                self.use_ssh: bool = False
+            self.git_private_key = git_private_key
+            self.git_user = git_user
+            self.git_password = password
+            if not password:
+                self.git_password = git_private_key
+            self.refresh_repository()
+
+    def clone_repository(self):
+        url = self.git_url
+        try:
+            if self.use_ssh is False:
+                self.git_url = self.git_url.replace(
+                    "https://", f"https://{self.git_user}:{self.git_password}@"
+                )
+                Repo.clone_from(self.git_url, self.path)
+            else:
+                if self.git_private_key:
+                    with tempfile.NamedTemporaryFile(delete=False) as tmp_key:
+                        tmp_key.write(self.git_private_key.encode())
+                        tmp_key.flush()  # Ensure the key is written to disk
+                        env = {"GIT_SSH_COMMAND": f"ssh -i {tmp_key.name}"}
+                        try:
+                            Repo.clone_from(self.git_url, self.path, env=env)
+                        finally:
+                            os.unlink(tmp_key.name)
+                else:
+                    Repo.clone_from(self.git_url, self.path)
+        except Exception as exc:
+            raise ConfigError(
+                f"Github Storage: Unable to Clone Repository: {exc}"
+            ) from exc
+        self.logger.info(f":: Cloned repository: {url}")
+
+    def refresh_repository(self):
+        try:
+            if not self.path.exists() or not any(self.path.iterdir()):
+                # If the directory is empty or doesn't exist, clone the repository
+                self.clone_repository()
+            else:
+                # If the directory exists and is not empty, pull the latest changes
+                try:
+                    repo = Repo(self.path)
+                except Exception as exc:
+                    raise ConfigError(
+                        f"Github Storage: Unable to sync with Repository: {exc}"
+                    ) from exc
+                env = {}
+                if self.use_ssh is True:
+                    env["GIT_SSH_COMMAND"] = f"ssh -i {self.git_private_key}"
+                    with repo.git.custom_environment(**env):
+                        repo.git.pull()
+                else:
+                    repo.git.fetch()
+                    if len(repo.remote().refs) > 0:
+                        upstream_branch = None
+                        # Determine the correct upstream branch
+                        for remote_ref in repo.remote().refs:
+                            if (
+                                remote_ref.remote_head == "master"
+                                or remote_ref.remote_head == "main"
+                            ):
+                                upstream_branch = remote_ref.remote_head
+                                break
+                        else:
+                            if upstream_branch:
+                                # Set the correct upstream branch
+                                repo.git.branch(
+                                    "--set-upstream-to", f"origin/{upstream_branch}"
+                                )
+                            else:
+                                self.logger.warning(
+                                    "Unable to determine the correct upstream branch"
+                                )
+                        # Pull the latest changes
+                        repo.git.pull()
+                    else:
+                        self.logger.warning(
+                            f"Empty Upstream repository: {repo.remote()}"
+                        )
+                self.logger.info(
+                    f"Pulled the latest changes from repository: {self.git_url}"
+                )
+        except GitCommandError as err:
+            raise FlowTaskError(
+                f"Error interacting with Git repository: {err}"
+            ) from err

flowtask/storages/tasks/memory.py

@@ -0,0 +1,45 @@
+from typing import Union
+from ...parsers import JSONParser, TOMLParser, YAMLParser
+from ...exceptions import TaskParseError
+from .abstract import AbstractTaskStorage
+
+
+class MemoryTaskStorage(AbstractTaskStorage):
+    """Executing Task from Memory."""
+    _name_: str = "Memory"
+
+    def __init__(self, *args, **kwargs):
+        super(MemoryTaskStorage, self).__init__(*args, **kwargs)
+        self.content = None
+        self.filename = ":memory:"
+
+    async def open_task(
+        self,
+        payload: str = None,
+        task: str = None,
+        program: str = None,
+        **kwargs
+    ) -> Union[dict, str]:
+        """open_task.
+        Open A Task from Memory.
+        """
+        error = None
+        for f in ("json", "yaml", "toml"):
+            # check every one:
+            try:
+                if f == "json":
+                    parse = JSONParser(content=payload)
+                    return await parse.run()
+                elif f == "yaml":
+                    parse = YAMLParser(content=payload)
+                    return await parse.run()
+                elif f == "toml":
+                    parse = TOMLParser(content=payload)
+                    return await parse.run()
+            except TaskParseError as e:
+                error = e
+                continue
+        else:
+            raise TaskParseError(
+                f"Task Parse Error: {error}"
+            )

flowtask/storages/tasks/row.py

@@ -0,0 +1,25 @@
+from typing import Union
+from flowtask.exceptions import TaskError
+from .abstract import AbstractTaskStorage
+
+
+class RowTaskStorage(AbstractTaskStorage):
+    """Task is saved directly into Task Table (SQL)."""
+    _name_: str = "Row"
+
+    def __init__(self, column_name: str = "task_definition", *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self._column = column_name
+
+    def set_definition(self, obj) -> None:
+        try:
+            self._task = obj
+        except (KeyError, AttributeError) as exc:
+            raise TaskError(
+                f"Unable to load Task definition from Table: {exc}"
+            ) from exc
+
+    async def open_task(
+        self, task: str, program: str, **kwargs
+    ) -> Union[dict, str]:
+        return self._task.get(self._column, None)