retsu 0.0.2__tar.gz

retsu-0.0.2/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Ivan Ogasawara
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.

retsu-0.0.2/PKG-INFO

@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: retsu
+Version: 0.0.2
+Summary: Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial tasks
+License: BSD 3 Clause
+Author: Ivan Ogasawara
+Author-email: ivan.ogasawara@gmail.com
+Requires-Python: >=3.8.1,<4
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: django
+Requires-Dist: atpublic (>=4.0)
+Requires-Dist: celery (>=5)
+Requires-Dist: django (>=3) ; extra == "django"
+Requires-Dist: redis (>=5)
+Description-Content-Type: text/markdown
+
+# Retsu
+
+Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial
+tasks
+
+- Software License: BSD 3 Clause
+- Documentation: https://osl-incubator.github.io/retsu
+
+## Features
+
+TBD
+
+## Credits
+
+This package was created with
+[scicookie](https://github.com/osl-incubator/scicookie) project template.
+

retsu-0.0.2/README.md

@@ -0,0 +1,16 @@
+# Retsu
+
+Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial
+tasks
+
+- Software License: BSD 3 Clause
+- Documentation: https://osl-incubator.github.io/retsu
+
+## Features
+
+TBD
+
+## Credits
+
+This package was created with
+[scicookie](https://github.com/osl-incubator/scicookie) project template.

retsu-0.0.2/pyproject.toml

@@ -0,0 +1,114 @@
+[tool.poetry]
+name = "retsu"
+version = "0.0.2"  # semantic-release
+description = "Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial tasks"
+readme = "README.md"
+authors = ["Ivan Ogasawara <ivan.ogasawara@gmail.com>"]
+packages = [
+  {include = "retsu", from="src"},
+]
+license = "BSD 3 Clause"
+exclude = [
+  ".git/*",
+  ".env*",
+]
+include = ["src/retsu/py.typed"]
+
+
+[tool.poetry.dependencies]
+python = ">=3.8.1,<4"
+atpublic = ">=4.0"
+celery = ">=5"
+redis = ">=5"
+django = { version = ">=3", optional = true }
+
+[tool.poetry.extras]
+django = [
+  "django",
+]
+
+
+[tool.poetry.group.dev.dependencies]
+pytest = ">=7.3.2"
+pytest-cov = ">=4.1.0"
+coverage = ">=7.2.7"
+pre-commit = ">=3.3.2"
+ruff = ">=0.2.0"
+mypy = ">=1.5.0"
+bandit = ">=1.7.5"
+vulture = ">=2.7"
+mccabe = ">=0.6.1"
+ipython = "<8"
+ipykernel = ">=6.0.0"
+Jinja2 = ">=3.1.2"
+mkdocs = ">=1.4.3"
+mkdocs-exclude = ">=1.0.2"
+mkdocs-jupyter = ">=0.24.1"
+mkdocs-literate-nav = ">=0.6.0"
+mkdocs-macros-plugin = ">=0.7.0,<1"
+mkdocs-material = ">=9.1.15"
+mkdocstrings = ">=0.21.2"
+mkdocstrings-python = ">=1.1.2"
+mkdocs-gen-files = ">=0.5.0"
+makim = "1.15.1"
+# 'PosixPath' object has no attribute 'endswith'
+virtualenv = "<=20.25.1"
+flask = ">=3"
+containers-sugar = "1.13.0"
+compose-go = "2.27.0"
+django = ">=3"
+django-stubs = ">=3"
+
+[tool.pytest.ini_options]
+testpaths = [
+    "tests",
+]
+
+[tool.bandit]
+exclude_dirs = ["tests"]
+targets = "./"
+
+[tool.vulture]
+exclude = ["tests"]
+ignore_decorators = []
+ignore_names = []
+make_whitelist = true
+min_confidence = 80
+paths = ["./"]
+sort_by_size = true
+verbose = false
+
+[tool.ruff]
+line-length = 79
+force-exclude = true
+src = ["./"]
+exclude = [
+  'docs',
+]
+
+[tool.ruff.lint]
+select = [
+  "E",   # pycodestyle
+  "F",   # pyflakes
+  "D",   # pydocstyle
+  "YTT", # flake8-2020
+  "RUF", # Ruff-specific rules
+  "I001",  # isort
+]
+
+[tool.ruff.lint.pydocstyle]
+convention = "numpy"
+
+[tool.ruff.lint.isort]
+# Use a single line between direct and from import
+lines-between-types = 1
+
+[tool.mypy]
+python_version = "3.8"
+check_untyped_defs = true
+strict = true
+ignore_missing_imports = true
+warn_unused_ignores = true
+warn_redundant_casts = true
+warn_unused_configs = true
+exclude = ["example/", "scripts/"]

retsu-0.0.2/src/retsu/__init__.py

@@ -0,0 +1,35 @@
+"""Retsu."""
+
+from importlib import metadata as importlib_metadata
+
+from retsu.core import (
+    ParallelTask,
+    ResultTask,
+    SerialTask,
+    TaskManager,
+)
+
+
+def get_version() -> str:
+    """Return the program version."""
+    try:
+        return importlib_metadata.version(__name__)
+    except importlib_metadata.PackageNotFoundError:  # pragma: no cover
+        return "0.0.2"  # semantic-release
+
+
+version = get_version()
+
+__version__ = version
+__author__ = "Ivan Ogasawara"
+__email__ = "ivan.ogasawara@gmail.com"
+
+__all__ = [
+    "__version__",
+    "__author__",
+    "__email__",
+    "ParallelTask",
+    "ResultTask",
+    "SerialTask",
+    "TaskManager",
+]

retsu-0.0.2/src/retsu/celery.py

@@ -0,0 +1,67 @@
+"""Retsu tasks with celery."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import celery
+
+from celery import chain, chord
+
+from retsu.core import ParallelTask, SerialTask
+
+
+class CeleryTask:
+    """Celery Task class."""
+
+    def task(self, *args, task_id: str, **kwargs) -> None:  # type: ignore
+        """Define the task to be executed."""
+        chord_tasks, chord_callback = self.get_chord_tasks(
+            *args, task_id=task_id, **kwargs
+        )
+        chain_tasks = self.get_chain_tasks(*args, task_id=task_id, **kwargs)
+
+        if chord_tasks:
+            if chord_callback:
+                workflow_chord = chord(chord_tasks, chord_callback)
+            else:
+                workflow_chord = chord(chord_tasks)
+            workflow_chord.apply_async()
+
+        if chain_tasks:
+            workflow_chain = chain(chord_tasks)
+            workflow_chain.apply_async()
+
+    def get_chord_tasks(  # type: ignore
+        self, *args, **kwargs
+    ) -> tuple[list[celery.Signature], Optional[celery.Signature]]:
+        """
+        Run tasks with chord.
+
+        Return
+        ------
+        tuple:
+            list of tasks for the chord, and the task to be used as a callback
+        """
+        chord_tasks: list[celery.Signature] = []
+        callback_task = None
+        return (chord_tasks, callback_task)
+
+    def get_chain_tasks(  # type: ignore
+        self, *args, **kwargs
+    ) -> list[celery.Signature]:
+        """Run tasks with chain."""
+        chain_tasks: list[celery.Signature] = []
+        return chain_tasks
+
+
+class ParallelCeleryTask(CeleryTask, ParallelTask):
+    """Parallel Task for Celery."""
+
+    ...
+
+
+class SerialCeleryTask(CeleryTask, SerialTask):
+    """Serial Task for Celery."""
+
+    ...

retsu-0.0.2/src/retsu/core.py

@@ -0,0 +1,198 @@
+"""Retsu core classes."""
+
+from __future__ import annotations
+
+import json
+import multiprocessing as mp
+import warnings
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Optional, cast
+from uuid import uuid4
+
+from public import public
+
+
+class ResultTask:
+    """Result from a task."""
+
+    def __init__(self, result_path: Path) -> None:
+        """Initialize ResultTask."""
+        self.result_path = result_path
+
+    @public
+    def save(self, task_id: str, result: Any) -> None:
+        """Save the result in a file."""
+        with open(self.result_path / f"{task_id}.json", "w") as f:
+            json.dump(
+                {"task_id": task_id, "result": result},
+                f,
+                indent=2,
+            )
+
+    @public
+    def load(self, task_id: str) -> dict[str, Any]:
+        """Load the result from a file."""
+        result_file = self.result_path / f"{task_id}.json"
+        if not result_file.exists():
+            raise Exception(f"File {result_file} doesn't exist.")
+        with open(result_file, "r") as f:
+            return cast(dict[str, Any], json.load(f))
+
+    @public
+    def status(self, task_id: str) -> bool:
+        """Return if the result for a given task was already stored."""
+        result_file = self.result_path / f"{task_id}.json"
+        return result_file.exists()
+
+    @public
+    def get(self, task_id: str) -> Any:
+        """Return the result for given task."""
+        if not self.status(task_id):
+            return {"status": False, "message": "Result not ready."}
+
+        return self.load(task_id)
+
+
+@public
+class Task:
+    """Main class for handling a task."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize a task object."""
+        self.active = True
+        self.workers = workers
+        self.result = ResultTask(result_path)
+        self.queue_in: mp.Queue[Any] = mp.Queue()
+        self.processes: list[mp.Process] = []
+
+    @public
+    def get_result(self, task_id: str) -> Any:
+        """Get the result for given task id."""
+        return self.result.get(task_id)
+
+    @public
+    def start(self) -> None:
+        """Start processes."""
+        for _ in range(self.workers):
+            p = mp.Process(target=self.run)
+            p.start()
+            self.processes.append(p)
+
+    @public
+    def stop(self) -> None:
+        """Stop processes."""
+        if not self.active:
+            return
+
+        self.active = False
+
+        for i in range(self.workers):
+            self.queue_in.put(None)
+
+        for i in range(self.workers):
+            p = self.processes[i]
+            p.join()
+
+        self.queue_in.close()
+        self.queue_in.join_thread()
+
+    @public
+    def request(self, *args, **kwargs) -> str:  # type: ignore
+        """Feed the queue with data from the request for the task."""
+        key = uuid4().hex
+        print(
+            {
+                "task_id": key,
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        self.queue_in.put(
+            {
+                "task_id": key,
+                "args": args,
+                "kwargs": kwargs,
+            },
+            block=False,
+        )
+        return key
+
+    @abstractmethod
+    def task(self, *args, task_id: str, **kwargs) -> None:  # type: ignore
+        """Define the task to be executed."""
+        raise Exception("`task` not implemented yet.")
+
+    def prepare_task(self, data: Any) -> None:
+        """Call the task with the necessary arguments."""
+        self.task(
+            *data["args"],
+            task_id=data["task_id"],
+            **data["kwargs"],
+        )
+
+    @public
+    def run(self) -> None:
+        """Run the task with data from the queue."""
+        while self.active:
+            data = self.queue_in.get()
+            if data is None:
+                print("Process terminated.")
+                self.active = False
+                return
+            self.prepare_task(data)
+
+
+class SerialTask(Task):
+    """Serial Task class."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize a serial task object."""
+        if workers != 1:
+            warnings.warn(
+                "SerialTask should have just 1 worker. "
+                "Switching automatically to 1 ..."
+            )
+            workers = 1
+        super().__init__(result_path, workers=workers)
+
+
+class ParallelTask(Task):
+    """Initialize a parallel task object."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize ParallelTask."""
+        if workers <= 1:
+            raise Exception("ParallelTask should have more than 1 worker.")
+
+        super().__init__(result_path, workers=workers)
+
+
+class TaskManager:
+    """Manage tasks."""
+
+    tasks: dict[str, Task]
+
+    def __init__(self) -> None:
+        """Create a list of retsu tasks."""
+        self.tasks: dict[str, Task] = {}
+
+    @public
+    def get_task(self, name: str) -> Optional[Task]:
+        """Get a task with the given name."""
+        return self.tasks.get(name)
+
+    @public
+    def start(self) -> None:
+        """Start tasks."""
+        for task_name, task in self.tasks.items():
+            print(f"Task `{task_name}` is starting ...")
+            task.start()
+
+    @public
+    def stop(self) -> None:
+        """Stop tasks."""
+        for task_name, task in self.tasks.items():
+            print(f"Task `{task_name}` is stopping ...")
+            task.stop()

retsu-0.0.2/src/retsu/plugins/__init__.py

@@ -0,0 +1 @@
+"""Sub-package for plugins."""

retsu-0.0.2/src/retsu/plugins/django.py

@@ -0,0 +1,32 @@
+"""Plugin for integrating with django."""
+
+from __future__ import annotations
+
+from typing import Type
+
+from django.apps import AppConfig
+from django.core.signals import request_finished
+
+from retsu.core import TaskManager
+
+
+def create_app_config(
+    manager: TaskManager, app_name: str = "myapp"
+) -> Type[AppConfig]:
+    """Create a django app config class."""
+
+    class RetsuAppConfig(AppConfig):
+        """RetsuAppConfig class."""
+
+        name = app_name
+
+        def ready(self) -> None:
+            """Start the task manager when the django app is ready."""
+            manager.start()
+            request_finished.connect(self.stop_multiprocessing)
+
+        def stop_multiprocessing(self, **kwargs) -> None:  # type: ignore
+            assert kwargs is not None
+            manager.stop()
+
+    return RetsuAppConfig