retsu 0.0.2__py3-none-any.whl

retsu/__init__.py

@@ -0,0 +1,35 @@
+"""Retsu."""
+
+from importlib import metadata as importlib_metadata
+
+from retsu.core import (
+    ParallelTask,
+    ResultTask,
+    SerialTask,
+    TaskManager,
+)
+
+
+def get_version() -> str:
+    """Return the program version."""
+    try:
+        return importlib_metadata.version(__name__)
+    except importlib_metadata.PackageNotFoundError:  # pragma: no cover
+        return "0.0.2"  # semantic-release
+
+
+version = get_version()
+
+__version__ = version
+__author__ = "Ivan Ogasawara"
+__email__ = "ivan.ogasawara@gmail.com"
+
+__all__ = [
+    "__version__",
+    "__author__",
+    "__email__",
+    "ParallelTask",
+    "ResultTask",
+    "SerialTask",
+    "TaskManager",
+]

retsu/celery.py

@@ -0,0 +1,67 @@
+"""Retsu tasks with celery."""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import celery
+
+from celery import chain, chord
+
+from retsu.core import ParallelTask, SerialTask
+
+
+class CeleryTask:
+    """Celery Task class."""
+
+    def task(self, *args, task_id: str, **kwargs) -> None:  # type: ignore
+        """Define the task to be executed."""
+        chord_tasks, chord_callback = self.get_chord_tasks(
+            *args, task_id=task_id, **kwargs
+        )
+        chain_tasks = self.get_chain_tasks(*args, task_id=task_id, **kwargs)
+
+        if chord_tasks:
+            if chord_callback:
+                workflow_chord = chord(chord_tasks, chord_callback)
+            else:
+                workflow_chord = chord(chord_tasks)
+            workflow_chord.apply_async()
+
+        if chain_tasks:
+            workflow_chain = chain(chord_tasks)
+            workflow_chain.apply_async()
+
+    def get_chord_tasks(  # type: ignore
+        self, *args, **kwargs
+    ) -> tuple[list[celery.Signature], Optional[celery.Signature]]:
+        """
+        Run tasks with chord.
+
+        Return
+        ------
+        tuple:
+            list of tasks for the chord, and the task to be used as a callback
+        """
+        chord_tasks: list[celery.Signature] = []
+        callback_task = None
+        return (chord_tasks, callback_task)
+
+    def get_chain_tasks(  # type: ignore
+        self, *args, **kwargs
+    ) -> list[celery.Signature]:
+        """Run tasks with chain."""
+        chain_tasks: list[celery.Signature] = []
+        return chain_tasks
+
+
+class ParallelCeleryTask(CeleryTask, ParallelTask):
+    """Parallel Task for Celery."""
+
+    ...
+
+
+class SerialCeleryTask(CeleryTask, SerialTask):
+    """Serial Task for Celery."""
+
+    ...

retsu/core.py

@@ -0,0 +1,198 @@
+"""Retsu core classes."""
+
+from __future__ import annotations
+
+import json
+import multiprocessing as mp
+import warnings
+
+from abc import abstractmethod
+from pathlib import Path
+from typing import Any, Optional, cast
+from uuid import uuid4
+
+from public import public
+
+
+class ResultTask:
+    """Result from a task."""
+
+    def __init__(self, result_path: Path) -> None:
+        """Initialize ResultTask."""
+        self.result_path = result_path
+
+    @public
+    def save(self, task_id: str, result: Any) -> None:
+        """Save the result in a file."""
+        with open(self.result_path / f"{task_id}.json", "w") as f:
+            json.dump(
+                {"task_id": task_id, "result": result},
+                f,
+                indent=2,
+            )
+
+    @public
+    def load(self, task_id: str) -> dict[str, Any]:
+        """Load the result from a file."""
+        result_file = self.result_path / f"{task_id}.json"
+        if not result_file.exists():
+            raise Exception(f"File {result_file} doesn't exist.")
+        with open(result_file, "r") as f:
+            return cast(dict[str, Any], json.load(f))
+
+    @public
+    def status(self, task_id: str) -> bool:
+        """Return if the result for a given task was already stored."""
+        result_file = self.result_path / f"{task_id}.json"
+        return result_file.exists()
+
+    @public
+    def get(self, task_id: str) -> Any:
+        """Return the result for given task."""
+        if not self.status(task_id):
+            return {"status": False, "message": "Result not ready."}
+
+        return self.load(task_id)
+
+
+@public
+class Task:
+    """Main class for handling a task."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize a task object."""
+        self.active = True
+        self.workers = workers
+        self.result = ResultTask(result_path)
+        self.queue_in: mp.Queue[Any] = mp.Queue()
+        self.processes: list[mp.Process] = []
+
+    @public
+    def get_result(self, task_id: str) -> Any:
+        """Get the result for given task id."""
+        return self.result.get(task_id)
+
+    @public
+    def start(self) -> None:
+        """Start processes."""
+        for _ in range(self.workers):
+            p = mp.Process(target=self.run)
+            p.start()
+            self.processes.append(p)
+
+    @public
+    def stop(self) -> None:
+        """Stop processes."""
+        if not self.active:
+            return
+
+        self.active = False
+
+        for i in range(self.workers):
+            self.queue_in.put(None)
+
+        for i in range(self.workers):
+            p = self.processes[i]
+            p.join()
+
+        self.queue_in.close()
+        self.queue_in.join_thread()
+
+    @public
+    def request(self, *args, **kwargs) -> str:  # type: ignore
+        """Feed the queue with data from the request for the task."""
+        key = uuid4().hex
+        print(
+            {
+                "task_id": key,
+                "args": args,
+                "kwargs": kwargs,
+            }
+        )
+        self.queue_in.put(
+            {
+                "task_id": key,
+                "args": args,
+                "kwargs": kwargs,
+            },
+            block=False,
+        )
+        return key
+
+    @abstractmethod
+    def task(self, *args, task_id: str, **kwargs) -> None:  # type: ignore
+        """Define the task to be executed."""
+        raise Exception("`task` not implemented yet.")
+
+    def prepare_task(self, data: Any) -> None:
+        """Call the task with the necessary arguments."""
+        self.task(
+            *data["args"],
+            task_id=data["task_id"],
+            **data["kwargs"],
+        )
+
+    @public
+    def run(self) -> None:
+        """Run the task with data from the queue."""
+        while self.active:
+            data = self.queue_in.get()
+            if data is None:
+                print("Process terminated.")
+                self.active = False
+                return
+            self.prepare_task(data)
+
+
+class SerialTask(Task):
+    """Serial Task class."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize a serial task object."""
+        if workers != 1:
+            warnings.warn(
+                "SerialTask should have just 1 worker. "
+                "Switching automatically to 1 ..."
+            )
+            workers = 1
+        super().__init__(result_path, workers=workers)
+
+
+class ParallelTask(Task):
+    """Initialize a parallel task object."""
+
+    def __init__(self, result_path: Path, workers: int = 1) -> None:
+        """Initialize ParallelTask."""
+        if workers <= 1:
+            raise Exception("ParallelTask should have more than 1 worker.")
+
+        super().__init__(result_path, workers=workers)
+
+
+class TaskManager:
+    """Manage tasks."""
+
+    tasks: dict[str, Task]
+
+    def __init__(self) -> None:
+        """Create a list of retsu tasks."""
+        self.tasks: dict[str, Task] = {}
+
+    @public
+    def get_task(self, name: str) -> Optional[Task]:
+        """Get a task with the given name."""
+        return self.tasks.get(name)
+
+    @public
+    def start(self) -> None:
+        """Start tasks."""
+        for task_name, task in self.tasks.items():
+            print(f"Task `{task_name}` is starting ...")
+            task.start()
+
+    @public
+    def stop(self) -> None:
+        """Stop tasks."""
+        for task_name, task in self.tasks.items():
+            print(f"Task `{task_name}` is stopping ...")
+            task.stop()

retsu/plugins/__init__.py

@@ -0,0 +1 @@
+"""Sub-package for plugins."""

retsu/plugins/django.py

@@ -0,0 +1,32 @@
+"""Plugin for integrating with django."""
+
+from __future__ import annotations
+
+from typing import Type
+
+from django.apps import AppConfig
+from django.core.signals import request_finished
+
+from retsu.core import TaskManager
+
+
+def create_app_config(
+    manager: TaskManager, app_name: str = "myapp"
+) -> Type[AppConfig]:
+    """Create a django app config class."""
+
+    class RetsuAppConfig(AppConfig):
+        """RetsuAppConfig class."""
+
+        name = app_name
+
+        def ready(self) -> None:
+            """Start the task manager when the django app is ready."""
+            manager.start()
+            request_finished.connect(self.stop_multiprocessing)
+
+        def stop_multiprocessing(self, **kwargs) -> None:  # type: ignore
+            assert kwargs is not None
+            manager.stop()
+
+    return RetsuAppConfig

retsu-0.0.2.dist-info/LICENSE

@@ -0,0 +1,29 @@
+BSD 3-Clause License
+
+Copyright (c) 2024, Ivan Ogasawara
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from this
+  software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
+OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
+OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+OF THE POSSIBILITY OF SUCH DAMAGE.

retsu-0.0.2.dist-info/METADATA

@@ -0,0 +1,38 @@
+Metadata-Version: 2.1
+Name: retsu
+Version: 0.0.2
+Summary: Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial tasks
+License: BSD 3 Clause
+Author: Ivan Ogasawara
+Author-email: ivan.ogasawara@gmail.com
+Requires-Python: >=3.8.1,<4
+Classifier: License :: Other/Proprietary License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: django
+Requires-Dist: atpublic (>=4.0)
+Requires-Dist: celery (>=5)
+Requires-Dist: django (>=3) ; extra == "django"
+Requires-Dist: redis (>=5)
+Description-Content-Type: text/markdown
+
+# Retsu
+
+Retsu aims to wrap-up Celery in way to facilitate to create parallel and serial
+tasks
+
+- Software License: BSD 3 Clause
+- Documentation: https://osl-incubator.github.io/retsu
+
+## Features
+
+TBD
+
+## Credits
+
+This package was created with
+[scicookie](https://github.com/osl-incubator/scicookie) project template.
+

retsu-0.0.2.dist-info/RECORD

@@ -0,0 +1,10 @@
+retsu/__init__.py,sha256=VadUF26rdNf4WofxcRRMHRLehGXMv9YTs8l21W8h8b4,666
+retsu/celery.py,sha256=qcSaE5qSEn7Ot1LuIT4LF9xcIYBMh1cp0exDFzyh1dU,1755
+retsu/core.py,sha256=5Tgx4OdmCUiJuWUu1cDvIpbMUE3oemlkM3k_aMw13vo,5494
+retsu/plugins/__init__.py,sha256=17Lin0MWK9SmSXUsb78YSfSDDDsJFr668cvo0NRymEw,31
+retsu/plugins/django.py,sha256=Wc3DYae5hWqiWXc1FDu_K5iNSE2hYdiE17WN7ToUlsU,827
+retsu/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+retsu-0.0.2.dist-info/LICENSE,sha256=VEAHetrmHicHpjqcUpjtcmdx7owGyXaJMhDU95IlnU8,1514
+retsu-0.0.2.dist-info/METADATA,sha256=I8TN6L624FKxQnrWAqJlLh3ZVVK9HK2SO84dwbGC1-Q,1075
+retsu-0.0.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+retsu-0.0.2.dist-info/RECORD,,

retsu-0.0.2.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: true
+Tag: py3-none-any