taskmanager-engine 0.1.0__py3-none-any.whl

taskmanager/api/events.py

@@ -0,0 +1,68 @@
+from __future__ import annotations
+
+import asyncio
+import logging
+from typing import Any
+
+from fastapi import WebSocket
+
+from taskmanager.core.broker import RedisBroker
+
+logger = logging.getLogger(__name__)
+
+
+class WebSocketEventManager:
+    """Manages active browser WebSocket clients and relays Redis events in real time."""
+
+    def __init__(self, broker: RedisBroker):
+        self.broker = broker
+        self.active_connections: set[WebSocket] = set()
+        self._listener_task: asyncio.Task[None] | None = None
+        self._running = False
+
+    async def connect(self, websocket: WebSocket) -> None:
+        await websocket.accept()
+        self.active_connections.add(websocket)
+
+    def disconnect(self, websocket: WebSocket) -> None:
+        self.active_connections.discard(websocket)
+
+    async def broadcast(self, message: dict[str, Any]) -> None:
+        """Sends a JSON message to all connected clients."""
+        if not self.active_connections:
+            return
+
+        dead_connections: set[WebSocket] = set()
+        for connection in list(self.active_connections):
+            try:
+                await connection.send_json(message)
+            except Exception:
+                dead_connections.add(connection)
+
+        for dead in dead_connections:
+            self.active_connections.discard(dead)
+
+    async def start_listener(self) -> None:
+        """Subscribes to Redis Pub/Sub and relays events to connected WebSockets."""
+        self._running = True
+        self._listener_task = asyncio.create_task(self._listen_loop())
+
+    async def stop_listener(self) -> None:
+        self._running = False
+        if self._listener_task and not self._listener_task.done():
+            self._listener_task.cancel()
+            try:
+                await self._listener_task
+            except asyncio.CancelledError:
+                pass
+
+    async def _listen_loop(self) -> None:
+        try:
+            async for event in self.broker.subscribe_events():
+                if not self._running:
+                    break
+                await self.broadcast(event)
+        except asyncio.CancelledError:
+            pass
+        except Exception as err:
+            logger.warning(f"WebSocket listener encountered error: {err}")

taskmanager/cli.py

@@ -0,0 +1,291 @@
+from __future__ import annotations
+
+import argparse
+import asyncio
+import importlib
+import importlib.util
+import logging
+import sys
+from pathlib import Path
+
+import fakeredis.aioredis
+import redis.asyncio as redis
+import uvicorn
+
+from taskmanager.api.app import create_app
+from taskmanager.config import settings
+from taskmanager.core.broker import RedisBroker
+from taskmanager.core.task import registry
+from taskmanager.scheduler.scheduler import Scheduler
+from taskmanager.worker.worker import Worker
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+logger = logging.getLogger("taskmanager.cli")
+
+_shared_fake_server = fakeredis.FakeServer()
+
+
+async def get_redis_client(redis_url: str | None = None, force_in_memory: bool = False) -> redis.Redis:
+    """Provides a connected Redis client with graceful fallback to in-memory FakeRedis."""
+    url = redis_url or settings.redis_url
+    if force_in_memory or (url and url.startswith("memory://")):
+        logger.info("🧠 Usando Redis In-Memory integrado (fakeredis).")
+        return fakeredis.aioredis.FakeRedis(server=_shared_fake_server, decode_responses=True)
+
+    try:
+        client = redis.from_url(url, decode_responses=True)
+        # Test connection vitality
+        await asyncio.wait_for(client.ping(), timeout=1.5)
+        return client
+    except Exception:
+        logger.warning(
+            f"⚠️ Servidor Redis em '{url}' não está acessível. "
+            "Iniciando automaticamente com Redis In-Memory integrado para desenvolvimento!"
+        )
+        logger.info("💡 Dica: Para conectar a um Redis real, execute: docker run -d -p 6379:6379 redis:alpine")
+        return fakeredis.aioredis.FakeRedis(server=_shared_fake_server, decode_responses=True)
+
+
+def auto_discover_tasks(modules: list[str]) -> None:
+    """Imports user modules or file paths to register decorated @task functions."""
+    cwd = str(Path.cwd().resolve())
+    if cwd not in sys.path:
+        sys.path.insert(0, cwd)
+
+    for mod in modules:
+        try:
+            # If path to a python file was passed directly (e.g. 'example_tasks.py')
+            if mod.endswith(".py") or "/" in mod or "\\" in mod:
+                path = Path(mod).resolve()
+                if path.exists():
+                    mod_name = path.stem
+                    spec = importlib.util.spec_from_file_location(mod_name, str(path))
+                    if spec and spec.loader:
+                        module = importlib.util.module_from_spec(spec)
+                        sys.modules[mod_name] = module
+                        spec.loader.exec_module(module)
+                        logger.info(f"Loaded tasks from file: {mod}")
+                        continue
+
+            # Standard python module import (e.g. 'example_tasks')
+            importlib.import_module(mod)
+            logger.info(f"Loaded tasks from module: {mod}")
+        except Exception as err:
+            logger.warning(f"Could not import module '{mod}': {err}")
+
+
+async def run_worker(
+    queues: list[str],
+    concurrency: int,
+    name: str | None,
+    max_memory_mb: float | None = None,
+    max_cpu_percent: float | None = None,
+    redis_url: str | None = None,
+    in_memory: bool = False,
+) -> None:
+    client = await get_redis_client(redis_url, force_in_memory=in_memory)
+    broker = RedisBroker(client, prefix=settings.redis_prefix)
+    registry.set_broker(broker)
+    worker = Worker(
+        queues=queues,
+        concurrency=concurrency,
+        name=name,
+        max_memory_mb=max_memory_mb,
+        max_cpu_percent=max_cpu_percent,
+        broker=broker,
+        task_registry=registry,
+    )
+    try:
+        await worker.start()
+    except KeyboardInterrupt:
+        await worker.stop()
+
+
+async def run_scheduler(redis_url: str | None = None, in_memory: bool = False) -> None:
+    client = await get_redis_client(redis_url, force_in_memory=in_memory)
+    broker = RedisBroker(client, prefix=settings.redis_prefix)
+    sched = Scheduler(broker)
+    try:
+        await sched.start()
+    except KeyboardInterrupt:
+        await sched.stop()
+
+
+async def run_server(
+    host: str,
+    port: int,
+    redis_url: str | None = None,
+    in_memory: bool = False,
+) -> None:
+    client = await get_redis_client(redis_url, force_in_memory=in_memory)
+    broker = RedisBroker(client, prefix=settings.redis_prefix)
+    registry.set_broker(broker)
+    app = create_app(broker=broker, task_reg=registry)
+    config = uvicorn.Config(app=app, host=host, port=port, log_level="info")
+    server = uvicorn.Server(config)
+    await server.serve()
+
+
+async def run_dev(
+    host: str,
+    port: int,
+    queues: list[str],
+    concurrency: int,
+    max_memory_mb: float | None = None,
+    max_cpu_percent: float | None = None,
+    redis_url: str | None = None,
+    in_memory: bool = False,
+) -> None:
+    """Runs API Server, Worker, and Scheduler in a single process with shared broker."""
+    client = await get_redis_client(redis_url, force_in_memory=in_memory)
+    broker = RedisBroker(client, prefix=settings.redis_prefix)
+    registry.set_broker(broker)
+
+    # In dev mode, auto-listen to all registered queues if default was selected
+    active_queues = list(queues)
+    if queues == ["default"]:
+        for t_name in registry.list_tasks():
+            t_obj = registry.get(t_name)
+            if t_obj and t_obj.queue not in active_queues:
+                active_queues.append(t_obj.queue)
+
+    worker = Worker(
+        queues=active_queues,
+        concurrency=concurrency,
+        name="dev-worker",
+        max_memory_mb=max_memory_mb,
+        max_cpu_percent=max_cpu_percent,
+        broker=broker,
+        task_registry=registry,
+    )
+    app = create_app(broker=broker, task_reg=registry)
+
+    config = uvicorn.Config(app=app, host=host, port=port, log_level="info")
+    server = uvicorn.Server(config)
+
+    logger.info(f"🚀 TaskManager Dev Environment running at http://{host}:{port}")
+
+    await asyncio.gather(
+        server.serve(),
+        worker.start(),
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        prog="taskmanager",
+        description="TaskManager: Celery/BullMQ-like background task manager and Linear dark SPA dashboard",
+    )
+    subparsers = parser.add_subparsers(dest="command", required=True)
+
+    # 1. Server Command
+    server_parser = subparsers.add_parser("server", help="Start the FastAPI dashboard & API server")
+    server_parser.add_argument("--host", default="0.0.0.0", help="Host address (default: 0.0.0.0)")
+    server_parser.add_argument("--port", type=int, default=8000, help="Port (default: 8000)")
+    server_parser.add_argument("--redis-url", default=None, help="Redis connection URL")
+    server_parser.add_argument("--in-memory", action="store_true", help="Force in-memory Redis")
+    server_parser.add_argument(
+        "--app-module", nargs="*", default=[], help="Python modules containing @task definitions"
+    )
+
+    # 2. Worker Command
+    worker_parser = subparsers.add_parser("worker", help="Start an async worker process")
+    worker_parser.add_argument(
+        "-q", "--queues", default="default", help="Comma-separated queue names (default: default)"
+    )
+    worker_parser.add_argument(
+        "-c", "--concurrency", type=int, default=5, help="Worker concurrency (default: 5)"
+    )
+    worker_parser.add_argument("-n", "--name", default=None, help="Custom worker name")
+    worker_parser.add_argument("--max-memory-mb", type=float, default=None, help="Max RSS memory (MB) before backpressure")
+    worker_parser.add_argument("--max-cpu-percent", type=float, default=None, help="Max CPU percentage before backpressure")
+    worker_parser.add_argument("--redis-url", default=None, help="Redis connection URL")
+    worker_parser.add_argument("--in-memory", action="store_true", help="Force in-memory Redis")
+    worker_parser.add_argument(
+        "-m", "--modules", nargs="*", default=[], help="Python modules to import for tasks"
+    )
+
+    # 3. Scheduler Command
+    sched_parser = subparsers.add_parser("scheduler", help="Start the dynamic cron scheduler")
+    sched_parser.add_argument("--redis-url", default=None, help="Redis connection URL")
+    sched_parser.add_argument("--in-memory", action="store_true", help="Force in-memory Redis")
+    sched_parser.add_argument(
+        "-m", "--modules", nargs="*", default=[], help="Python modules to import for tasks"
+    )
+
+    # 4. Dev Command (All-in-one)
+    dev_parser = subparsers.add_parser(
+        "dev", help="Start Server, Worker, and Scheduler in one command"
+    )
+    dev_parser.add_argument("--host", default="0.0.0.0", help="Host address (default: 0.0.0.0)")
+    dev_parser.add_argument("--port", type=int, default=8000, help="Port (default: 8000)")
+    dev_parser.add_argument("-q", "--queues", default="default", help="Comma-separated queue names")
+    dev_parser.add_argument("-c", "--concurrency", type=int, default=5, help="Worker concurrency")
+    dev_parser.add_argument("--max-memory-mb", type=float, default=None, help="Max RSS memory (MB) before backpressure")
+    dev_parser.add_argument("--max-cpu-percent", type=float, default=None, help="Max CPU percentage before backpressure")
+    dev_parser.add_argument("--redis-url", default=None, help="Redis connection URL")
+    dev_parser.add_argument("--in-memory", action="store_true", help="Force in-memory Redis")
+    dev_parser.add_argument(
+        "-m", "--modules", nargs="*", default=[], help="Python modules to import"
+    )
+
+    args = parser.parse_args()
+
+    if args.command == "server":
+        auto_discover_tasks(args.app_module)
+        asyncio.run(
+            run_server(
+                host=args.host,
+                port=args.port,
+                redis_url=args.redis_url,
+                in_memory=args.in_memory,
+            )
+        )
+
+    elif args.command == "worker":
+        auto_discover_tasks(args.modules)
+        queues = [q.strip() for q in args.queues.split(",") if q.strip()]
+        asyncio.run(
+            run_worker(
+                queues=queues,
+                concurrency=args.concurrency,
+                name=args.name,
+                max_memory_mb=args.max_memory_mb,
+                max_cpu_percent=args.max_cpu_percent,
+                redis_url=args.redis_url,
+                in_memory=args.in_memory,
+            )
+        )
+
+    elif args.command == "scheduler":
+        auto_discover_tasks(args.modules)
+        asyncio.run(
+            run_scheduler(
+                redis_url=args.redis_url,
+                in_memory=args.in_memory,
+            )
+        )
+
+    elif args.command == "dev":
+        auto_discover_tasks(args.modules)
+        queues = [q.strip() for q in args.queues.split(",") if q.strip()]
+        asyncio.run(
+            run_dev(
+                host=args.host,
+                port=args.port,
+                queues=queues,
+                concurrency=args.concurrency,
+                max_memory_mb=args.max_memory_mb,
+                max_cpu_percent=args.max_cpu_percent,
+                redis_url=args.redis_url,
+                in_memory=args.in_memory,
+            )
+        )
+
+
+if __name__ == "__main__":
+    main()

taskmanager/config.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+import os
+
+from pydantic import BaseModel, Field
+
+
+class Settings(BaseModel):
+    redis_url: str = Field(
+        default_factory=lambda: os.getenv("REDIS_URL", "redis://localhost:6379/0")
+    )
+    redis_prefix: str = Field(default_factory=lambda: os.getenv("REDIS_PREFIX", "tm"))
+    default_queue: str = Field(default_factory=lambda: os.getenv("DEFAULT_QUEUE", "default"))
+    worker_heartbeat_interval: float = 3.0
+    worker_heartbeat_ttl: float = 10.0
+    scheduler_poll_interval: float = 1.0
+
+
+settings = Settings()

taskmanager/contrib/__init__.py

@@ -0,0 +1,3 @@
+"""
+Framework integrations and contrib modules for TaskManager (FastAPI, Django, Flask).
+"""

taskmanager/contrib/django/__init__.py

@@ -0,0 +1,6 @@
+"""
+Django Contrib Package for TaskManager.
+Provides automatic task discovery, management commands, and dashboard integration.
+"""
+
+default_app_config = "taskmanager.contrib.django.apps.TaskManagerConfig"

taskmanager/contrib/django/apps.py

@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+import importlib
+import logging
+
+try:
+    from django.apps import AppConfig
+    from django.conf import settings
+    DJANGO_AVAILABLE = True
+except ImportError:
+    AppConfig = object  # type: ignore
+    settings = None  # type: ignore
+    DJANGO_AVAILABLE = False
+
+logger = logging.getLogger("taskmanager.contrib.django")
+
+
+def autodiscover_tasks() -> list[str]:
+    """Scans all INSTALLED_APPS for <app>.tasks modules and imports them."""
+    if not DJANGO_AVAILABLE or not settings:
+        return []
+
+    discovered: list[str] = []
+    installed_apps = getattr(settings, "INSTALLED_APPS", [])
+
+    for app_path in installed_apps:
+        if app_path.startswith("taskmanager") or app_path.startswith("django."):
+            continue
+        try:
+            mod_name = f"{app_path}.tasks"
+            importlib.import_module(mod_name)
+            discovered.append(mod_name)
+            logger.debug("Discovered tasks module: %s", mod_name)
+        except (ImportError, AttributeError):
+            pass
+
+    return discovered
+
+
+class TaskManagerConfig(AppConfig):  # type: ignore
+    name = "taskmanager.contrib.django"
+    verbose_name = "TaskManager Background Engine"
+
+    def ready(self) -> None:
+        if DJANGO_AVAILABLE:
+            autodiscover_tasks()

taskmanager/contrib/django/management/__init__.py

@@ -0,0 +1 @@
+"""Management module for Django integration."""

taskmanager/contrib/django/management/commands/__init__.py

@@ -0,0 +1 @@
+"""Management commands for TaskManager Django integration."""

taskmanager/contrib/django/management/commands/run_scheduler.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+
+try:
+    from django.conf import settings
+    from django.core.management.base import BaseCommand
+    DJANGO_AVAILABLE = True
+except ImportError:
+    BaseCommand = object  # type: ignore
+    settings = None  # type: ignore
+    DJANGO_AVAILABLE = False
+
+from taskmanager.cli import get_redis_client
+from taskmanager.core.broker import RedisBroker
+from taskmanager.scheduler.scheduler import Scheduler
+
+
+class Command(BaseCommand):  # type: ignore
+    help = "Starts the TaskManager dynamic cron and scheduled job runner."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--redis-url",
+            type=str,
+            default=None,
+            help="Redis connection URI (defaults to TASKMANAGER_REDIS_URL in settings or env).",
+        )
+
+    def handle(self, *args, **options):
+        if not DJANGO_AVAILABLE:
+            sys.stderr.write("Django is required to run this command.\n")
+            sys.exit(1)
+
+        redis_url = options.get("redis_url") or getattr(settings, "TASKMANAGER_REDIS_URL", None)
+
+        async def _run():
+            client = await get_redis_client(redis_url)
+            broker = RedisBroker(client)
+            scheduler = Scheduler(broker)
+            await scheduler.start()
+
+        asyncio.run(_run())

taskmanager/contrib/django/management/commands/run_worker.py

@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+
+try:
+    from django.conf import settings
+    from django.core.management.base import BaseCommand
+    DJANGO_AVAILABLE = True
+except ImportError:
+    BaseCommand = object  # type: ignore
+    settings = None  # type: ignore
+    DJANGO_AVAILABLE = False
+
+from taskmanager.cli import get_redis_client
+from taskmanager.core.broker import RedisBroker
+from taskmanager.core.task import registry
+from taskmanager.worker.worker import Worker
+
+
+class Command(BaseCommand):  # type: ignore
+    help = "Starts a TaskManager background worker instance consuming from designated queues."
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--queues",
+            type=str,
+            default="default",
+            help="Comma-separated list of queue names to consume (default: 'default').",
+        )
+        parser.add_argument(
+            "--concurrency",
+            type=int,
+            default=5,
+            help="Number of concurrent task execution slots (default: 5).",
+        )
+        parser.add_argument(
+            "--name",
+            type=str,
+            default=None,
+            help="Worker identification name (defaults to hostname-worker-<uuid>).",
+        )
+        parser.add_argument(
+            "--redis-url",
+            type=str,
+            default=None,
+            help="Redis connection URI (defaults to TASKMANAGER_REDIS_URL in settings or env).",
+        )
+
+    def handle(self, *args, **options):
+        if not DJANGO_AVAILABLE:
+            sys.stderr.write("Django is required to run this command.\n")
+            sys.exit(1)
+
+        queues = [q.strip() for q in options["queues"].split(",") if q.strip()]
+        concurrency = options["concurrency"]
+        name = options["name"]
+        redis_url = options.get("redis_url") or getattr(settings, "TASKMANAGER_REDIS_URL", None)
+
+        async def _run():
+            client = await get_redis_client(redis_url)
+            broker = RedisBroker(client)
+            worker = Worker(
+                broker=broker,
+                registry=registry,
+                queues=queues,
+                concurrency=concurrency,
+                name=name,
+            )
+            await worker.start()
+
+        asyncio.run(_run())

taskmanager/contrib/django/urls.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+try:
+    from django.http import FileResponse, HttpResponse
+    from django.urls import path
+    DJANGO_AVAILABLE = True
+except ImportError:
+    DJANGO_AVAILABLE = False
+
+
+app_name = "taskmanager"
+
+
+def dashboard_view(request):
+    """Serves the TaskManager SPA dashboard in standard Django setups."""
+    ui_dir = Path(__file__).resolve().parent.parent.parent / "ui"
+    index_file = ui_dir / "index.html"
+    if index_file.exists():
+        return FileResponse(open(index_file, "rb"), content_type="text/html")
+    return HttpResponse("TaskManager Dashboard UI not found.", status=404)
+
+
+if DJANGO_AVAILABLE:
+    urlpatterns = [
+        path("", dashboard_view, name="dashboard"),
+    ]
+else:
+    urlpatterns = []

taskmanager/contrib/fastapi.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from taskmanager.api.app import create_app
+from taskmanager.core.broker import RedisBroker
+from taskmanager.core.task import TaskRegistry, registry
+
+
+def mount_taskmanager(
+    parent_app: FastAPI,
+    path: str = "/tasks",
+    broker: RedisBroker | None = None,
+    task_reg: TaskRegistry | None = None,
+    redis_url: str | None = None,
+    prefix: str | None = None,
+) -> FastAPI:
+    """
+    Mounts the TaskManager dashboard and REST API onto an existing FastAPI/Starlette application.
+
+    Example:
+        app = FastAPI()
+        mount_taskmanager(app, path="/tasks")
+    """
+    sub_app = create_app(
+        broker=broker,
+        task_reg=task_reg or registry,
+        root_path=path,
+    )
+    parent_app.mount(path, sub_app)
+    return sub_app