aqworker 0.1.0__py3-none-any.whl

aqworker/__init__.py

@@ -0,0 +1,18 @@
+"""
+AQWorker - Async Queue Worker for Python applications.
+"""
+
+from aqworker.core import AQWorker
+from aqworker.handler.base import BaseHandler
+from aqworker.handler.registry import HandlerRegistry
+from aqworker.worker.base import BaseWorker, WorkerConfig
+from aqworker.worker.registry import WorkerRegistry
+
+__all__ = [
+    "AQWorker",
+    "WorkerRegistry",
+    "HandlerRegistry",
+    "BaseWorker",
+    "BaseHandler",
+    "WorkerConfig",
+]

aqworker/cli/__init__.py

@@ -0,0 +1,3 @@
+from aqworker.cli.cli import app
+
+__all__ = ["app"]

aqworker/cli/cli.py

@@ -0,0 +1,180 @@
+"""
+Command line entry points for aqworker.
+
+Expose management commands via `python -m aqworker` or the installed
+console script `aqworker`.
+"""
+
+from __future__ import annotations
+
+from typing import Optional
+
+import typer
+
+from aqworker.cli.commands import (
+    RegistryError,
+    get_queue_stats,
+    list_handler_descriptors,
+    list_queue_names,
+    list_worker_names,
+    run_worker,
+)
+from aqworker.cli.loader import get_aqworker_file, load_aqworker_from_file
+from aqworker.core import AQWorker
+
+app = typer.Typer(name="aqworker", help="AQWorker management CLI")
+
+# Create a default AQWorker instance for CLI
+# Users should register workers/handlers before using CLI commands
+_default_aq_worker = AQWorker()
+
+
+def _get_aqworker_instance(file_path: Optional[str] = None) -> AQWorker:
+    """
+    Get AQWorker instance from file or use default.
+
+    Args:
+        file_path: Optional path to Python file containing AQWorker instance
+
+    Returns:
+        AQWorker instance
+    """
+    # Priority: file_path argument > env variable > default
+    if file_path:
+        try:
+            return load_aqworker_from_file(file_path)
+        except Exception as e:
+            typer.echo(f"Error loading AQWorker from {file_path}: {e}", err=True)
+            raise typer.Exit(code=1) from e
+
+    env_file = get_aqworker_file()
+    if env_file:
+        try:
+            return load_aqworker_from_file(env_file)
+        except Exception as e:
+            typer.echo(
+                f"Error loading AQWorker from AQWORKER_FILE={env_file}: {e}", err=True
+            )
+            raise typer.Exit(code=1) from e
+
+    return _default_aq_worker
+
+
+def _handle_registry_error(func):
+    """Decorator-like helper to convert RegistryError to Typer exits."""
+
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except RegistryError as exc:
+            typer.echo(str(exc))
+            raise typer.Exit(code=1) from exc
+
+    return wrapper
+
+
+@app.command("list:aq_worker")
+def list_workers(
+    file_path: Optional[str] = typer.Argument(
+        None, help="Path to Python file containing AQWorker instance"
+    )
+):
+    """List available aq_worker names."""
+
+    @_handle_registry_error
+    def _impl():
+        aq_worker = _get_aqworker_instance(file_path)
+        for name in list_worker_names(aq_worker):
+            typer.echo(name)
+
+    _impl()
+
+
+@app.command("list:handlers")
+def list_handlers(
+    file_path: Optional[str] = typer.Argument(
+        None, help="Path to Python file containing AQWorker instance"
+    )
+):
+    """List registered handlers names."""
+
+    @_handle_registry_error
+    def _impl():
+        aq_worker = _get_aqworker_instance(file_path)
+        for name, dotted in list_handler_descriptors(aq_worker):
+            typer.echo(f"{name} ({dotted})")
+
+    _impl()
+
+
+@app.command("list:queue")
+def list_queues(
+    file_path: Optional[str] = typer.Argument(
+        None, help="Path to Python file containing AQWorker instance"
+    )
+):
+    """List queues declared by registered workers."""
+
+    @_handle_registry_error
+    def _impl():
+        aq_worker = _get_aqworker_instance(file_path)
+        for queue in list_queue_names(aq_worker):
+            typer.echo(queue)
+
+    _impl()
+
+
+@app.command("start")
+def start_worker(
+    worker_name: str = typer.Argument(..., help="Worker name to start"),
+    file_path: Optional[str] = typer.Argument(
+        None, help="Path to Python file containing AQWorker instance"
+    ),
+):
+    """Start a aq_worker by name."""
+
+    @_handle_registry_error
+    def _impl():
+        aq_worker = _get_aqworker_instance(file_path)
+        run_worker(aq_worker, worker_name)
+
+    try:
+        _impl()
+    except typer.Exit:
+        raise
+    except KeyboardInterrupt:
+        typer.echo("Worker interrupted, shutting down...")
+    except Exception as exc:  # pragma: no cover
+        typer.echo(f"Failed to run aq_worker: {exc}")
+        raise typer.Exit(code=1) from exc
+
+
+@app.command("stats")
+def queue_stats(
+    queue_name: str = typer.Argument(..., help="Queue name to get statistics for"),
+    file_path: Optional[str] = typer.Argument(
+        None, help="Path to Python file containing AQWorker instance"
+    ),
+):
+    """Print queue statistics for a given queue."""
+
+    @_handle_registry_error
+    def _impl():
+        aq_worker = _get_aqworker_instance(file_path)
+        job_service = aq_worker.job_service
+        stats = get_queue_stats(queue_name, job_service)
+        if not stats:
+            typer.echo(f"No stats available for queue '{queue_name}'.")
+            raise typer.Exit(code=1)
+
+        typer.echo(f"Queue: {queue_name}")
+        typer.echo(f"  Pending:    {stats.get('pending', 0)}")
+        typer.echo(f"  Processing: {stats.get('processing', 0)}")
+        typer.echo(f"  Completed:  {stats.get('completed', 0)}")
+        typer.echo(f"  Failed:     {stats.get('failed', 0)}")
+
+    _impl()
+
+
+if __name__ == "__main__":
+    app()

aqworker/cli/commands.py

@@ -0,0 +1,67 @@
+"""
+Reusable logic for CLI commands, extracted for easier testing.
+"""
+
+import asyncio
+from typing import TYPE_CHECKING, Dict, List, Optional, Tuple
+
+from aqworker.core import AQWorker
+from aqworker.job.service import JobService
+
+if TYPE_CHECKING:
+    from aqworker.worker.base import BaseWorker
+
+
+class RegistryError(RuntimeError):
+    """Raised when required registry data is missing."""
+
+
+def list_worker_names(aq_worker: AQWorker) -> List[str]:
+    """Return sorted aq_worker names."""
+    return sorted(defn.name for defn in aq_worker.worker_registry.list_definitions())
+
+
+def list_handler_descriptors(aq_worker: AQWorker) -> List[Tuple[str, str]]:
+    """Return (name, dotted path) tuples for handlers."""
+    handlers = aq_worker.handler_registry.snapshot()
+    descriptors = []
+    for name, handler_cls in handlers.items():
+        descriptors.append((name, f"{handler_cls.__module__}.{handler_cls.__name__}"))
+    return sorted(descriptors, key=lambda item: item[0])
+
+
+def list_queue_names(aq_worker: AQWorker) -> List[str]:
+    """Return sorted queue names inferred from workers."""
+    definitions = aq_worker.worker_registry.list_definitions()
+    queues = set[str]()
+    for definition in definitions:
+        queues.update(definition.queue_names)
+    return sorted(queues)
+
+
+def build_worker(aq_worker: AQWorker, worker_name: str) -> "BaseWorker":
+    """Construct an aq_worker instance without starting it."""
+    return aq_worker.create_worker(worker_name)  # type: ignore[no-any-return]
+
+
+def run_worker(aq_worker: AQWorker, worker_name: str) -> None:
+    """Run a aq_worker event loop."""
+    worker = build_worker(aq_worker, worker_name)
+    asyncio.run(worker.run())
+
+
+async def get_queue_stats_async(
+    queue_name: str, job_service: Optional[JobService] = None
+) -> Dict[str, int]:
+    """Async helper for queue stats (handy for unit tests)."""
+    if job_service is None:
+        raise ValueError("job_service is required for get_queue_stats_async")
+    stats = await job_service.get_queue_stats([queue_name])
+    return stats or {}
+
+
+def get_queue_stats(
+    queue_name: str, job_service: Optional[JobService] = None
+) -> Dict[str, int]:
+    """Sync wrapper for queue stats."""
+    return asyncio.run(get_queue_stats_async(queue_name, job_service))

aqworker/cli/loader.py

@@ -0,0 +1,97 @@
+"""
+Utility to load AQWorker instance from a Python file.
+"""
+
+import importlib.util
+import os
+import sys
+from pathlib import Path
+from typing import Optional
+
+from aqworker.core import AQWorker
+
+
+def load_aqworker_from_file(file_path: str) -> AQWorker:
+    """
+    Load AQWorker instance from a Python file.
+
+    Args:
+        file_path: Path to Python file containing AQWorker instance
+
+    Returns:
+        AQWorker instance found in the file
+
+    Raises:
+        FileNotFoundError: If file doesn't exist
+        ValueError: If no AQWorker instance found in file
+    """
+    file_path_obj = Path(file_path)
+
+    # Resolve absolute path
+    if not file_path_obj.is_absolute():
+        # Try relative to current working directory
+        abs_path = Path.cwd() / file_path_obj
+        if not abs_path.exists():
+            # Try relative to project root (if we can detect it)
+            abs_path = file_path_obj.resolve()
+    else:
+        abs_path = file_path_obj
+
+    if not abs_path.exists():
+        raise FileNotFoundError(f"File not found: {file_path}")
+
+    if not abs_path.suffix == ".py":
+        raise ValueError(f"File must be a Python file (.py): {file_path}")
+
+    # Convert path to module path
+    # e.g., examples/simple/aq_worker.py -> examples.simple.aq_worker
+    module_name = abs_path.stem
+
+    # Load the module
+    spec = importlib.util.spec_from_file_location(module_name, abs_path)
+    if spec is None or spec.loader is None:
+        raise ValueError(f"Could not load module from file: {file_path}")
+
+    module = importlib.util.module_from_spec(spec)
+
+    # Add parent directory to sys.path if needed for imports
+    parent_dir = str(abs_path.parent)
+    if parent_dir not in sys.path:
+        sys.path.insert(0, parent_dir)
+
+    try:
+        spec.loader.exec_module(module)
+    except Exception as e:
+        raise ValueError(f"Error loading module from {file_path}: {e}") from e
+
+    # Find AQWorker instance in the module
+    # Try common names first
+    common_names = ["aq_worker", "aq_worker", "aqworker", "aq"]
+    for name in common_names:
+        if hasattr(module, name):
+            attr = getattr(module, name)
+            if isinstance(attr, AQWorker):
+                return attr
+
+    # Search all attributes for AQWorker instances
+    for attr_name in dir(module):
+        if not attr_name.startswith("_"):  # Skip private attributes
+            attr = getattr(module, attr_name)
+            if isinstance(attr, AQWorker):
+                return attr
+
+    raise ValueError(
+        f"No AQWorker instance found in {file_path}. "
+        f"Expected an instance variable named one of: {', '.join(common_names)}, "
+        f"or any attribute of type AQWorker."
+    )
+
+
+def get_aqworker_file() -> Optional[str]:
+    """
+    Get AQWorker file path from environment variable.
+
+    Returns:
+        File path if AQWORKER_FILE is set, None otherwise
+    """
+    return os.environ.get("AQWORKER_FILE")

aqworker/constants.py

@@ -0,0 +1,31 @@
+"""
+Constants for the aq_worker system.
+"""
+
+LOGGER_NAME = "aqworker"
+
+# Queue prefixes
+JOB_QUEUE_PREFIX = "aqw"
+JOB_STATUS_PREFIX = "aqw:job:"
+JOB_LOCK_PREFIX = "aqw:jl:"
+
+# Special queue names
+PROCESSING_QUEUE = f"{JOB_QUEUE_PREFIX}:processing"
+COMPLETED_QUEUE = f"{JOB_QUEUE_PREFIX}:completed"
+FAILED_QUEUE = f"{JOB_QUEUE_PREFIX}:failed"
+
+
+# Queue name patterns
+def get_queue_name(queue_name: str) -> str:
+    """Get full queue name for a specific queue."""
+    return f"{JOB_QUEUE_PREFIX}:{queue_name}"
+
+
+def get_job_status_key(job_id: str) -> str:
+    """Get job status key for a job ID."""
+    return f"{JOB_STATUS_PREFIX}{job_id}"
+
+
+def get_job_lock_key(job_id: str) -> str:
+    """Get Redis key used for per-job locks."""
+    return f"{JOB_LOCK_PREFIX}{job_id}"

aqworker/core.py

@@ -0,0 +1,264 @@
+"""
+Core AQWorker class - Main entry point for the aq_worker system.
+"""
+
+import importlib
+import inspect
+import pkgutil
+from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, Optional, Type
+
+from aqworker.handler.base import BaseHandler
+from aqworker.handler.registry import HandlerRegistry
+from aqworker.job.service import JobService
+from aqworker.logger import logger
+from aqworker.worker.dispatcher import HandlerDispatcher
+from aqworker.worker.registry import WorkerRegistry
+
+if TYPE_CHECKING:
+    from worker.base import BaseWorker
+
+
+class AQWorker:
+    """
+    Main AQWorker instance containing WorkerRegistry, HandlerRegistry, and JobService.
+
+    This is the primary entry point for registering workers and handlers, and managing jobs.
+    """
+
+    def __init__(
+        self,
+        worker_registry: Optional[WorkerRegistry] = None,
+        handler_registry: Optional[HandlerRegistry] = None,
+        job_service: Optional[JobService] = None,
+        redis_host: str = "localhost",
+        redis_port: int = 6379,
+        redis_db: int = 0,
+        include_packages: Optional[Iterable[str]] = None,
+    ):
+        """
+        Initialize AQWorker with registries and job service.
+
+        Args:
+            worker_registry: Optional WorkerRegistry instance. Creates new one if not provided.
+            handler_registry: Optional HandlerRegistry instance. Creates new one if not provided.
+            job_service: Optional JobService instance. Creates new one if not provided.
+            redis_host: Redis server hostname. Defaults to "localhost".
+                       Only used when job_service is not provided.
+            redis_port: Redis server port. Defaults to 6379.
+                       Only used when job_service is not provided.
+            redis_db: Redis database number. Defaults to 0.
+                      Only used when job_service is not provided.
+            include_packages: Optional iterable of package/module names to import
+                               for handler auto-discovery.
+        """
+        self.worker_registry = worker_registry or WorkerRegistry()
+        self.handler_registry = handler_registry or HandlerRegistry()
+        self.job_service: Optional[JobService] = job_service or JobService(
+            redis_host=redis_host, redis_port=redis_port, redis_db=redis_db
+        )
+        self.handler_dispatcher = HandlerDispatcher(self.handler_registry)
+        self._workers: Dict[str, Optional[Type["BaseWorker"]]] = {}
+        self._include_packages = list(include_packages or [])
+
+        if self._include_packages:
+            self.autodiscover_handlers(self._include_packages)
+
+    def register_worker(
+        self, worker_class: type["BaseWorker"], name: Optional[str] = None
+    ) -> None:
+        """
+        Register a aq_worker class.
+
+        Args:
+            worker_class: Worker class to register
+            name: Optional custom name for the aq_worker
+        """
+        self.worker_registry.register(worker_class, name)
+        # Clear cached workers when new aq_worker is registered
+        self._workers = {}
+
+    def register_handler(self, handler_class: type["BaseHandler"]) -> None:
+        """
+        Register a handler class.
+
+        Args:
+            handler_class: Handler class to register
+        """
+        self.handler_registry.register(handler_class)
+
+    def listen(self, job_service: JobService) -> None:
+        """
+            Register Job Service
+        Args:
+            job_service: JobService instance
+        """
+        self.job_service = job_service
+
+    def create_worker(
+        self, worker_type: str, custom_config: Optional[Dict[str, Any]] = None
+    ) -> "BaseWorker":
+        """
+        Create a aq_worker instance.
+
+        Args:
+            worker_type: Type of aq_worker to create
+            custom_config: Custom configuration to override defaults
+
+        Returns:
+            Worker instance
+        """
+        if not self._workers:
+            # Copy snapshot of registered workers
+            self._workers = {
+                name: self.worker_registry.get(name)
+                for name in self.worker_registry.list_names()
+            }
+
+        if worker_type not in self._workers:
+            raise ValueError(
+                f"Unknown aq_worker type: {worker_type}. Available: {list(self._workers.keys())}"
+            )
+
+        # Create aq_worker instance (will use worker_config from class)
+        worker_class = self._workers[worker_type]
+        if worker_class is None:
+            raise ValueError(
+                f"Unknown aq_worker type: {worker_type}. Available: {list(self._workers.keys())}"
+            )
+        return worker_class(
+            custom_config,
+            handler_dispatcher=self.handler_dispatcher,
+            job_service=self.job_service,
+        )
+
+    def get_available_workers(self) -> list[str]:
+        """Get list of available aq_worker types."""
+        if not self._workers:
+            self._workers = {
+                name: self.worker_registry.get(name)
+                for name in self.worker_registry.list_names()
+            }
+        return list(self._workers.keys())
+
+    def handler(self, name: Optional[str] = None) -> Callable[..., Any]:
+        """
+        Decorator to convert a function into a handler and automatically register it.
+
+        Usage:
+            aq_worker = AQWorker()
+
+            @aq_worker.handler(name='send_email')
+            async def send_email(data: dict) -> bool:
+                # Handler logic
+                return True
+
+            # Or with default name (function name)
+            @aq_worker.handler()
+            def process_data(data: dict) -> bool:
+                return True
+
+        Args:
+            name: Optional handler name. If not provided, uses function name.
+
+        Returns:
+            The original function (decorator registers handler automatically)
+        """
+
+        def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
+            # Determine if function is async
+            is_async = inspect.iscoroutinefunction(func)
+
+            # Get handler name
+            handler_name = name or func.__name__
+
+            # Create handler class dynamically
+            class FunctionHandler(BaseHandler):
+                """Handler class created from function decorator."""
+
+                name = handler_name
+
+                async def handle(self, data: Dict[str, Any]) -> bool:
+                    """Process job data by calling the original function."""
+                    if is_async:
+                        return bool(await func(data))
+                    else:
+                        return bool(func(data))
+
+            # Set class name for better debugging
+            FunctionHandler.__name__ = f"{func.__name__}Handler"
+            FunctionHandler.__qualname__ = f"{func.__qualname__}Handler"
+
+            # Preserve original function for reference
+            FunctionHandler._original_func = func  # type: ignore[attr-defined]
+
+            # Automatically register handler
+            self.register_handler(FunctionHandler)
+
+            # Return original function so it can still be called directly if needed
+            return func
+
+        return decorator
+
+    def autodiscover_handlers(self, packages: Iterable[str]) -> None:
+        """
+        Import every module inside the provided packages so handler decorators run.
+        """
+        before_snapshot = set(self.handler_registry.snapshot().keys())
+        visited: set[str] = set()
+        for package_name in packages:
+            self._import_package(package_name, visited)
+
+        current_handlers = sorted(self.handler_registry.snapshot().keys())
+        if current_handlers:
+            logger.info(
+                "Available handlers after discovery: %s",
+                ", ".join(current_handlers),
+            )
+        elif before_snapshot:
+            logger.info("Handler registry unchanged during discovery.")
+        else:
+            logger.warning("No handlers registered after discovery.")
+
+    def _import_package(self, module_name: str, visited: set[str]) -> None:
+        if not module_name or module_name in visited:
+            return
+
+        visited.add(module_name)
+
+        try:
+            module = importlib.import_module(module_name)
+        except ModuleNotFoundError as exc:
+            logger.warning("Include package '%s' not found: %s", module_name, exc)
+            return
+        except Exception as exc:
+            logger.exception(
+                "Failed to import include package '%s': %s", module_name, exc
+            )
+            return
+
+        self._register_module_handlers(module)
+
+        module_path = getattr(module, "__path__", None)
+        if not module_path:
+            return
+
+        for _finder, name, _ispkg in pkgutil.walk_packages(
+            module_path, prefix=f"{module.__name__}."
+        ):
+            self._import_package(name, visited)
+
+    def _register_module_handlers(self, module) -> None:
+        for attribute_name in dir(module):
+            attribute = getattr(module, attribute_name, None)
+            if (
+                inspect.isclass(attribute)
+                and issubclass(attribute, BaseHandler)
+                and attribute is not BaseHandler
+            ):
+                try:
+                    self.register_handler(attribute)
+                except ValueError:
+                    # Skip duplicates with informative log for debugging
+                    logger.debug(
+                        "Handler '%s' already registered", attribute.get_name()
+                    )

aqworker/handler/__init__.py

@@ -0,0 +1,7 @@
+from .base import BaseHandler
+from .registry import HandlerRegistry
+
+__all__ = [
+    "BaseHandler",
+    "HandlerRegistry",
+]

aqworker/handler/base.py

@@ -0,0 +1,24 @@
+from abc import ABC
+from typing import Any, Dict, Optional
+
+
+class BaseHandler(ABC):
+    """Abstract base for job handlers."""
+
+    name: Optional[str] = None
+
+    @classmethod
+    def get_name(cls) -> str:
+        return cls.name or cls.__name__
+
+    async def handle(self, data: Dict[str, Any]) -> bool:
+        """
+        Process job data.
+
+        Args:
+            data: Job data dictionary containing all information needed for processing
+
+        Returns:
+            True if processing succeeded, False otherwise
+        """
+        raise NotImplementedError