django-cfg 1.1.61__py3-none-any.whl → 1.1.62__py3-none-any.whl

django_cfg/__init__.py

@@ -38,7 +38,7 @@ default_app_config = "django_cfg.apps.DjangoCfgConfig"
 from typing import TYPE_CHECKING
 
 # Version information
-__version__ = "1.1.60"
+__version__ = "1.1.62"
 __author__ = "Unrealos Team"
 __email__ = "info@unrealos.com"
 __license__ = "MIT"

django_cfg/management/commands/rundramatiq.py

@@ -1,263 +1,235 @@
 """
 Django management command for running Dramatiq workers.
 
-This command provides a Django-integrated way to start Dramatiq workers
-with configuration from Django-CFG TaskConfig.
+Based on django_dramatiq.management.commands.rundramatiq with Django-CFG integration.
+Simple, clean, and working approach.
 """
 
-from django.core.management.base import BaseCommand, CommandError
-from django.conf import settings
-from typing import Any, Optional, List
-import logging
-import sys
+import argparse
+import importlib
+import multiprocessing
 import os
+import sys
+
+from django.apps import apps
+from django.conf import settings
+from django.core.management.base import BaseCommand
+from django.utils.module_loading import module_has_submodule
 
 from django_cfg.modules.django_tasks import get_task_service
 
-logger = logging.getLogger(__name__)
+
+# Default values
+NPROCS = multiprocessing.cpu_count()
+NTHREADS = 8
 
 
 class Command(BaseCommand):
-    """
-    Run Dramatiq workers with Django-CFG configuration.
-    
-    This command starts Dramatiq workers using the configuration
-    defined in Django-CFG TaskConfig, with support for custom
-    process counts, queue selection, and worker options.
-    """
-    
-    help = "Run Dramatiq workers for background task processing"
-    
+    help = "Run Dramatiq workers with Django-CFG configuration."
+
     def add_arguments(self, parser):
-        """Add command line arguments."""
+        parser.formatter_class = argparse.ArgumentDefaultsHelpFormatter
+        
         parser.add_argument(
-            "--processes",
+            "--processes", "-p",
+            default=NPROCS,
             type=int,
-            help="Number of worker processes (overrides config)",
+            help="The number of processes to run",
         )
         parser.add_argument(
-            "--threads", 
+            "--threads", "-t",
+            default=NTHREADS,
             type=int,
-            help="Number of threads per process (overrides config)",
+            help="The number of threads per process to use",
         )
         parser.add_argument(
-            "--queues",
+            "--queues", "-Q",
+            nargs="*",
             type=str,
-            help="Comma-separated list of queues to process (overrides config)",
+            help="Listen to a subset of queues, or all when empty",
         )
         parser.add_argument(
-            "--log-level",
-            choices=["DEBUG", "INFO", "WARNING", "ERROR"],
-            help="Log level for workers (overrides config)",
+            "--watch",
+            dest="watch_dir",
+            help="Reload workers when changes are detected in the given directory",
         )
         parser.add_argument(
-            "--watch",
+            "--pid-file",
             type=str,
-            help="Watch directory for code changes and auto-reload",
+            help="Write the PID of the master process to this file",
         )
         parser.add_argument(
-            "--pid-file",
+            "--log-file",
             type=str,
-            help="Write process ID to file",
+            help="Write all logs to a file, or stderr when empty",
+        )
+        parser.add_argument(
+            "--worker-shutdown-timeout", 
+            type=int, 
+            default=600000,
+            help="Timeout for worker shutdown, in milliseconds"
         )
         parser.add_argument(
             "--dry-run",
             action="store_true",
             help="Show configuration without starting workers",
         )
-    
-    def handle(self, *args, **options):
-        """Handle the command execution."""
-        try:
-            # Get task service
-            task_service = get_task_service()
+
+    def handle(self, watch_dir, processes, threads, verbosity, queues, 
+               pid_file, log_file, worker_shutdown_timeout, dry_run, **options):
+        
+        # Get task service and validate
+        task_service = get_task_service()
+        if not task_service.is_enabled():
+            self.stdout.write(
+                self.style.ERROR("Task system is not enabled in Django-CFG configuration")
+            )
+            return
+        
+        # Discover task modules
+        tasks_modules = self._discover_tasks_modules()
+        
+        # Show configuration info
+        self.stdout.write(self.style.SUCCESS("Dramatiq Worker Configuration:"))
+        self.stdout.write(f"Processes: {processes}")
+        self.stdout.write(f"Threads: {threads}")
+        if queues:
+            self.stdout.write(f"Queues: {', '.join(queues)}")
+        else:
+            self.stdout.write("Queues: all")
+        
+        self.stdout.write(f"\nDiscovered task modules:")
+        for module in tasks_modules:
+            self.stdout.write(f"  - {module}")
+        
+        # If dry run, show command and exit
+        if dry_run:
+            executable_name = "dramatiq"
+            process_args = [
+                executable_name,
+                "--processes", str(processes),
+                "--threads", str(threads),
+                "--worker-shutdown-timeout", str(worker_shutdown_timeout),
+            ]
             
-            # Check if task system is enabled
-            if not task_service.is_enabled():
-                raise CommandError(
-                    "Task system is not enabled. "
-                    "Please configure 'tasks' in your Django-CFG configuration."
-                )
+            if watch_dir:
+                process_args.extend(["--watch", watch_dir])
             
-            # Validate configuration
-            if not task_service.validate_configuration():
-                raise CommandError(
-                    "Task system configuration is invalid. "
-                    "Please check your Redis connection and task settings."
-                )
+            verbosity_args = ["-v"] * (verbosity - 1)
+            process_args.extend(verbosity_args)
             
-            # Get effective configuration
-            config = task_service.config
-            if not config:
-                raise CommandError("Task configuration not available")
+            if queues:
+                process_args.extend(["--queues", *queues])
             
-            # Build worker arguments
-            worker_args = self._build_worker_args(config, options)
+            if pid_file:
+                process_args.extend(["--pid-file", pid_file])
             
-            if options["dry_run"]:
-                self._show_configuration(config, worker_args)
-                return
+            if log_file:
+                process_args.extend(["--log-file", log_file])
             
-            # Start workers
-            self._start_workers(worker_args, options)
+            process_args.extend(tasks_modules)
             
-        except ImportError:
-            raise CommandError(
-                "Dramatiq dependencies not installed. "
-                "Install with: pip install django-cfg[tasks]"
-            )
-        except Exception as e:
-            logger.exception("Failed to start Dramatiq workers")
-            raise CommandError(f"Failed to start workers: {e}")
-    
-    def _build_worker_args(self, config, options) -> List[str]:
-        """Build command line arguments for Dramatiq workers."""
-        args = ["dramatiq"]
-        
-        # Process count
-        processes = options.get("processes") or config.get_effective_processes()
-        args.extend(["--processes", str(processes)])
+            self.stdout.write(f"\nCommand that would be executed:")
+            self.stdout.write(f'  {" ".join(process_args)}')
+            return
         
-        # Thread count
-        threads = options.get("threads") or config.dramatiq.threads
-        args.extend(["--threads", str(threads)])
+        # Show startup info
+        self.stdout.write(self.style.SUCCESS("\nStarting Dramatiq workers..."))
         
-        # Queues
-        if options.get("queues"):
-            queues = [q.strip() for q in options["queues"].split(",")]
-        else:
-            queues = config.get_effective_queues()
+        # Build dramatiq command
+        executable_name = "dramatiq"
+        executable_path = self._resolve_executable(executable_name)
         
-        for queue in queues:
-            args.extend(["--queues", queue])
+        # Build process arguments exactly like django_dramatiq
+        process_args = [
+            executable_name,
+            "--processes", str(processes),
+            "--threads", str(threads),
+            "--worker-shutdown-timeout", str(worker_shutdown_timeout),
+        ]
         
-        # Log level (not supported by standard dramatiq CLI)
-        # log_level = options.get("log_level") or config.worker.log_level
-        # args.extend(["--log-level", log_level])
+        # Add watch directory if specified
+        if watch_dir:
+            process_args.extend(["--watch", watch_dir])
         
-        # Watch directory
-        if options.get("watch"):
-            args.extend(["--watch", options["watch"]])
+        # Add verbosity
+        verbosity_args = ["-v"] * (verbosity - 1)
+        process_args.extend(verbosity_args)
         
-        # PID file
-        if options.get("pid_file"):
-            args.extend(["--pid-file", options["pid_file"]])
+        # Add queues if specified
+        if queues:
+            process_args.extend(["--queues", *queues])
         
-        # Note: Using Python API instead of CLI, so no broker argument needed
+        # Add PID file if specified
+        if pid_file:
+            process_args.extend(["--pid-file", pid_file])
         
-        # Add discovered task modules
-        discovered_modules = get_task_service().discover_tasks()
-        for module in discovered_modules:
-            args.append(module)
+        # Add log file if specified
+        if log_file:
+            process_args.extend(["--log-file", log_file])
         
-        # If no modules discovered, add default patterns
-        if not discovered_modules:
-            # Add common task module patterns
-            for app in settings.INSTALLED_APPS:
-                if not app.startswith("django.") and not app.startswith("django_"):
-                    args.append(f"{app}.tasks")
+        # Add task modules (broker module first, then discovered modules)
+        process_args.extend(tasks_modules)
         
-        return args
-    
-    def _show_configuration(self, config, worker_args):
-        """Show worker configuration without starting."""
-        self.stdout.write(
-            self.style.SUCCESS("Dramatiq Worker Configuration:")
-        )
+        self.stdout.write(f'Running dramatiq: "{" ".join(process_args)}"\n')
         
-        self.stdout.write(f"  Processes: {config.get_effective_processes()}")
-        self.stdout.write(f"  Threads: {config.dramatiq.threads}")
-        self.stdout.write(f"  Queues: {', '.join(config.get_effective_queues())}")
-        self.stdout.write(f"  Log Level: {config.worker.log_level}")
-        self.stdout.write(f"  Redis DB: {config.dramatiq.redis_db}")
-        
-        self.stdout.write("\nDiscovered task modules:")
-        discovered = get_task_service().discover_tasks()
-        if discovered:
-            for module in discovered:
-                self.stdout.write(f"  - {module}")
-        else:
-            self.stdout.write("  (none found)")
+        # Ensure DJANGO_SETTINGS_MODULE is set for worker processes
+        if not os.environ.get('DJANGO_SETTINGS_MODULE'):
+            if hasattr(settings, 'SETTINGS_MODULE'):
+                os.environ['DJANGO_SETTINGS_MODULE'] = settings.SETTINGS_MODULE
+            else:
+                # Try to detect from manage.py or current settings
+                import django
+                from django.conf import settings as django_settings
+                if hasattr(django_settings, '_wrapped') and hasattr(django_settings._wrapped, '__module__'):
+                    module_name = django_settings._wrapped.__module__
+                    os.environ['DJANGO_SETTINGS_MODULE'] = module_name
+                else:
+                    self.stdout.write(
+                        self.style.WARNING("Could not detect DJANGO_SETTINGS_MODULE")
+                    )
+        
+        # Use os.execvp like django_dramatiq to preserve environment
+        if sys.platform == "win32":
+            import subprocess
+            command = [executable_path] + process_args[1:]
+            sys.exit(subprocess.run(command))
         
-        self.stdout.write(f"\nCommand that would be executed:")
-        self.stdout.write(f"  {' '.join(worker_args)}")
-    
-    def _start_workers(self, worker_args, options):
-        """Start Dramatiq workers using subprocess like django_dramatiq does."""
-        self.stdout.write(
-            self.style.SUCCESS("Starting Dramatiq workers...")
-        )
+        os.execvp(executable_path, process_args)
+
+    def _discover_tasks_modules(self):
+        """Discover task modules like django_dramatiq does."""
+        # Always include our broker setup module first
+        tasks_modules = ["django_cfg.modules.dramatiq_setup"]
         
-        # Show startup info
+        # Get task service for configuration
         task_service = get_task_service()
-        config = task_service.config
-        
-        processes = options.get("processes") or config.get_effective_processes()
-        threads = options.get("threads") or config.dramatiq.threads
-        queues = config.get_effective_queues()
-        if options.get("queues"):
-            queues = [q.strip() for q in options["queues"].split(",")]
-        
-        self.stdout.write(f"Processes: {processes}")
-        self.stdout.write(f"Threads: {threads}")
-        self.stdout.write(f"Queues: {', '.join(queues)}")
         
-        # Write PID file if requested
-        if options.get("pid_file"):
-            with open(options["pid_file"], "w") as f:
-                f.write(str(os.getpid()))
-            self.stdout.write(f"PID written to: {options['pid_file']}")
+        # Try to get task modules from Django-CFG config
+        if task_service.config and task_service.config.auto_discover_tasks:
+            discovered = task_service.discover_tasks()
+            for module_name in discovered:
+                self.stdout.write(f"Discovered tasks module: '{module_name}'")
+                tasks_modules.append(module_name)
         
-        try:
-            # Build process arguments like django_dramatiq does
-            executable_name = "dramatiq"
-            executable_path = self._resolve_executable(executable_name)
-            
-            # Discover task modules (including django_dramatiq.setup)
-            discovered_modules = self._discover_task_modules()
+        # Fallback: use django_dramatiq discovery logic
+        if len(tasks_modules) == 1:  # Only broker module found
+            task_module_names = getattr(settings, "DRAMATIQ_AUTODISCOVER_MODULES", ("tasks",))
             
-            process_args = [
-                executable_name,
-                "--processes", str(processes),
-                "--threads", str(threads),
-            ]
-            
-            # Add queues
-            if queues:
-                process_args.extend(["--queues"] + queues)
-            
-            # Add task modules (including django_dramatiq.setup)
-            process_args.extend(discovered_modules)
-            
-            self.stdout.write(f"Running dramatiq: {' '.join(process_args)}")
-            
-            # Use subprocess like django_dramatiq does
-            import subprocess
-            if sys.platform == "win32":
-                command = [executable_path] + process_args[1:]
-                sys.exit(subprocess.run(command))
-            else:
-                # Use os.execvp like django_dramatiq
-                os.execvp(executable_path, process_args)
-                
-        except KeyboardInterrupt:
-            self.stdout.write("\nShutting down workers...")
-        except Exception as e:
-            logger.exception("Worker execution failed")
-            raise CommandError(f"Worker execution failed: {e}")
-    
-    def _discover_task_modules(self):
-        """Discover task modules like django_dramatiq does."""
-        task_modules = ["django_dramatiq.setup"]  # Always include setup module
-        
-        # Add discovered modules from django-cfg
-        discovered_modules = get_task_service().discover_tasks()
-        for module_name in discovered_modules:
-            self.stdout.write(f"Discovered tasks module: {module_name}")
-            task_modules.append(module_name)
-        
-        return task_modules
-    
+            for app_config in apps.get_app_configs():
+                for task_module in task_module_names:
+                    if module_has_submodule(app_config.module, task_module):
+                        module_name = f"{app_config.name}.{task_module}"
+                        try:
+                            importlib.import_module(module_name)
+                            self.stdout.write(f"Discovered tasks module: '{module_name}'")
+                            tasks_modules.append(module_name)
+                        except ImportError:
+                            # Module exists but has import errors, skip it
+                            pass
+        
+        return tasks_modules
+
     def _resolve_executable(self, exec_name):
         """Resolve executable path like django_dramatiq does."""
         bin_dir = os.path.dirname(sys.executable)
@@ -266,4 +238,4 @@ class Command(BaseCommand):
                 exec_path = os.path.join(d, exec_name)
                 if os.path.isfile(exec_path):
                     return exec_path
-        return exec_name
+        return exec_name