django-cfg 1.4.21__py3-none-any.whl → 1.4.24__py3-none-any.whl

django_cfg/modules/django_tasks/management/commands/rundramatiq.py

@@ -15,10 +15,10 @@ from django.apps import apps
 from django.conf import settings
 from django.core.management.base import BaseCommand
 from django.utils.module_loading import module_has_submodule
+
 from django_cfg.modules.django_logging import get_logger
 from django_cfg.modules.django_tasks import get_task_service
 
-
 # Default values
 NPROCS = multiprocessing.cpu_count()
 NTHREADS = 8
@@ -37,7 +37,7 @@ class Command(BaseCommand):
 
     def add_arguments(self, parser):
         parser.formatter_class = argparse.ArgumentDefaultsHelpFormatter
-        
+
         parser.add_argument(
             "--processes", "-p",
             default=NPROCS,
@@ -72,8 +72,8 @@ class Command(BaseCommand):
             help="Write all logs to a file, or stderr when empty",
         )
         parser.add_argument(
-            "--worker-shutdown-timeout", 
-            type=int, 
+            "--worker-shutdown-timeout",
+            type=int,
             default=600000,
             help="Timeout for worker shutdown, in milliseconds"
         )
@@ -83,10 +83,10 @@ class Command(BaseCommand):
             help="Show configuration without starting workers",
         )
 
-    def handle(self, watch_dir, processes, threads, verbosity, queues, 
+    def handle(self, watch_dir, processes, threads, verbosity, queues,
                pid_file, log_file, worker_shutdown_timeout, dry_run, **options):
         logger.info("Starting rundramatiq command")
-        
+
         # Get task service and validate
         task_service = get_task_service()
         if not task_service.is_enabled():
@@ -94,10 +94,10 @@ class Command(BaseCommand):
                 self.style.ERROR("Task system is not enabled in Django-CFG configuration")
             )
             return
-        
+
         # Discover task modules
         tasks_modules = self._discover_tasks_modules()
-        
+
         # Show configuration info
         self.stdout.write(self.style.SUCCESS("Dramatiq Worker Configuration:"))
         self.stdout.write(f"Processes: {processes}")
@@ -106,15 +106,15 @@ class Command(BaseCommand):
             self.stdout.write(f"Queues: {', '.join(queues)}")
         else:
             self.stdout.write("Queues: all")
-        
-        self.stdout.write(f"\nDiscovered task modules:")
+
+        self.stdout.write("\nDiscovered task modules:")
         for module in tasks_modules:
             self.stdout.write(f"  - {module}")
-        
+
         # If dry run, show command and exit
         if dry_run:
             executable_name = "dramatiq"
-            
+
             process_args = [
                 executable_name,
                 "django_cfg.modules.django_tasks.dramatiq_setup",  # Broker module
@@ -122,36 +122,36 @@ class Command(BaseCommand):
                 "--threads", str(threads),
                 "--worker-shutdown-timeout", str(worker_shutdown_timeout),
             ]
-            
+
             if watch_dir:
                 process_args.extend(["--watch", watch_dir])
-            
+
             verbosity_args = ["-v"] * (verbosity - 1)
             process_args.extend(verbosity_args)
-            
+
             if queues:
                 process_args.extend(["--queues"] + queues)
-            
+
             if pid_file:
                 process_args.extend(["--pid-file", pid_file])
-            
+
             if log_file:
                 process_args.extend(["--log-file", log_file])
-            
+
             # Add task modules (broker module is already first in tasks_modules)
             process_args.extend(tasks_modules)
-            
-            self.stdout.write(f"\nCommand that would be executed:")
+
+            self.stdout.write("\nCommand that would be executed:")
             self.stdout.write(f'  {" ".join(process_args)}')
             return
-        
+
         # Show startup info
         self.stdout.write(self.style.SUCCESS("\nStarting Dramatiq workers..."))
-        
+
         # Build dramatiq command
         executable_name = "dramatiq"
         executable_path = self._resolve_executable(executable_name)
-        
+
         # Build process arguments exactly like django_dramatiq
         process_args = [
             executable_name,
@@ -160,39 +160,38 @@ class Command(BaseCommand):
             "--threads", str(threads),
             "--worker-shutdown-timeout", str(worker_shutdown_timeout),
         ]
-        
+
         # Add watch directory if specified
         if watch_dir:
             process_args.extend(["--watch", watch_dir])
-        
+
         # Add verbosity
         verbosity_args = ["-v"] * (verbosity - 1)
         process_args.extend(verbosity_args)
-        
+
         # Add queues if specified
         if queues:
             process_args.extend(["--queues"] + queues)
-        
+
         # Add PID file if specified
         if pid_file:
             process_args.extend(["--pid-file", pid_file])
-        
+
         # Add log file if specified
         if log_file:
             process_args.extend(["--log-file", log_file])
-        
+
         # Add task modules (broker module is already first in tasks_modules)
         process_args.extend(tasks_modules)
-        
+
         self.stdout.write(f'Running dramatiq: "{" ".join(process_args)}"\n')
-        
+
         # Ensure DJANGO_SETTINGS_MODULE is set for worker processes
         if not os.environ.get('DJANGO_SETTINGS_MODULE'):
             if hasattr(settings, 'SETTINGS_MODULE'):
                 os.environ['DJANGO_SETTINGS_MODULE'] = settings.SETTINGS_MODULE
             else:
                 # Try to detect from manage.py or current settings
-                import django
                 from django.conf import settings as django_settings
                 if hasattr(django_settings, '_wrapped') and hasattr(django_settings._wrapped, '__module__'):
                     module_name = django_settings._wrapped.__module__
@@ -201,34 +200,34 @@ class Command(BaseCommand):
                     self.stdout.write(
                         self.style.WARNING("Could not detect DJANGO_SETTINGS_MODULE")
                     )
-        
+
         # Use os.execvp like django_dramatiq to preserve environment
         if sys.platform == "win32":
             import subprocess
             command = [executable_path] + process_args[1:]
             sys.exit(subprocess.run(command))
-        
+
         os.execvp(executable_path, process_args)
 
     def _discover_tasks_modules(self):
         """Discover task modules like django_dramatiq does."""
         # Always include our broker setup module first
         tasks_modules = ["django_cfg.modules.django_tasks.dramatiq_setup"]
-        
+
         # Get task service for configuration
         task_service = get_task_service()
-        
+
         # Try to get task modules from Django-CFG config
         if task_service.config and task_service.config.auto_discover_tasks:
             discovered = task_service.discover_tasks()
             for module_name in discovered:
                 self.stdout.write(f"Discovered tasks module: '{module_name}'")
                 tasks_modules.append(module_name)
-        
+
         # Fallback: use django_dramatiq discovery logic
         if len(tasks_modules) == 1:  # Only broker module found
             task_module_names = getattr(settings, "DRAMATIQ_AUTODISCOVER_MODULES", ("tasks",))
-            
+
             for app_config in apps.get_app_configs():
                 for task_module in task_module_names:
                     if module_has_submodule(app_config.module, task_module):
@@ -240,7 +239,7 @@ class Command(BaseCommand):
                         except ImportError:
                             # Module exists but has import errors, skip it
                             pass
-        
+
         return tasks_modules
 
     def _resolve_executable(self, exec_name):
@@ -251,4 +250,4 @@ class Command(BaseCommand):
                 exec_path = os.path.join(d, exec_name)
                 if os.path.isfile(exec_path):
                     return exec_path
-        return exec_name
+        return exec_name

django_cfg/modules/django_tasks/management/commands/rundramatiq_simulator.py

@@ -8,54 +8,53 @@ Usage:
     python manage.py rundramatiq_simulator --show-keys
 """
 
-import redis
 import json
+import random
 import time
 from datetime import datetime, timezone
-import random
-from typing import Dict, Any, Optional
+from typing import Any, Dict
 
+import redis
 from django.core.management.base import BaseCommand, CommandError
-from django.conf import settings
 
 from django_cfg.modules.django_tasks import DjangoTasks
 
 
 class TaskSimulator:
     """Task data simulator for Tasks Dashboard."""
-    
+
     def __init__(self):
         """Initialize the simulator."""
         self.tasks_service = DjangoTasks()
-        
+
         # Get Redis client using the same logic as DjangoTasks
         try:
             redis_url = self.tasks_service.get_redis_url()
             if not redis_url:
                 raise RuntimeError("No Redis URL available")
-            
+
             # Parse URL for connection
             from urllib.parse import urlparse
             parsed = urlparse(redis_url)
-            
+
             self.redis_client = redis.Redis(
                 host=parsed.hostname or 'localhost',
                 port=parsed.port or 6379,
                 db=int(parsed.path.lstrip('/')) if parsed.path else 1,
                 decode_responses=True
             )
-                
+
         except Exception as e:
             raise CommandError(f"Failed to connect to Redis: {e}")
-        
+
         # Get queue configuration
         try:
             config = self.tasks_service.get_config()
             self.queues = config.tasks.dramatiq.queues
-        except Exception as e:
+        except Exception:
             # Use default queues if we can't get configuration
             self.queues = ['critical', 'high', 'default', 'low', 'background', 'payments', 'agents', 'knowbase']
-    
+
     def clear_all_data(self) -> int:
         """
         Clear all test data.
@@ -68,7 +67,7 @@ class TaskSimulator:
             deleted = self.redis_client.delete(*keys)
             return deleted
         return 0
-    
+
     def simulate_queues(self, pending_tasks_per_queue=None, failed_tasks_per_queue=None) -> Dict[str, Dict[str, int]]:
         """
         Simulate queues with tasks.
@@ -91,7 +90,7 @@ class TaskSimulator:
                 'agents': 7,
                 'knowbase': 4
             }
-        
+
         if failed_tasks_per_queue is None:
             failed_tasks_per_queue = {
                 'critical': 0,
@@ -103,17 +102,17 @@ class TaskSimulator:
                 'agents': 1,
                 'knowbase': 0
             }
-        
+
         results = {}
-        
+
         for queue_name in self.queues:
             queue_results = {'pending': 0, 'failed': 0}
-            
+
             # Pending tasks
             pending_count = pending_tasks_per_queue.get(queue_name, 0)
             if pending_count > 0:
                 queue_key = f"dramatiq:default.DQ.{queue_name}"
-                
+
                 # Add fake tasks to queue
                 for i in range(pending_count):
                     task_data = {
@@ -126,14 +125,14 @@ class TaskSimulator:
                         "message_timestamp": int(time.time() * 1000)
                     }
                     self.redis_client.lpush(queue_key, json.dumps(task_data))
-                
+
                 queue_results['pending'] = pending_count
-            
+
             # Failed tasks
             failed_count = failed_tasks_per_queue.get(queue_name, 0)
             if failed_count > 0:
                 failed_key = f"dramatiq:default.DQ.{queue_name}.failed"
-                
+
                 # Add fake failed tasks
                 for i in range(failed_count):
                     failed_task_data = {
@@ -147,14 +146,14 @@ class TaskSimulator:
                         "error": f"Simulated error for {queue_name} task {i}"
                     }
                     self.redis_client.lpush(failed_key, json.dumps(failed_task_data))
-                
+
                 queue_results['failed'] = failed_count
-            
+
             if queue_results['pending'] > 0 or queue_results['failed'] > 0:
                 results[queue_name] = queue_results
-        
+
         return results
-    
+
     def simulate_workers(self, worker_count=3) -> list:
         """
         Симулировать активных воркеров.
@@ -166,32 +165,32 @@ class TaskSimulator:
             Список ID созданных воркеров
         """
         worker_ids = []
-        
+
         for i in range(worker_count):
             worker_id = f"worker_{i}_{int(time.time())}"
             worker_key = f"dramatiq:worker:{worker_id}"
-            
+
             worker_data = {
                 "worker_id": worker_id,
-                "hostname": f"localhost",
+                "hostname": "localhost",
                 "pid": 1000 + i,
                 "queues": self.queues,
                 "started_at": datetime.now(timezone.utc).isoformat(),
                 "last_heartbeat": datetime.now(timezone.utc).isoformat(),
                 "status": "active"
             }
-            
+
             # Устанавливаем данные воркера с TTL
             self.redis_client.setex(
-                worker_key, 
+                worker_key,
                 300,  # 5 минут TTL
                 json.dumps(worker_data)
             )
-            
+
             worker_ids.append(worker_id)
-        
+
         return worker_ids
-    
+
     def simulate_task_statistics(self) -> Dict[str, Any]:
         """
         Симулировать статистику задач.
@@ -206,12 +205,12 @@ class TaskSimulator:
             "processing_time_avg": round(random.uniform(1.5, 4.0), 2),
             "last_updated": datetime.now(timezone.utc).isoformat()
         }
-        
+
         stats_key = "dramatiq:stats"
         self.redis_client.setex(stats_key, 3600, json.dumps(stats_data))
-        
+
         return stats_data
-    
+
     def run_simulation(self, workers=3, clear_first=True) -> Dict[str, Any]:
         """
         Запустить полную симуляцию.
@@ -229,16 +228,16 @@ class TaskSimulator:
             'workers': [],
             'statistics': {}
         }
-        
+
         if clear_first:
             results['cleared_keys'] = self.clear_all_data()
-        
+
         results['queues'] = self.simulate_queues()
         results['workers'] = self.simulate_workers(workers)
         results['statistics'] = self.simulate_task_statistics()
-        
+
         return results
-    
+
     def get_redis_summary(self) -> Dict[str, Any]:
         """Получить сводку по данным в Redis."""
         summary = {
@@ -247,29 +246,29 @@ class TaskSimulator:
             'workers': 0,
             'statistics': None
         }
-        
+
         # Подсчитываем все ключи
         all_keys = self.redis_client.keys("dramatiq:*")
         summary['total_keys'] = len(all_keys)
-        
+
         # Анализируем очереди
         for queue_name in self.queues:
             pending_key = f"dramatiq:default.DQ.{queue_name}"
             failed_key = f"dramatiq:default.DQ.{queue_name}.failed"
-            
+
             pending = self.redis_client.llen(pending_key)
             failed = self.redis_client.llen(failed_key)
-            
+
             if pending > 0 or failed > 0:
                 summary['queues'][queue_name] = {
                     'pending': pending,
                     'failed': failed
                 }
-        
+
         # Подсчитываем воркеров
         worker_keys = self.redis_client.keys("dramatiq:worker:*")
         summary['workers'] = len(worker_keys)
-        
+
         # Получаем статистику
         stats_key = "dramatiq:stats"
         if self.redis_client.exists(stats_key):
@@ -278,7 +277,7 @@ class TaskSimulator:
                 summary['statistics'] = json.loads(stats_data)
             except:
                 pass
-        
+
         return summary
 
 
@@ -291,7 +290,7 @@ class Command(BaseCommand):
     is_destructive = False
 
     help = 'Simulate Dramatiq tasks and workers for dashboard testing'
-    
+
     def add_arguments(self, parser):
         """Добавить аргументы команды."""
         parser.add_argument(
@@ -300,41 +299,41 @@ class Command(BaseCommand):
             default=3,
             help='Number of workers to simulate (default: 3)'
         )
-        
+
         parser.add_argument(
             '--no-clear',
             action='store_true',
             help='Do not clear existing data before simulation'
         )
-        
+
         parser.add_argument(
             '--clear-only',
             action='store_true',
             help='Only clear data, do not simulate'
         )
-        
+
         parser.add_argument(
             '--show-keys',
             action='store_true',
             help='Show Redis keys after operation'
         )
-        
+
         parser.add_argument(
             '--summary',
             action='store_true',
             help='Show summary of current Redis data'
         )
-    
+
     def handle(self, *args, **options):
         """Выполнить команду."""
         try:
             simulator = TaskSimulator()
-            
+
             # Показать только сводку
             if options['summary']:
                 self.show_summary(simulator)
                 return
-            
+
             # Только очистка
             if options['clear_only']:
                 self.stdout.write("🧹 Clearing all test data...")
@@ -343,87 +342,87 @@ class Command(BaseCommand):
                     self.style.SUCCESS(f"✅ Cleared {cleared} Redis keys")
                 )
                 return
-            
+
             # Полная симуляция
             self.stdout.write("🎭 Starting Dramatiq Task Simulation")
             self.stdout.write("=" * 50)
-            
+
             results = simulator.run_simulation(
                 workers=options['workers'],
                 clear_first=not options['no_clear']
             )
-            
+
             # Показываем результаты
             if results['cleared_keys'] > 0:
                 self.stdout.write(f"🧹 Cleared {results['cleared_keys']} existing keys")
-            
-            self.stdout.write(f"📋 Created queues:")
+
+            self.stdout.write("📋 Created queues:")
             total_pending = 0
             total_failed = 0
-            
+
             for queue_name, counts in results['queues'].items():
                 pending = counts['pending']
                 failed = counts['failed']
                 total_pending += pending
                 total_failed += failed
-                
+
                 self.stdout.write(f"   {queue_name}: {pending} pending, {failed} failed")
-            
+
             self.stdout.write(f"👷 Created {len(results['workers'])} workers")
-            self.stdout.write(f"📊 Added task statistics")
-            
+            self.stdout.write("📊 Added task statistics")
+
             self.stdout.write("=" * 50)
             self.stdout.write(self.style.SUCCESS("✅ Simulation completed!"))
-            
-            self.stdout.write(f"\n📊 Summary:")
+
+            self.stdout.write("\n📊 Summary:")
             active_queues = len(results['queues'])
             self.stdout.write(f"   Active Queues: {active_queues}")
             self.stdout.write(f"   Active Workers: {len(results['workers'])}")
             self.stdout.write(f"   Pending Tasks: {total_pending}")
             self.stdout.write(f"   Failed Tasks: {total_failed}")
-            
-            self.stdout.write(f"\n🌐 Dashboard URL: http://localhost:8000/cfg/admin/django_cfg_tasks/admin/dashboard/")
-            
+
+            self.stdout.write("\n🌐 Dashboard URL: http://localhost:8000/cfg/admin/django_cfg_tasks/admin/dashboard/")
+
             # Показать ключи если запрошено
             if options['show_keys']:
                 self.show_redis_keys(simulator)
-                
+
         except Exception as e:
             raise CommandError(f"Simulation failed: {e}")
-    
+
     def show_summary(self, simulator: TaskSimulator):
         """Показать сводку текущих данных."""
         self.stdout.write("📊 Current Redis Data Summary")
         self.stdout.write("=" * 40)
-        
+
         summary = simulator.get_redis_summary()
-        
+
         self.stdout.write(f"Total Redis keys: {summary['total_keys']}")
         self.stdout.write(f"Active workers: {summary['workers']}")
-        
+
         if summary['queues']:
             self.stdout.write("\nQueues:")
             for queue_name, counts in summary['queues'].items():
                 self.stdout.write(f"   {queue_name}: {counts['pending']} pending, {counts['failed']} failed")
         else:
             self.stdout.write("\nNo active queues found")
-        
+
         if summary['statistics']:
             stats = summary['statistics']
-            self.stdout.write(f"\nStatistics:")
+            self.stdout.write("\nStatistics:")
             self.stdout.write(f"   Total processed: {stats.get('total_processed', 'N/A')}")
             self.stdout.write(f"   Total failed: {stats.get('total_failed', 'N/A')}")
             self.stdout.write(f"   Avg processing time: {stats.get('processing_time_avg', 'N/A')}s")
-    
+
     def show_redis_keys(self, simulator: TaskSimulator):
         """Показать все Redis ключи."""
         self.stdout.write("\n🔍 Redis Keys:")
         keys = simulator.redis_client.keys("dramatiq:*")
-        
+
         if not keys:
             self.stdout.write("   No Dramatiq keys found")
             return
-        
+
         for key in sorted(keys):
             key_type = simulator.redis_client.type(key)
             if key_type == 'list':