qBitrr2 5.6.2__py3-none-any.whl → 5.7.1__py3-none-any.whl

qBitrr/main.py

@@ -86,6 +86,7 @@ class qBitManager:
     def __init__(self):
         self._name = "Manager"
         self.shutdown_event = Event()
+        self.database_restart_event = Event()  # Signal for coordinated database recovery restart
         self.qBit_Host = CONFIG.get("qBit.Host", fallback="localhost")
         self.qBit_Port = CONFIG.get("qBit.Port", fallback=8105)
         self.qBit_UserName = CONFIG.get("qBit.UserName", fallback=None)
@@ -102,6 +103,11 @@ class qBitManager:
         self._validated_version = False
         self.client = None
         self.current_qbit_version = None
+        # Multi-instance support
+        self.clients: dict[str, qbittorrentapi.Client] = {}
+        self.qbit_versions: dict[str, VersionClass] = {}
+        self.instance_metadata: dict[str, dict] = {}
+        self.instance_health: dict[str, bool] = {}
         if not (QBIT_DISABLED or SEARCH_ONLY):
             self.client = qbittorrentapi.Client(
                 host=self.qBit_Host,
@@ -120,6 +126,15 @@ class qBitManager:
                     e,
                 )
             self._version_validator()
+            # Register default instance in multi-instance dictionaries
+            self.clients["default"] = self.client
+            self.qbit_versions["default"] = self.current_qbit_version
+            self.instance_metadata["default"] = {
+                "host": self.qBit_Host,
+                "port": self.qBit_Port,
+                "username": self.qBit_UserName,
+            }
+            self.instance_health["default"] = self._validated_version
         self.expiring_bool = ExpiringSet(max_age_seconds=10)
         self.cache = {}
         self.name_cache = {}
@@ -137,6 +152,8 @@ class qBitManager:
         self._process_restart_counts: dict[tuple[str, str], list[float]] = (
             {}
         )  # (category, role) -> [timestamps]
+        self._failed_spawn_attempts: dict[tuple[str, str], int] = {}  # Track failed spawn attempts
+        self._pending_spawns: list[tuple] = []  # (arr_instance, meta) tuples to retry
         self.auto_restart_enabled = CONFIG.get("Settings.AutoRestartProcesses", fallback=True)
         self.max_process_restarts = CONFIG.get("Settings.MaxProcessRestarts", fallback=5)
         self.process_restart_window = CONFIG.get("Settings.ProcessRestartWindow", fallback=300)
@@ -383,6 +400,8 @@ class qBitManager:
     def _complete_startup(self) -> None:
         started_at = monotonic()
         try:
+            # Initialize all qBit instances before Arr managers
+            self._initialize_qbit_instances()
             arr_manager = ArrManager(self)
             self.arr_manager = arr_manager
             arr_manager.build_arr_instances()
@@ -427,10 +446,195 @@ class qBitManager:
             )
             sys.exit(1)
 
+    def _initialize_qbit_instances(self) -> None:
+        """
+        Initialize all qBittorrent instances from config.
+
+        Scans config for [qBit] and [qBit-XXX] sections, initializes clients,
+        and populates multi-instance dictionaries. The default [qBit] section
+        is registered as "default" instance.
+        """
+        if QBIT_DISABLED or SEARCH_ONLY:
+            self.logger.debug("qBit disabled or search-only mode; skipping instance init")
+            return
+
+        # Default instance already initialized in __init__
+        self.logger.info("Initialized qBit instance: default")
+
+        # Scan for additional instances (qBit-XXX sections)
+        for section in CONFIG.sections():
+            if section.startswith("qBit-") and section != "qBit":
+                instance_name = section.replace("qBit-", "", 1)
+                try:
+                    self._init_instance(section, instance_name)
+                    self.logger.info("Initialized qBit instance: %s", instance_name)
+                except Exception as e:
+                    self.logger.error(
+                        "Failed to initialize qBit instance '%s': %s", instance_name, e
+                    )
+                    self.instance_health[instance_name] = False
+
+        self.logger.info("Total qBit instances initialized: %d", len(self.clients))
+
+    def _init_instance(self, section_name: str, instance_name: str) -> None:
+        """
+        Initialize a single qBittorrent instance.
+
+        Args:
+            section_name: Config section name (e.g., "qBit-Seedbox")
+            instance_name: Short instance identifier (e.g., "Seedbox")
+
+        Raises:
+            Exception: If connection fails or version is unsupported
+        """
+        host = CONFIG.get(f"{section_name}.Host", fallback="localhost")
+        port = CONFIG.get(f"{section_name}.Port", fallback=8105)
+        username = CONFIG.get(f"{section_name}.UserName", fallback=None)
+        password = CONFIG.get(f"{section_name}.Password", fallback=None)
+
+        self.logger.debug(
+            "Connecting to qBit instance '%s': %s:%s (user: %s)",
+            instance_name,
+            host,
+            port,
+            username,
+        )
+
+        client = qbittorrentapi.Client(
+            host=host,
+            port=port,
+            username=username,
+            password=password,
+            SIMPLE_RESPONSES=False,
+        )
+
+        # Test connection and get version
+        try:
+            version = version_parser.parse(client.app_version())
+            self.logger.debug("Instance '%s' version: %s", instance_name, version)
+        except Exception as e:
+            self.logger.error(
+                "Could not connect to qBit instance '%s' at %s:%s: %s",
+                instance_name,
+                host,
+                port,
+                e,
+            )
+            raise
+
+        # Validate version
+        if version < self.min_supported_version:
+            self.logger.critical(
+                "Instance '%s' version %s is below minimum supported %s",
+                instance_name,
+                version,
+                self.min_supported_version,
+            )
+            raise ValueError(
+                f"Unsupported qBittorrent version {version} for instance {instance_name}"
+            )
+
+        # Register instance
+        self.clients[instance_name] = client
+        self.qbit_versions[instance_name] = version
+        self.instance_metadata[instance_name] = {
+            "host": host,
+            "port": port,
+            "username": username,
+        }
+        self.instance_health[instance_name] = True
+
+    def is_instance_alive(self, instance_name: str = "default") -> bool:
+        """
+        Check if a specific qBittorrent instance is alive and responding.
+
+        Args:
+            instance_name: The instance identifier (default: "default")
+
+        Returns:
+            bool: True if instance is healthy and responding, False otherwise
+        """
+        if instance_name not in self.clients:
+            self.logger.warning("Instance '%s' not found in clients", instance_name)
+            return False
+
+        client = self.clients[instance_name]
+        if client is None:
+            return False
+
+        try:
+            # Quick health check - just get app version
+            client.app_version()
+            self.instance_health[instance_name] = True
+            return True
+        except Exception as e:
+            self.logger.debug("Instance '%s' health check failed: %s", instance_name, e)
+            self.instance_health[instance_name] = False
+            return False
+
+    def get_all_instances(self) -> list[str]:
+        """
+        Get list of all configured qBittorrent instance names.
+
+        Returns:
+            list[str]: List of instance identifiers (e.g., ["default", "Seedbox"])
+        """
+        return list(self.clients.keys())
+
+    def get_healthy_instances(self) -> list[str]:
+        """
+        Get list of all healthy (responding) qBittorrent instances.
+
+        Returns:
+            list[str]: List of healthy instance identifiers
+        """
+        return [name for name in self.clients.keys() if self.is_instance_alive(name)]
+
+    def get_instance_info(self, instance_name: str = "default") -> dict:
+        """
+        Get metadata about a specific qBittorrent instance.
+
+        Args:
+            instance_name: The instance identifier (default: "default")
+
+        Returns:
+            dict: Instance metadata including host, port, version, health status
+        """
+        if instance_name not in self.clients:
+            return {"error": f"Instance '{instance_name}' not found"}
+
+        metadata = self.instance_metadata.get(instance_name, {})
+        return {
+            "name": instance_name,
+            "host": metadata.get("host"),
+            "port": metadata.get("port"),
+            "version": str(self.qbit_versions.get(instance_name, "unknown")),
+            "healthy": self.instance_health.get(instance_name, False),
+        }
+
+    def get_client(self, instance_name: str = "default") -> qbittorrentapi.Client | None:
+        """
+        Get qBittorrent client for a specific instance.
+
+        Args:
+            instance_name: The instance identifier (default: "default")
+
+        Returns:
+            qbittorrentapi.Client | None: Client instance, or None if not found/unhealthy
+        """
+        if instance_name not in self.clients:
+            self.logger.warning("Instance '%s' not found in clients", instance_name)
+            return None
+        return self.clients[instance_name]
+
     # @response_text(str)
     # @login_required
-    def app_version(self, **kwargs):
-        return self.client._get(
+    def app_version(self, instance_name: str = "default", **kwargs):
+        """Get qBittorrent app version for a specific instance."""
+        client = self.get_client(instance_name)
+        if client is None:
+            return None
+        return client._get(
             _name=APINames.Application,
             _method="version",
             _retries=0,
@@ -438,23 +642,44 @@ class qBitManager:
             **kwargs,
         )
 
-    def transfer_info(self, **kwargs):
-        """Proxy transfer info requests to the underlying qBittorrent client."""
-        if self.client is None:
+    def transfer_info(self, instance_name: str = "default", **kwargs):
+        """
+        Proxy transfer info requests to a specific qBittorrent instance.
+
+        Args:
+            instance_name: The instance identifier (default: "default")
+            **kwargs: Additional arguments to pass to transfer_info
+
+        Returns:
+            dict: Transfer info or connection status
+        """
+        client = self.get_client(instance_name)
+        if client is None:
             return {"connection_status": "disconnected"}
-        return self.client.transfer_info(**kwargs)
+        return client.transfer_info(**kwargs)
 
     @property
     def is_alive(self) -> bool:
+        """
+        Check if the default qBittorrent instance is alive.
+
+        Backward-compatible property that delegates to is_instance_alive("default").
+        Uses caching via expiring_bool to avoid excessive health checks.
+        """
         try:
             if self.client is None:
                 return False
             if 1 in self.expiring_bool:
                 return True
-            self.client.app_version()
-            self.logger.trace("Successfully connected to %s:%s", self.qBit_Host, self.qBit_Port)
-            self.expiring_bool.add(1)
-            return True
+            # Delegate to instance health check
+            alive = self.is_instance_alive("default")
+            if alive:
+                self.logger.trace(
+                    "Successfully connected to %s:%s", self.qBit_Host, self.qBit_Port
+                )
+                self.expiring_bool.add(1)
+                return True
+            self.logger.warning("Could not connect to %s:%s", self.qBit_Host, self.qBit_Port)
         except requests.RequestException:
             self.logger.warning("Could not connect to %s:%s", self.qBit_Host, self.qBit_Port)
         self.should_delay_torrent_scan = True
@@ -474,6 +699,8 @@ class qBitManager:
                 self.logger.warning(
                     "Startup thread still running after 60s; managing available workers."
                 )
+            started_processes = []
+            failed_processes = []
             for proc in list(self.child_processes):
                 try:
                     # Check if process has already been started
@@ -488,20 +715,153 @@ class qBitManager:
                         )
                         continue
 
-                    proc.start()
                     meta = self._process_registry.get(proc, {})
-                    self.logger.debug(
-                        "Started %s worker for category '%s'",
+                    self.logger.info(
+                        "Starting %s worker for category '%s'...",
                         meta.get("role", "worker"),
                         meta.get("category", "unknown"),
                     )
+                    proc.start()
+
+                    # Verify process actually started (give it a moment)
+                    time.sleep(0.1)
+                    if proc.is_alive():
+                        self.logger.info(
+                            "Successfully started %s worker for category '%s' (PID: %s)",
+                            meta.get("role", "worker"),
+                            meta.get("category", "unknown"),
+                            proc.pid,
+                        )
+                        started_processes.append((meta.get("role"), meta.get("category")))
+                    else:
+                        self.logger.error(
+                            "Process %s worker for category '%s' started but immediately died (exitcode: %s)",
+                            meta.get("role", "worker"),
+                            meta.get("category", "unknown"),
+                            proc.exitcode,
+                        )
+                        failed_processes.append((meta.get("role"), meta.get("category")))
                 except Exception as exc:
-                    self.logger.exception(
-                        "Failed to start worker process %s",
-                        getattr(proc, "name", repr(proc)),
+                    meta = self._process_registry.get(proc, {})
+                    self.logger.critical(
+                        "FAILED to start %s worker for category '%s': %s",
+                        meta.get("role", "worker"),
+                        meta.get("category", "unknown"),
+                        exc,
                         exc_info=exc,
                     )
+                    failed_processes.append((meta.get("role"), meta.get("category")))
+
+            # Log summary
+            if started_processes:
+                self.logger.info(
+                    "Started %d worker process(es): %s",
+                    len(started_processes),
+                    ", ".join(f"{role}({cat})" for role, cat in started_processes),
+                )
+            if failed_processes:
+                self.logger.critical(
+                    "FAILED to start %d worker process(es): %s - Will retry periodically",
+                    len(failed_processes),
+                    ", ".join(f"{role}({cat})" for role, cat in failed_processes),
+                )
+                # Track failed processes for retry
+                for role, category in failed_processes:
+                    key = (category, role)
+                    self._failed_spawn_attempts[key] = self._failed_spawn_attempts.get(key, 0) + 1
+                    # Add to retry queue if not already there
+                    if hasattr(self, "arr_manager") and self.arr_manager:
+                        for arr in self.arr_manager.managed_objects.values():
+                            if arr.category == category:
+                                # Check if already in pending spawns (avoid duplicates)
+                                meta = {"category": category, "role": role, "name": arr._name}
+                                already_pending = any(
+                                    m.get("category") == category and m.get("role") == role
+                                    for _, m in self._pending_spawns
+                                )
+                                if not already_pending:
+                                    self._pending_spawns.append((arr, meta))
+                                break
             while not self.shutdown_event.is_set():
+                # Check for database restart signal
+                if self.database_restart_event.is_set():
+                    self.logger.critical(
+                        "Database restart signal detected - terminating ALL processes for coordinated restart..."
+                    )
+                    # Terminate all child processes
+                    for proc in list(self.child_processes):
+                        if proc.is_alive():
+                            self.logger.warning(
+                                "Terminating %s process for database recovery",
+                                self._process_registry.get(proc, {}).get("role", "worker"),
+                            )
+                            proc.terminate()
+                    # Wait for processes to terminate
+                    time.sleep(2)
+                    # Force kill any that didn't terminate
+                    for proc in list(self.child_processes):
+                        if proc.is_alive():
+                            self.logger.error(
+                                "Force killing %s process",
+                                self._process_registry.get(proc, {}).get("role", "worker"),
+                            )
+                            proc.kill()
+                    # Clear all processes
+                    self.child_processes.clear()
+                    self._process_registry.clear()
+                    # Clear the event
+                    self.database_restart_event.clear()
+                    # Restart all Arr instances
+                    self.logger.critical("Restarting all Arr instances after database recovery...")
+                    if hasattr(self, "arr_manager") and self.arr_manager:
+                        for arr in self.arr_manager.managed_objects.values():
+                            try:
+                                worker_count, procs = arr.spawn_child_processes()
+                                for proc in procs:
+                                    role = (
+                                        "search"
+                                        if getattr(arr, "process_search_loop", None) is proc
+                                        else "torrent"
+                                    )
+                                    self._process_registry[proc] = {
+                                        "category": getattr(arr, "category", ""),
+                                        "name": getattr(arr, "_name", ""),
+                                        "role": role,
+                                    }
+                                    # CRITICAL: Actually start the process!
+                                    try:
+                                        proc.start()
+                                        time.sleep(0.1)  # Brief pause to let process initialize
+                                        if proc.is_alive():
+                                            self.logger.info(
+                                                "Started %s worker for %s (PID: %s)",
+                                                role,
+                                                arr._name,
+                                                proc.pid,
+                                            )
+                                        else:
+                                            self.logger.error(
+                                                "Respawned %s worker for %s died immediately (exitcode: %s)",
+                                                role,
+                                                arr._name,
+                                                proc.exitcode,
+                                            )
+                                    except Exception as start_exc:
+                                        self.logger.error(
+                                            "Failed to start respawned %s worker for %s: %s",
+                                            role,
+                                            arr._name,
+                                            start_exc,
+                                        )
+                                self.logger.info(
+                                    "Respawned %d process(es) for %s", worker_count, arr._name
+                                )
+                            except Exception as e:
+                                self.logger.exception(
+                                    "Failed to respawn processes for %s: %s", arr._name, e
+                                )
+                    continue
+
                 any_alive = False
                 for proc in list(self.child_processes):
                     if proc.is_alive():
@@ -544,6 +904,78 @@ class qBitManager:
                     with contextlib.suppress(ValueError):
                         self.child_processes.remove(proc)
 
+                # Retry failed process spawns
+                if self._pending_spawns and self.auto_restart_enabled:
+                    retry_spawns = []
+                    for arr, meta in self._pending_spawns:
+                        category = meta.get("category", "")
+                        role = meta.get("role", "")
+                        key = (category, role)
+                        attempts = self._failed_spawn_attempts.get(key, 0)
+
+                        # Exponential backoff: 30s, 60s, 120s, 240s, 480s (max 8min)
+                        # Retry indefinitely but with increasing delays
+                        self.logger.info(
+                            "Retrying spawn of %s worker for '%s' (attempt #%d)...",
+                            role,
+                            category,
+                            attempts + 1,
+                        )
+
+                        try:
+                            worker_count, procs = arr.spawn_child_processes()
+                            if worker_count > 0:
+                                for proc in procs:
+                                    proc_role = (
+                                        "search"
+                                        if getattr(arr, "process_search_loop", None) is proc
+                                        else "torrent"
+                                    )
+                                    if proc_role == role:  # Only start the one we're retrying
+                                        try:
+                                            proc.start()
+                                            time.sleep(0.1)
+                                            if proc.is_alive():
+                                                self.logger.info(
+                                                    "Successfully spawned %s worker for '%s' on retry (PID: %s)",
+                                                    role,
+                                                    category,
+                                                    proc.pid,
+                                                )
+                                                self._process_registry[proc] = meta
+                                                # CRITICAL: Add to child_processes so it's monitored
+                                                if proc not in self.child_processes:
+                                                    self.child_processes.append(proc)
+                                                # Clear failed attempts on success
+                                                self._failed_spawn_attempts.pop(key, None)
+                                            else:
+                                                self.logger.error(
+                                                    "Retry spawn failed: %s worker for '%s' died immediately",
+                                                    role,
+                                                    category,
+                                                )
+                                                retry_spawns.append((arr, meta))
+                                                self._failed_spawn_attempts[key] = attempts + 1
+                                        except Exception as exc:
+                                            self.logger.error(
+                                                "Retry spawn failed for %s worker '%s': %s",
+                                                role,
+                                                category,
+                                                exc,
+                                            )
+                                            retry_spawns.append((arr, meta))
+                                            self._failed_spawn_attempts[key] = attempts + 1
+                        except Exception as exc:
+                            self.logger.error(
+                                "Failed to respawn processes for retry: %s",
+                                exc,
+                            )
+                            retry_spawns.append((arr, meta))
+                            self._failed_spawn_attempts[key] = attempts + 1
+
+                    # Update pending spawns list
+                    self._pending_spawns = retry_spawns
+
                 if not self.child_processes:
                     if not any_alive:
                         break