PyFunceble-dev 4.3.0a15__py3-none-any.whl → 4.3.0a16__py3-none-any.whl

PyFunceble/cli/processes/base.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -50,451 +50,13 @@ License:
     limitations under the License.
 """
 
-import functools
-import multiprocessing
-import os
-import queue
-from typing import Any, List, Optional
+# pylint: disable=import-error,no-name-in-module,no-member
 
+import PyFunceble.ext.process_manager
 import PyFunceble.facility
-from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
-from PyFunceble.cli.processes.workers.base import WorkerBase
 
 
-class ProcessesManagerBase:
+class ProcessesManagerBase(PyFunceble.ext.process_manager.ProcessManagerCore):
     """
     Provides the base of all classes.
     """
-
-    CPU_COUNT = os.cpu_count()
-
-    if CPU_COUNT > 2:
-        STD_MAX_WORKER: int = CPU_COUNT - 2
-    else:
-        STD_MAX_WORKER: int = 1
-
-    WORKER_OBJ: Optional[WorkerBase] = None
-
-    input_datasets: Optional[List] = []
-    """
-    Use this variable if you want to differ the addition in the input queue.
-    """
-
-    output_datasets: Optional[List] = []
-    """
-    Use this variable if you want to differ the addition in the output queue.
-    """
-
-    input_queue: Optional[queue.Queue] = None
-    """
-    The input queue. Dataset will be given through this.
-    """
-
-    output_queue: Optional[List[queue.Queue]] = None
-    """
-    The output queue. This is where the result of a worker will be put.
-    """
-
-    daemon: Optional[bool] = None
-
-    manager: Optional[multiprocessing.Manager] = None
-
-    global_exit_event: Optional[multiprocessing.Event] = None
-    continuous_integration: Optional[ContinuousIntegrationBase] = None
-
-    _created_workers: Optional[List[WorkerBase]] = None
-    _running_workers: Optional[List[WorkerBase]] = None
-    _output_workers_count: Optional[int] = None
-
-    _max_worker: Optional[int] = None
-
-    def __init__(
-        self,
-        manager: Optional[multiprocessing.Manager] = None,
-        max_worker: Optional[int] = None,
-        *,
-        continuous_integration: Optional[ContinuousIntegrationBase] = None,
-        input_queue: Optional[queue.Queue] = None,
-        output_queue: Optional[queue.Queue] = None,
-        daemon: bool = False,
-        generate_input_queue: bool = True,
-        generate_output_queue: bool = True,
-        output_queue_num: int = 1,
-        output_workers_count: Optional[int] = None,
-    ) -> None:
-        if manager is not None:
-            self.manager = manager
-        else:
-            self.manager = multiprocessing.Manager()
-
-        if input_queue is None:
-            if generate_input_queue:
-                self.input_queue = self.manager.Queue()
-            else:
-                self.input_queue = None
-        else:
-            self.input_queue = input_queue
-
-        if output_queue is None:
-            if generate_output_queue:
-                self.output_queue = [
-                    self.manager.Queue() for _ in range(output_queue_num)
-                ]
-            else:
-                self.output_queue = None
-        else:
-            if not isinstance(output_queue, list):
-                self.output_queue = [output_queue]
-            else:
-                self.output_queue = output_queue
-
-        if max_worker is not None:
-            self.max_worker = max_worker
-        else:
-            self.max_worker = self.STD_MAX_WORKER
-
-        if continuous_integration is not None:
-            self.continuous_integration = continuous_integration
-
-        self.daemon = daemon
-
-        self.global_exit_event = multiprocessing.Event()
-
-        self._running_workers = []
-        self._created_workers = []
-
-        if output_workers_count is None:
-            self._output_workers_count = 1
-        else:
-            self._output_workers_count = int(output_workers_count)
-
-    def ensure_worker_obj_is_given(func):  # pylint: disable=no-self-argument
-        """
-        Ensures that the worker is properly declared before launching the
-        decorated method.
-        """
-
-        @functools.wraps(func)
-        def wrapper(self, *args, **kwargs):
-            if self.WORKER_OBJ is None:
-                raise TypeError(f"<self.WORKER_OBJ> should not be {None}!")
-
-            return func(self, *args, **kwargs)  # pylint: disable=not-callable
-
-        return wrapper
-
-    def create_workers_if_missing(func):  # pylint: disable=no-self-argument
-        """
-        Creates the workers if they are missing before launching the decorated
-        method.
-        """
-
-        @functools.wraps(func)
-        def wrapper(self, *args, **kwargs):
-            # pylint: disable=protected-access
-            if not self._created_workers:
-                self.create()
-
-            return func(self, *args, **kwargs)  # pylint: disable=not-callable
-
-        return wrapper
-
-    def ignore_if_running(func):  # pylint: disable=no-self-argument
-        """
-        Ignore the launching of the decorated method if the workers are
-        running.
-        """
-
-        @functools.wraps(func)
-        def wrapper(self, *args, **kwargs):
-            if not self.is_running():
-                return func(self, *args, **kwargs)  # pylint: disable=not-callable
-
-            return self
-
-        return wrapper
-
-    @property
-    def max_worker(self) -> Optional[int]:
-        """
-        Provides the number of maximum worker we are allowed to generate.
-        """
-
-        return self._max_worker
-
-    @max_worker.setter
-    def max_worker(self, value: int) -> None:
-        """
-        Sets the number of maximum worker we are authorized to generate.
-
-        :param value:
-            The value to set.
-
-        :raise TypeError:
-            When the given :code:`value` is not a :py:class:`int`.
-        :raise ValueError:
-            When the given :code:`value` is less than :code:`1`.
-        """
-
-        if not isinstance(value, int):
-            raise TypeError(f"<value> should be {int}, {type(value)} given.")
-
-        if value < 1:
-            raise ValueError("<value> should be greater or equal to one.")
-
-        self._max_worker = value
-
-    def set_max_worker(self, value: int) -> "ProcessesManagerBase":
-        """
-        Sets the number of maximum worker we are authorized to generate.
-
-        :param value:
-            The value to set.
-        """
-
-        self.max_worker = value
-
-        return self
-
-    def is_running(self) -> bool:
-        """
-        Checks if a worker is running.
-        """
-
-        if not self._running_workers:
-            return False
-
-        for worker in self._running_workers:
-            if worker.is_alive():
-                return True
-
-        return False
-
-    def send_stop_signal(
-        self, *, worker_name: Optional[str] = None
-    ) -> "ProcessesManagerBase":
-        """
-        Sends a stop message to the input queue.
-        """
-
-        self.add_to_all_input_queues(
-            "stop", worker_name=worker_name, include_destination=True
-        )
-
-    def terminate(self) -> "ProcessesManagerBase":
-        """
-        Terminates all workers and send a stop message to the declared output
-        queues - which are implicitly dependend of this process "pool".
-        """
-
-        if self._running_workers:
-            workers = self._running_workers
-        else:
-            workers = self._created_workers
-
-        if workers[0].global_exit_event:
-            workers[0].global_exit_event.set()
-
-        for worker in workers:
-            worker.terminate()
-            worker.join()
-
-        # When all worker of the current process are down or finished, send the
-        # stop message to the depending workers.
-
-        self.add_to_all_output_queues("stop")
-
-    def wait(self) -> "ProcessesManagerBase":
-        """
-        Wait until all workers are done.
-        """
-
-        for worker in self._running_workers:
-            PyFunceble.facility.Logger.info(
-                "Waiting for %r to finish.",
-                worker.name,
-            )
-            worker.join()
-
-            self._running_workers.remove(worker)
-            PyFunceble.facility.Logger.info(
-                "Still running: %r.",
-                self._running_workers,
-            )
-
-        for worker in self._created_workers:
-            if worker.exception:
-                worker_error, _ = worker.exception
-
-                self.terminate()
-                raise worker_error
-
-        self.terminate()
-
-    @create_workers_if_missing
-    def add_to_all_input_queues(
-        self,
-        data: Any,
-        *,
-        worker_name: Optional[str] = None,
-        include_destination: bool = False,
-    ) -> "ProcessesManagerBase":
-        """
-        Adds the given data to the input queues.
-
-        :param data:
-            The data to add into the queue.
-
-        :param include_destination:
-            Authorizes the addition of the destination into the message.
-        """
-
-        if self.is_running():
-            workers = self._running_workers
-        else:
-            workers = self._created_workers
-
-        for worker in workers:
-            if include_destination:
-                worker.add_to_input_queue(
-                    data, worker_name=worker_name, destination_worker=worker.name
-                )
-            else:
-                worker.add_to_input_queue(data, worker_name=worker_name)
-
-        PyFunceble.facility.Logger.debug("Added to all (input) queues: %r", data)
-
-    @create_workers_if_missing
-    def add_to_all_output_queues(
-        self,
-        data: Any,
-        *,
-        worker_name: Optional[str] = None,
-    ) -> "ProcessesManagerBase":
-        """
-        Adds the given data to the output queues.
-
-        :param data:
-            The data to add into the queue.
-        :param worker_name:
-            The name of the worker that is sending the message.
-        """
-
-        for _ in range(self._output_workers_count):
-            self.add_to_output_queue(data, worker_name=worker_name)
-
-        PyFunceble.facility.Logger.debug("Added to all (output) queues: %r", data)
-
-        return self
-
-    @create_workers_if_missing
-    def add_to_input_queue(
-        self, data: Any, *, worker_name: Optional[str] = None
-    ) -> "ProcessesManagerBase":
-        """
-        Adds the given data to the current queue.
-
-        :param data:
-            The data to add into the queue.
-        """
-
-        if self.is_running():
-            self._running_workers[0].add_to_input_queue(data, worker_name=worker_name)
-        else:
-            self._created_workers[0].add_to_input_queue(data, worker_name=worker_name)
-
-        PyFunceble.facility.Logger.debug("Added to the (main) queue: %r", data)
-
-    @create_workers_if_missing
-    def add_to_output_queue(
-        self, data: Any, *, worker_name: Optional[str] = None
-    ) -> "ProcessesManagerBase":
-        """
-        Adds the given data to the output queue.
-
-        :param data:
-            The data to add into the queue.
-        """
-
-        if self.is_running():
-            self._running_workers[0].add_to_output_queue(data, worker_name=worker_name)
-        else:
-            self._created_workers[0].add_to_output_queue(data, worker_name=worker_name)
-
-        PyFunceble.facility.Logger.debug("Added to the (output) queue: %r", data)
-
-        return self
-
-    def create(self) -> "ProcessesManagerBase":
-        """
-        Creates the defined amount of worker.
-        """
-
-        def share_concurrent_worker_names() -> None:
-            """
-            Share the name of all concurrent worker to all workers.
-            """
-
-            concurrent_names = [x.name for x in self._created_workers]
-
-            for worker in self._created_workers:
-                worker.concurrent_worker_names = list(concurrent_names)
-                worker.concurrent_worker_names.remove(worker.name)
-
-        for i in range(self.max_worker):
-            worker = self.WORKER_OBJ(  # pylint: disable=not-callable
-                self.input_queue,
-                self.output_queue,
-                self.global_exit_event,
-                name=f"{self.WORKER_OBJ.STD_NAME}_{i + 1}",
-                daemon=self.daemon,
-                continuous_integration=self.continuous_integration,
-                configuration=PyFunceble.storage.CONFIGURATION.to_dict(),
-            )
-
-            self._created_workers.append(worker)
-
-        share_concurrent_worker_names()
-
-        PyFunceble.facility.Logger.info(
-            "Created %r workers of %r. Details:\n%r",
-            len(self._created_workers),
-            self.WORKER_OBJ,
-            self._created_workers,
-        )
-
-        return self
-
-    @ensure_worker_obj_is_given
-    @create_workers_if_missing
-    @ignore_if_running
-    def start(self) -> "ProcessesManagerBase":
-        """
-        Starts all - previously - created workers.
-        """
-
-        for worker in self._created_workers:
-            try:
-                worker.start()
-            except AssertionError:
-                # Example: cannot start a process twice
-                pass
-
-            self._running_workers.append(worker)
-
-        if self.input_datasets:
-            while self.input_datasets:
-                self.add_to_input_queue(self.input_datasets.pop())
-
-        if self.output_datasets:
-            while self.output_datasets:
-                self.add_to_output_queue(self.output_datasets.pop())
-
-        PyFunceble.facility.Logger.info(
-            "Started %r workers of %r. Details:\n%r",
-            len(self._running_workers),
-            self.WORKER_OBJ,
-            self._running_workers,
-        )
-
-        return self

PyFunceble/cli/processes/chancy_producer.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -66,4 +66,5 @@ class ChancyProducerProcessesManager(ProcessesManagerBase):
         if you believe in your own luck.
     """
 
-    WORKER_OBJ: ChancyProducerWorker = ChancyProducerWorker
+    STD_NAME: str = "pyfunceble_chancy_producer_worker"
+    WORKER_CLASS: ChancyProducerWorker = ChancyProducerWorker

PyFunceble/cli/processes/chancy_tester.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -66,4 +66,5 @@ class ChancyTesterProcessesManager(ProcessesManagerBase):
         if you believe in your own luck.
     """
 
-    WORKER_OBJ: ChancyTesterWorker = ChancyTesterWorker
+    STD_NAME: str = "pyfunceble_chancy_tester_worker"
+    WORKER_CLASS: ChancyTesterWorker = ChancyTesterWorker

PyFunceble/cli/processes/dir_files_sorter.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -59,4 +59,5 @@ class DirFileSorterProcessesManager(ProcessesManagerBase):
     Provides the directory files sorter manager.
     """
 
-    WORKER_OBJ: DireFileSorterWorker = DireFileSorterWorker
+    STD_NAME: str = "pyfunceble_dir_files_sorter_worker"
+    WORKER_CLASS: DireFileSorterWorker = DireFileSorterWorker

PyFunceble/cli/processes/file_sorter.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -59,4 +59,5 @@ class FileSorterProcessesManager(ProcessesManagerBase):
     Provides the file sorter manager.
     """
 
-    WORKER_OBJ: FileSorterWorker = FileSorterWorker
+    STD_NAME: str = "pyfunceble_file_sorter_worker"
+    WORKER_CLASS: FileSorterWorker = FileSorterWorker

PyFunceble/cli/processes/migrator.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -89,7 +89,8 @@ class MigratorProcessesManager(ProcessesManagerBase):
     Provides the migrator manager.
     """
 
-    WORKER_OBJ: MigratorWorker = MigratorWorker
+    STD_NAME: str = "pyfunceble_migrator_worker"
+    WORKER_CLASS: MigratorWorker = MigratorWorker
 
     @staticmethod
     def json2csv_inactive_target(
@@ -430,27 +431,37 @@ class MigratorProcessesManager(ProcessesManagerBase):
                 "Stopped csv_file_add_registrar_column_target. File does not exist."
             )
 
-    def create(self) -> "ProcessesManagerBase":
+    def spawn_workers(self, *, start: bool = False) -> "ProcessesManagerBase":
+        """
+        Spawn the necessary workers.
+
+        :param bool start:
+            Whether we should start the workers or not.
+        """
+
         for method in dir(self):
             if not method.endswith("_target"):
                 continue
 
-            worker = MigratorWorker(
-                None,
-                name=f"pyfunceble_{method}",
-                daemon=True,
-                continuous_integration=self.continuous_integration,
-            )
-
+            worker = self.spawn_worker(start=False, daemon=True, force=True)
+            worker.name = f"pyfunceble_{method}"
             worker.target = getattr(self, method)
 
-            self._created_workers.append(worker)
+            if start:
+                worker.start()
+                self.running_workers.append(worker)
+
             PyFunceble.facility.Logger.info("Created worker for %r", method)
 
-    @ProcessesManagerBase.ensure_worker_obj_is_given
-    @ProcessesManagerBase.create_workers_if_missing
+    @ProcessesManagerBase.ensure_worker_class_is_set
+    @ProcessesManagerBase.ensure_worker_spawned
+    @ProcessesManagerBase.ignore_if_running
     def start(self) -> "ProcessesManagerBase":
+        """
+        Starts the migration process.
+        """
+
         # We start the migration (as a standalone)
-        Alembic(self._created_workers[0].db_session).upgrade()
+        Alembic(self.created_workers[0].db_session).upgrade()
 
         return super().start()

PyFunceble/cli/processes/miner.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -59,4 +59,5 @@ class MinerProcessesManager(ProcessesManagerBase):
     Provides the miner manager.
     """
 
-    WORKER_OBJ: MinerWorker = MinerWorker
+    STD_NAME: str = "pyfunceble_miner_worker"
+    WORKER_CLASS: MinerWorker = MinerWorker

PyFunceble/cli/processes/producer.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -59,4 +59,5 @@ class ProducerProcessesManager(ProcessesManagerBase):
     Provides the producer manager.
     """
 
-    WORKER_OBJ: ProducerWorker = ProducerWorker
+    STD_NAME: str = "pyfunceble_producer_worker"
+    WORKER_CLASS: ProducerWorker = ProducerWorker

PyFunceble/cli/processes/tester.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -59,4 +59,5 @@ class TesterProcessesManager(ProcessesManagerBase):
     Provides the tester manager.
     """
 
-    WORKER_OBJ: TesterWorker = TesterWorker
+    STD_NAME: str = "pyfunceble_tester_worker"
+    WORKER_CLASS: TesterWorker = TesterWorker

PyFunceble/cli/processes/workers/__init__.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.