PyFunceble-dev 4.3.0a14__py3-none-any.whl → 4.3.0a16__py3-none-any.whl

PyFunceble/cli/processes/workers/base.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -50,24 +50,21 @@ License:
     limitations under the License.
 """
 
+# pylint: disable=import-error,no-name-in-module,no-member
+
 import multiprocessing
-import multiprocessing.connection
-import queue
-import time
-import traceback
-from datetime import datetime, timedelta, timezone
-from typing import Any, List, Optional, Tuple
+from typing import Optional
 
 import PyFunceble.cli.facility
 import PyFunceble.cli.factory
+import PyFunceble.ext.process_manager
 import PyFunceble.facility
 import PyFunceble.sessions
 import PyFunceble.storage
-from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
 from PyFunceble.query.requests.requester import Requester
 
 
-class WorkerBase(multiprocessing.Process):
+class WorkerBase(PyFunceble.ext.process_manager.WorkerCore):
     """
     Provides the base of all our workers.
 
@@ -77,61 +74,15 @@ class WorkerBase(multiprocessing.Process):
         The output queue to write.
     """
 
-    STD_NAME: str = "pyfunceble_base_worker"
-
-    MINING_WAIT_TIME: int = 60
-    BREAKOFF: float = 2.0
-
-    input_queue: Optional[queue.Queue] = None
-    output_queue: Optional[queue.Queue] = None
-
-    continuous_integration: ContinuousIntegrationBase = None
-
-    global_exit_event: Optional[multiprocessing.Event] = None
-    exit_it: Optional[multiprocessing.Event] = None
-
-    send_stop_message: Optional[bool] = None
-    accept_waiting_delay: Optional[bool] = None
-
-    concurrent_worker_names: Optional[List[str]] = None
     db_session: Optional[PyFunceble.cli.factory.db_session] = None
-
-    _parent_connection: Optional[multiprocessing.connection.Connection] = None
-    _child_connection: Optional[multiprocessing.connection.Connection] = None
-    _exception: Optional[multiprocessing.Pipe] = None
-
-    _params: Optional[dict] = {}
     requester: Optional[Requester] = None
 
-    def __init__(
-        self,
-        input_queue: Optional[queue.Queue],
-        output_queue: Optional[queue.Queue] = None,
-        global_exit_event: Optional[multiprocessing.Event] = None,
-        *,
-        name: Optional[str] = None,
-        daemon: Optional[bool] = None,
-        continuous_integration: Optional[ContinuousIntegrationBase] = None,
-        configuration: Optional[dict] = None,
-    ) -> None:
-        self.configuration = self._params["configuration"] = configuration
-        self.input_queue = self._params["input_queue"] = input_queue
-        self.output_queue = self._params["output_queue"] = output_queue
-
-        self.continuous_integration = self._params["continuous_integration"] = (
-            continuous_integration
-        )
-
-        self.global_exit_event = self._params["global_exit_event"] = global_exit_event
-        self.exit_it = multiprocessing.Event()
-
-        self._parent_connection, self._child_connection = multiprocessing.Pipe()
-        self._exception = None
-
-        self.send_stop_message = True
-        self.accept_waiting_delay = True
-        self.concurrent_worker_names = []
+    def __del__(self) -> None:
+        if self.db_session is not None:
+            self.db_session.close()
 
+    def __post_init__(self) -> None:
+        self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
         try:
             self.db_session = (
                 PyFunceble.cli.factory.DBSession.get_db_session().get_new_session()()
@@ -139,281 +90,39 @@ class WorkerBase(multiprocessing.Process):
         except TypeError:
             self.db_session = None
 
-        super().__init__(name=name, daemon=daemon)
-
-        self.__post_init__()
-
-    def __del__(self) -> None:
-        if self.db_session is not None:
-            self.db_session.close()
-
-    def __post_init__(self) -> None:
-        """
-        A method which will be executed after the :code:`__init__` method.
-        """
-
-        self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
-
-    @property
-    def exception(self):
-        """
-        Provides the exception of the current worker.
-        """
-
-        if self._parent_connection.poll():
-            self._exception = self._parent_connection.recv()
-
-        return self._exception
-
-    def add_to_input_queue(
-        self,
-        data: Any,
-        *,
-        worker_name: Optional[str] = None,
-        destination_worker: Optional[str] = None,
-    ) -> "WorkerBase":
+    def perform_external_poweron_checks(self) -> bool:
         """
-        Adds the given data to the current queue.
-
-        :param data:
-            The data to add into the queue.
-        :param destination_worker:
-            The name of the worker which is supposed to read the message.
-        """
-
-        if worker_name:
-            to_send = (worker_name, destination_worker, data)
-        else:
-            to_send = (self.name, destination_worker, data)
-
-        self.input_queue.put(to_send)
-
-        PyFunceble.facility.Logger.debug("Added to the (input) queue: %r", data)
-
-    def add_to_output_queue(
-        self,
-        data: Any,
-        *,
-        worker_name: Optional[str] = None,
-        destination_worker: Optional[str] = None,
-    ) -> "WorkerBase":
-        """
-        Adds the given data to the output queue queue.
-
-        :param data:
-            The data to add into the queue.
+        Perform the external poweron checks.
         """
 
-        if worker_name:
-            to_send = (worker_name, destination_worker, data)
-        else:
-            to_send = (self.name, destination_worker, data)
-
-        if self.output_queue is not None:
-            for output_queue in self.output_queue:
-                output_queue.put(to_send)
-
-        PyFunceble.facility.Logger.debug("Added to the (output) queue: %r", data)
-
-        return self
-
-    def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
-        """
-        This the target that is run to process something.
-        This method should return a result which will pu send to the output
-        queue.
-        """
-
-        raise NotImplementedError()
-
-    def share_waiting_message(
-        self,
-        *,
-        overall: bool = False,
-        ignore_current_worker: bool = False,
-        apply_breakoff: bool = False,
-    ) -> "WorkerBase":
-        """
-        Shares a waiting message to both input and output queues.
-        The idea is to keep our worker awake when necessary. This functionality
-        is really important when running the mining worker because sometime
-        the flow is so long, that we may need some time.
-
-        :param overall:
-            Shares the message multiple time to specific destination in our
-            current stack.
-        :param ignore_current_worker:
-            Doesn't share the message with ourself.
-        :param apply_breakoff:
-            Activates the breakoff (sleep) before starting to share the message.
-        """
-
-        if overall:
-            for worker_name in self.concurrent_worker_names:
-                if apply_breakoff:
-                    time.sleep(self.BREAKOFF)
-
-                if not ignore_current_worker:
-                    # Give the whole stack a bit more of time.
-                    self.add_to_input_queue("wait", destination_worker=worker_name)
-
-                self.add_to_output_queue("wait")
-        else:
-            if apply_breakoff:
-                time.sleep(self.BREAKOFF)
-
-            if not ignore_current_worker:
-                self.add_to_input_queue("wait", destination_worker=self.name)
-
-            self.add_to_output_queue("wait")
-
-    def run(self) -> None:  # pylint: disable=too-many-statements
-        def break_now() -> bool:
-            """
-            Checks if it is time to make a break.
-            """
-
-            if not wait_for_stop or not self.accept_waiting_delay:
-                return True
-
-            return datetime.now(timezone.utc) > break_time
-
-        if self.configuration is not None:
+        if hasattr(self, "configuration") and self.configuration is not None:
             PyFunceble.facility.ConfigLoader.set_custom_config(self.configuration)
 
-        if multiprocessing.get_start_method() != "fork":
+        if (
+            multiprocessing.get_start_method() != "fork"
+            or not PyFunceble.storage.CONFIGURATION
+        ):
             PyFunceble.facility.ConfigLoader.start()
             PyFunceble.cli.facility.CredentialLoader.start()
             PyFunceble.cli.factory.DBSession.init_db_sessions()
 
-        wait_for_stop = (
-            bool(PyFunceble.storage.CONFIGURATION.cli_testing.mining) is True
-        )
-        break_time = datetime.now(timezone.utc) + timedelta(
-            seconds=self.MINING_WAIT_TIME
-        )
-
-        try:  # pylint: disable=too-many-nested-blocks
-            while True:
-                if self.global_exit_event.is_set():
-                    PyFunceble.facility.Logger.info(
-                        "Got global exit event. Stopping worker."
-                    )
-
-                    break
-
-                if self.exit_it.is_set():
-                    PyFunceble.facility.Logger.info("Got exit event. Stopping worker.")
-                    break
-
-                if (
-                    self.continuous_integration
-                    and self.continuous_integration.is_time_exceeded()
-                ):
-                    PyFunceble.facility.Logger.info(
-                        "CI time exceeded. Stopping worker."
-                    )
-
-                    if break_now():
-                        self.exit_it.set()
-                        continue
-
-                    continue
-
-                try:
-                    worker_name, destination_worker, consumed = self.input_queue.get()
-                except (EOFError, KeyboardInterrupt):
-                    PyFunceble.facility.Logger.info(
-                        "Got EOFError/KeyboardInterrupt. Stopping worker."
-                    )
-                    self.global_exit_event.set()
-                    break
-
-                if destination_worker and destination_worker != self.name:
-                    self.add_to_input_queue(
-                        consumed,
-                        worker_name=worker_name,
-                        destination_worker=destination_worker,
-                    )
-                    continue
-
-                PyFunceble.facility.Logger.info(
-                    "Got (from %r): %r",
-                    worker_name,
-                    consumed,
-                )
-
-                if consumed == "stop":
-                    if break_now():
-                        PyFunceble.facility.Logger.info(
-                            "Got stop message from %r. Applying.",
-                            worker_name,
-                        )
-
-                        self.exit_it.set()
-                        continue
-
-                    self.share_waiting_message(apply_breakoff=wait_for_stop)
-                    continue
-
-                if consumed == "wait":
-                    if not wait_for_stop:
-                        continue
-
-                    if break_now():
-                        PyFunceble.facility.Logger.debug(
-                            "Waited sufficiently. Stopping current worker."
-                        )
-
-                        # The real (break) action shouldn't be handled here.
-                        self.exit_it.set()
-                        continue
-
-                    PyFunceble.facility.Logger.debug(
-                        "We need to wait a bit more. Continue waiting."
-                    )
-
-                    self.share_waiting_message(apply_breakoff=wait_for_stop)
-                    continue
-
-                try:
-                    result = self.target(consumed)
-                except (EOFError, KeyboardInterrupt):
-                    PyFunceble.facility.Logger.info(
-                        "Got EOFError/KeyboardInterrupt. Stopping worker."
-                    )
-                    self.global_exit_event.set()
-                    break
-
-                if result is not None:
-                    self.add_to_output_queue(result)
-
-                    PyFunceble.facility.Logger.info(
-                        "Produced: %r",
-                        result,
-                    )
-
-                break_time = datetime.now(timezone.utc) + timedelta(
-                    seconds=self.MINING_WAIT_TIME
-                )
-
-        except Exception as exception:  # pylint: disable=broad-except
-            PyFunceble.facility.Logger.critical(
-                "Error while running target", exc_info=True
-            )
-            trace = traceback.format_exc()
-            self._child_connection.send((exception, trace))
+        return super().perform_external_poweron_checks()
 
-            self.exit_it.set()
-            raise exception
-
-    def terminate(self) -> None:
+    def perform_external_preflight_checks(self) -> bool:
         """
-        Terminate our worker.
+        Perform the external preflight checks.
         """
 
-        self.exit_it.set()
+        if (
+            hasattr(self, "continuous_integration")
+            and self.continuous_integration
+            and self.continuous_integration.is_time_exceeded()
+        ):
+            PyFunceble.facility.Logger.info("CI time exceeded. Stopping worker.")
+
+            if not self.delay_shutdown:
+                self.exit_it.set()
+
+            return False
 
-        if self.input_queue:
-            # Necessary to avoid blocking.
-            self.add_to_input_queue("stop", destination_worker=self.name)
+        return super().perform_external_preflight_checks()

PyFunceble/cli/processes/workers/chancy_producer.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -69,8 +69,6 @@ class ChancyProducerWorker(ProducerWorker):
         USE AT YOUR OWN RISK. GOOD LUCK!
     """
 
-    STD_NAME: str = "pyfunceble_chancy_producer_worker"
-
     def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
         if not isinstance(consumed, tuple):
             PyFunceble.facility.Logger.info(

PyFunceble/cli/processes/workers/chancy_tester.py

@@ -37,7 +37,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -70,10 +70,10 @@ class ChancyTesterWorker(TesterWorker):
         USE AT YOUR OWN RISK. GOOD LUCK!
     """
 
-    STD_NAME: str = "pyfunceble_chancy_tester_worker"
-
     def __post_init__(self) -> None:
-        self.producer_worker = ProducerWorker(**self._params)
+        self.producer_worker = ProducerWorker(**self._all_args)
+        self.producer_worker.name = f"{self.name}.producer"
+        self.producer_worker.perform_external_poweron_checks()
 
         return super().__post_init__()
 
@@ -81,7 +81,7 @@ class ChancyTesterWorker(TesterWorker):
         """
         The actually wall destructor.
 
-        :param consummed:
+        :param consumed:
             The data that needs to be tested.
         """
 

PyFunceble/cli/processes/workers/dir_files_sorter.py

@@ -37,7 +37,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -85,8 +85,6 @@ class DireFileSorterWorker(FileSorterWorkerBase):
         None
     """
 
-    STD_NAME: str = "pyfunceble_dir_files_sorter_worker"
-
     @staticmethod
     def get_files_to_sort(directory: str) -> List[str]:
         """
@@ -130,6 +128,13 @@ class DireFileSorterWorker(FileSorterWorkerBase):
         return result
 
     def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
+        """
+        The producer of the worker.
+
+        :param consumed:
+            The consumed data to work with.
+        """
+
         if (
             not isinstance(consumed, dict)
             and "directory" not in consumed

PyFunceble/cli/processes/workers/file_sorter.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -81,9 +81,14 @@ class FileSorterWorker(FileSorterWorkerBase):
         None
     """
 
-    STD_NAME: str = "pyfunceble_file_sorter_worker"
-
     def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
+        """
+        The producer of the worker.
+
+        :param consumed:
+            The consumed data to work with.
+        """
+
         if (
             not isinstance(consumed, dict)
             and "file" not in consumed

PyFunceble/cli/processes/workers/file_sorter_base.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.

PyFunceble/cli/processes/workers/migrator.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -64,9 +64,11 @@ class MigratorWorker(WorkerBase):
     to handle the mining of dataset to test.
     """
 
-    STD_NAME: str = "pyfunceble_migrator_worker"
-
     def run(self) -> None:
+        """
+        This is the brain of the worker.
+        """
+
         try:
             try:
                 self.target(self.continuous_integration, db_session=self.db_session)

PyFunceble/cli/processes/workers/miner.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -71,8 +71,6 @@ class MinerWorker(WorkerBase):
     the mining of dataset to test.
     """
 
-    STD_NAME: str = "pyfunceble_miner_worker"
-
     INACTIVE_STATUSES: Tuple[str, ...] = (
         PyFunceble.storage.STATUS.down,
         PyFunceble.storage.STATUS.invalid,
@@ -123,6 +121,13 @@ class MinerWorker(WorkerBase):
         return result
 
     def target(self, consumed: Tuple[dict, CheckerStatusBase]) -> None:
+        """
+        The producer of the worker.
+
+        :param consumed:
+            The consumed data to work with.
+        """
+
         if not isinstance(consumed, tuple) or not isinstance(
             consumed[1], CheckerStatusBase
         ):
@@ -155,7 +160,7 @@ class MinerWorker(WorkerBase):
 
         print_single_line("M")
 
-        self.add_to_output_queue("pyfunceble")
+        self.push_to_output_queues("pyfunceble")
         self.share_waiting_message()
         mined = self.mine_from(subject)
 
@@ -184,7 +189,7 @@ class MinerWorker(WorkerBase):
                 )
                 continue
 
-            self.add_to_output_queue(to_send)
+            self.push_to_output_queues(to_send)
 
         # Returning None because we manually add into the queue.
         return None

PyFunceble/cli/processes/workers/producer.py

@@ -35,7 +35,7 @@ License:
 ::
 
 
-    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024 Nissar Chababy
+    Copyright 2017, 2018, 2019, 2020, 2022, 2023, 2024, 2025 Nissar Chababy
 
     Licensed under the Apache License, Version 2.0 (the "License");
     you may not use this file except in compliance with the License.
@@ -85,8 +85,6 @@ class ProducerWorker(WorkerBase):
     the production of output to stdout or files.
     """
 
-    STD_NAME: str = "pyfunceble_producer_worker"
-
     stdout_printer: Optional[StdoutPrinter] = None
     file_printer: Optional[FilePrinter] = None
     whois_dataset: Optional[WhoisDatasetBase] = None
@@ -104,7 +102,9 @@ class ProducerWorker(WorkerBase):
         PyFunceble.storage.STATUS.invalid,
     )
 
-    def __post_init__(self) -> None:
+    def perform_external_poweron_checks(self) -> None:
+        result = super().perform_external_poweron_checks()
+
         skip_columns = []
         extra_formatters = {}
 
@@ -137,7 +137,7 @@ class ProducerWorker(WorkerBase):
 
         self.header_already_printed = False
 
-        return super().__post_init__()
+        return result
 
     @staticmethod
     def should_we_ignore(test_result: CheckerStatusBase) -> bool:
@@ -177,7 +177,7 @@ class ProducerWorker(WorkerBase):
         """
         Checks if we should block the file printer.
 
-        The reason behindn this is that we don't want to generate an output
+        The reason behind this is that we don't want to generate an output
         when a subject was already into the inactive database.
         """
 
@@ -303,7 +303,7 @@ class ProducerWorker(WorkerBase):
         Runs the analytic behind the file printer.
 
         .. warning::
-            Thie method assume that the givne dataset is ignored from the normal
+            Thie method assume that the given dataset is ignored from the normal
             file printer.
         """
 
@@ -388,6 +388,13 @@ class ProducerWorker(WorkerBase):
                 ).count(test_result.registrar)
 
     def target(self, consumed: Any) -> Optional[Tuple[Any, ...]]:
+        """
+        The producer of the worker.
+
+        :param consumed:
+            The consumed data to work with.
+        """
+
         if not isinstance(consumed, tuple):
             PyFunceble.facility.Logger.info(
                 "Skipping latest dataset because consumed data was not a tuple."
@@ -399,7 +406,7 @@ class ProducerWorker(WorkerBase):
 
         if not isinstance(test_dataset, dict):
             PyFunceble.facility.Logger.info(
-                "Skipping because test dataset is not a dictionnary."
+                "Skipping because test dataset is not a dictionary."
             )
             return None