PyFunceble-dev 4.2.7__py3-none-any.whl → 4.2.10__py3-none-any.whl

PyFunceble/checker/availability/base.py

@@ -76,7 +76,6 @@ from PyFunceble.checker.syntax.domain import DomainSyntaxChecker
 from PyFunceble.checker.syntax.ip import IPSyntaxChecker
 from PyFunceble.checker.syntax.url import URLSyntaxChecker
 from PyFunceble.converter.url2netloc import Url2Netloc
-from PyFunceble.helpers.regex import RegexHelper
 from PyFunceble.query.dns.query_tool import DNSQueryTool
 from PyFunceble.query.http_status_code import HTTPStatusCode
 from PyFunceble.query.netinfo.address import AddressInfo
@@ -919,12 +918,7 @@ class AvailabilityCheckerBase(CheckerBase):
         if from_domain_test and self.status.url_syntax:
             return self
 
-        if not self.status.url_syntax and not RegexHelper("[^a-z0-9._]").match(
-            self.idna_subject, return_match=False
-        ):
-            # The regex is there because while testing for domain, sometime we
-            # may see something like mailto:xxx@yyy.de
-
+        if not self.status.url_syntax and not self.idna_subject.startswith("http"):
             self.http_status_code_query_tool.set_subject(
                 f"http://{self.idna_subject}:80"
             )
@@ -1049,7 +1043,13 @@ class AvailabilityCheckerBase(CheckerBase):
                 self.collection_query_tool.preferred_status_origin == "latest"
                 and data["status"]["availability"]["latest"]
             ):
-                self.status.status = data["status"]["availability"]["latest"]["status"]
+                try:
+                    # legacy
+                    self.status.status = data["status"]["availability"]["latest"][
+                        "status"
+                    ]
+                except KeyError:
+                    self.status.status = data["status"]["availability"]["latest"]
                 self.status.status_source = "COLLECTION"
             elif (
                 self.collection_query_tool.preferred_status_origin == "recommended"

PyFunceble/checker/availability/domain.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import PyFunceble.facility
 import PyFunceble.factory
 import PyFunceble.storage

PyFunceble/checker/availability/extras/base.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import functools
 import socket
 from typing import Callable, Dict, List, Optional, Union
@@ -423,7 +422,7 @@ class ExtraRuleHandlerBase:
         of the given one.
         """
 
-        if not isinstance(status_code, (list, tuple)):
+        if not isinstance(status_code, (list, tuple, set)):
             status_code = [status_code]
 
         if any(self.status.http_status_code == x for x in status_code):

PyFunceble/checker/availability/extras/rules.py

@@ -78,7 +78,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
     def __init__(self, status: Optional[AvailabilityCheckerStatus] = None) -> None:
         self.regex_active2inactive = {
             r"\.000webhostapp\.com": [
-                (self.switch_to_down_if_status_code, 410),
+                (self.switch_to_down_if_status_code, {410, 424}),
             ],
             r"\.24\.eu$": [(self.switch_to_down_if_status_code, 503)],
             r"\.altervista\.org$": [(self.switch_to_down_if_status_code, 403)],

PyFunceble/checker/availability/ip.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import PyFunceble.facility
 import PyFunceble.factory
 import PyFunceble.storage

PyFunceble/checker/availability/url.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import PyFunceble.facility
 import PyFunceble.factory
 import PyFunceble.storage

PyFunceble/checker/reputation/base.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 from typing import List, Optional
 
 from sqlalchemy.orm import Session

PyFunceble/checker/reputation/domain.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 from typing import List, Optional
 
 from PyFunceble.checker.reputation.base import ReputationCheckerBase

PyFunceble/checker/reputation/ip.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 from typing import List, Optional
 
 from PyFunceble.checker.reputation.base import ReputationCheckerBase

PyFunceble/checker/reputation/url.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 from typing import List, Optional
 
 from PyFunceble.checker.reputation.base import ReputationCheckerBase

PyFunceble/checker/syntax/second_lvl_domain.py

@@ -65,8 +65,12 @@ class SecondLvlDomainSyntaxChecker(DomainSyntaxCheckerBase):
     """
 
     # pylint: disable=line-too-long
-    REGEX_VALID_DOMAIN: str = r"^(?=.{0,253}$)(([a-z0-9][a-z0-9-]{0,61}[a-z0-9]|[a-z0-9])\.)+((?=.*[^0-9])([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
-    REGEX_VALID_RELAXED_DOMAIN: str = r"^(?=.{0,253}$)(([a-z0-9][a-z0-9_-]{0,61}[a-z0-9_-]|[a-z0-9])\.)+((?=.*[^0-9])([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
+    REGEX_VALID_DOMAIN: str = (
+        r"^(?=.{0,253}$)(([a-z0-9][a-z0-9-]{0,61}[a-z0-9]|[a-z0-9])\.)+((?=.*[^0-9])([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
+    )
+    REGEX_VALID_RELAXED_DOMAIN: str = (
+        r"^(?=.{0,253}$)(([a-z0-9][a-z0-9_-]{0,61}[a-z0-9_-]|[a-z0-9])\.)+((?=.*[^0-9])([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
+    )
 
     last_point_index: Optional[int] = None
     """

PyFunceble/checker/syntax/status.py

@@ -50,6 +50,7 @@ License:
     See the License for the specific language governing permissions and
     limitations under the License.
 """
+
 import dataclasses
 from typing import Optional
 

PyFunceble/checker/syntax/subdomain.py

@@ -63,7 +63,9 @@ class SubDomainSyntaxChecker(DomainSyntaxCheckerBase):
     """
 
     # pylint: disable=line-too-long
-    REGEX_VALID_SUBDOMAIN: str = r"^(?=.{0,253}$)(([a-z0-9_][a-z0-9-_]{0,61}[a-z0-9_-]|[a-z0-9])\.)+((?=.*)([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
+    REGEX_VALID_SUBDOMAIN: str = (
+        r"^(?=.{0,253}$)(([a-z0-9_][a-z0-9-_]{0,61}[a-z0-9_-]|[a-z0-9])\.)+((?=.*)([a-z0-9][a-z0-9-]{0,61}[a-z0-9](?:\.)?|[a-z0-9](?:\.)?))$"
+    )
 
     @DomainSyntaxCheckerBase.ensure_subject_is_given
     def is_valid(self) -> bool:

PyFunceble/checker/syntax/url.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import urllib.parse
 from typing import Optional
 

PyFunceble/cli/credential_loader.py

@@ -266,9 +266,9 @@ class CredentialLoader:
             # We directly share the credential object into the DBSession object.
             # This will let us use the DBSession without having to think about
             # any other headache.
-            self.credential = (
-                PyFunceble.cli.factory.DBSession.credential
-            ) = self.DB_TYPE2OBJ[self.db_type]()
+            self.credential = PyFunceble.cli.factory.DBSession.credential = (
+                self.DB_TYPE2OBJ[self.db_type]()
+            )
 
             env_var_helper = EnvironmentVariableHelper(
                 env_file_path=self.credential.get_dot_env_file()

PyFunceble/cli/entry_points/pyfunceble/cli.py

@@ -69,7 +69,9 @@ from PyFunceble.cli.system.launcher import SystemLauncher
 from PyFunceble.helpers.regex import RegexHelper
 
 
-def get_configured_value(entry: str, *, negate=False) -> Any:
+def get_configured_value(
+    entry: str, *, negate: bool = False, value_only: bool = False
+) -> Any:
     """
     Provides the currently configured value.
 
@@ -81,6 +83,9 @@ def get_configured_value(entry: str, *, negate=False) -> Any:
     :param negate:
         Allows us to negate the result from the configuration.
 
+    :param value_only:
+        Whether we should only return the value or the full message.
+
     :raise ValueError:
         When the given :code:`entry` is not found.
     """
@@ -102,10 +107,14 @@ def get_configured_value(entry: str, *, negate=False) -> Any:
         result = not result
 
     return (
-        f"\n{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}"
-        f"Configured value: {colorama.Fore.BLUE}"
-        f"{result!r}"
-        f"{colorama.Style.RESET_ALL}"
+        (
+            f"\n{colorama.Fore.YELLOW}{colorama.Style.BRIGHT}"
+            f"Configured value: {colorama.Fore.BLUE}"
+            f"{result!r}"
+            f"{colorama.Style.RESET_ALL}"
+        )
+        if not value_only
+        else result
     )
 
 
@@ -122,6 +131,12 @@ def add_arguments_to_parser(
         if "dest" in opt_args:
             opt_args["dest"] = opt_args["dest"].replace(".", "__")
 
+        for index, value in enumerate(pos_args):
+            if value.startswith("-") and "." not in value:
+                continue
+
+            pos_args[index] = value.replace(".", "__")
+
         parser.add_argument(*pos_args, **opt_args)
 
 
@@ -1184,6 +1199,42 @@ def get_default_group_data() -> List[Tuple[List[str], dict]]:
     ]
 
 
+def platform_parser(
+    parser: Union[argparse.ArgumentParser, argparse._SubParsersAction]
+) -> None:
+    """
+    Adds the platform group to the given parser.
+    """
+
+    platform = parser.add_parser(
+        "platform",
+        add_help=False,
+        epilog=PyFunceble.cli.storage.STD_EPILOG,
+    )
+
+    args = [
+        (
+            ["cli_testing.testing_mode.platform_contribution"],
+            {
+                "default": get_configured_value(
+                    "cli_testing.testing_mode.platform_contribution", value_only=True
+                ),
+                "action": "store_%s"
+                % str(
+                    not get_configured_value(
+                        "cli_testing.testing_mode.platform_contribution",
+                        value_only=True,
+                    )
+                ).lower(),
+                "help": argparse.SUPPRESS,
+            },
+        )
+    ]
+
+    add_arguments_to_parser(platform, args)
+    add_arguments_to_parser(platform, get_default_group_data())
+
+
 def ask_authorization_to_merge_config(missing_key: Optional[str] = None) -> bool:
     """
     Asks the end-user for the authorization to merge the upstream
@@ -1292,6 +1343,8 @@ def tool() -> None:
 
     # pylint:  disable=possibly-unused-variable
 
+    command_sub = parser.add_subparsers(dest="command", help=argparse.SUPPRESS)
+
     shtab.add_argument_to(
         parser,
         option_string=["--show-completion"],
@@ -1321,6 +1374,8 @@ def tool() -> None:
         get_ci_group_data,
     ]
 
+    parse_funcs = [platform_parser]
+
     for func in funcs:
         parser_name = func.__name__.replace("get_", "").replace("_data", "")
 
@@ -1348,10 +1403,21 @@ def tool() -> None:
                 )
                 sys.exit(1)
 
+    for func in parse_funcs:
+        func(command_sub)
+
     add_arguments_to_parser(parser, get_default_group_data())
 
     args = parser.parse_args()
 
-    if any(getattr(args, x) for x in ["domains", "urls", "files", "url_files"]):
+    if any(
+        getattr(args, x)
+        for x in [
+            "domains",
+            "urls",
+            "files",
+            "url_files",
+        ]
+    ) or bool(args.command):
         SystemIntegrator(args).start()
         SystemLauncher(args).start()

PyFunceble/cli/file_preloader.py

@@ -481,9 +481,9 @@ class FilePreloader:
                 raise exception
 
         if not broken:
-            self.__description[self.__matching_index][
-                "previous_hash"
-            ] = self.__description[self.__matching_index]["hash"]
+            self.__description[self.__matching_index]["previous_hash"] = (
+                self.__description[self.__matching_index]["hash"]
+            )
 
         self.__save_description()
 

PyFunceble/cli/migrators/base.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 from typing import Optional
 
 from sqlalchemy.orm import Session

PyFunceble/cli/processes/workers/base.py

@@ -115,9 +115,9 @@ class WorkerBase(multiprocessing.Process):
         self.input_queue = self._params["input_queue"] = input_queue
         self.output_queue = self._params["output_queue"] = output_queue
 
-        self.continuous_integration = self._params[
-            "continuous_integration"
-        ] = continuous_integration
+        self.continuous_integration = self._params["continuous_integration"] = (
+            continuous_integration
+        )
 
         self.global_exit_event = self._params["global_exit_event"] = global_exit_event
         self.exit_it = multiprocessing.Event()
@@ -318,8 +318,10 @@ class WorkerBase(multiprocessing.Process):
 
                 try:
                     worker_name, destination_worker, consumed = self.input_queue.get()
-                except EOFError:
-                    PyFunceble.facility.Logger.info("Got EOFError. Stopping worker.")
+                except (EOFError, KeyboardInterrupt):
+                    PyFunceble.facility.Logger.info(
+                        "Got EOFError/KeyboardInterrupt. Stopping worker."
+                    )
                     self.global_exit_event.set()
                     break
 
@@ -370,7 +372,14 @@ class WorkerBase(multiprocessing.Process):
                     self.share_waiting_message(apply_breakoff=wait_for_stop)
                     continue
 
-                result = self.target(consumed)
+                try:
+                    result = self.target(consumed)
+                except (EOFError, KeyboardInterrupt):
+                    PyFunceble.facility.Logger.info(
+                        "Got EOFError/KeyboardInterrupt. Stopping worker."
+                    )
+                    self.global_exit_event.set()
+                    break
 
                 if result is not None:
                     self.add_to_output_queue(result)

PyFunceble/cli/processes/workers/producer.py

@@ -399,6 +399,11 @@ class ProducerWorker(WorkerBase):
             )
             return None
 
+        if test_dataset["type"] == "platform-contribution":
+            self.collection_query_tool.deliver_contract(
+                test_dataset["contract"], test_result
+            )
+
         if (
             PyFunceble.storage.CONFIGURATION.collection.push
             and test_result.status_source != "COLLECTION"

PyFunceble/cli/scripts/production.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-
 import copy
 import functools
 import os
@@ -516,9 +515,9 @@ class ProductionPrep:
             if to_append not in self.version_file_content["deprecated"]:
                 self.version_file_content["deprecated"].append(to_append)
 
-        self.version_file_content[
-            "current_version"
-        ] = PyFunceble.storage.PROJECT_VERSION
+        self.version_file_content["current_version"] = (
+            PyFunceble.storage.PROJECT_VERSION
+        )
 
         self.dict_helper.set_subject(self.version_file_content).to_yaml_file(
             self.VERSION_FILE_PATH

PyFunceble/cli/storage.py

@@ -253,9 +253,11 @@ UNIVERSAL_OUTPUTS: dict = {
 }
 
 OUTPUTS: Optional[Box] = Box(
-    Merge(UNIX_OUTPUTS).into(UNIVERSAL_OUTPUTS)
-    if PlatformUtility.is_unix()
-    else Merge(WIN_OUTPUTS).into(UNIVERSAL_OUTPUTS),
+    (
+        Merge(UNIX_OUTPUTS).into(UNIVERSAL_OUTPUTS)
+        if PlatformUtility.is_unix()
+        else Merge(WIN_OUTPUTS).into(UNIVERSAL_OUTPUTS)
+    ),
     frozen_box=True,
 )
 

PyFunceble/cli/system/launcher.py

@@ -119,6 +119,7 @@ from PyFunceble.dataset.inactive.base import InactiveDatasetBase
 from PyFunceble.helpers.directory import DirectoryHelper
 from PyFunceble.helpers.download import DownloadHelper
 from PyFunceble.helpers.file import FileHelper
+from PyFunceble.query.collection import CollectionQueryTool
 
 
 class SystemLauncher(SystemBase):
@@ -222,7 +223,7 @@ class SystemLauncher(SystemBase):
             )
             self.producer_process_manager = ChancyProducerProcessesManager(
                 self.manager,
-                max_worker=1,
+                max_worker=PyFunceble.storage.CONFIGURATION.cli_testing.max_workers,
                 continuous_integration=self.continuous_integration,
                 input_queue=self.tester_process_manager.output_queue[0],
                 daemon=True,
@@ -406,6 +407,21 @@ class SystemLauncher(SystemBase):
                     "Added to the protocol:\n%r", to_append
                 )
 
+        # pylint: disable=line-too-long
+        if (
+            PyFunceble.storage.CONFIGURATION.cli_testing.testing_mode.platform_contribution
+        ):
+            self.testing_protocol.append(
+                {
+                    "type": "platform-contribution",
+                    "subject_type": "any",
+                    "destination": None,
+                    "checker_type": None,
+                    "output_dir": None,
+                    "session_id": None,
+                }
+            )
+
     def ci_stop_in_the_middle_if_time_exceeded(self) -> "SystemLauncher":
         """
         Stops our processes as soon as the time is exceeded.
@@ -648,7 +664,37 @@ class SystemLauncher(SystemBase):
                     )
             elif protocol["type"] == "file":
                 handle_file(protocol)
+            elif protocol["type"] == "platform-contribution":
+                query_tool = CollectionQueryTool()
+
+                while True:
+                    for next_contract in next(
+                        query_tool.pull_contract(
+                            PyFunceble.storage.CONFIGURATION.cli_testing.max_workers
+                        )
+                    ):
+                        if (
+                            "subject" not in next_contract
+                            or not next_contract["subject"]
+                        ):
+                            continue
 
+                        protocol_data = copy.deepcopy(protocol)
+
+                        protocol_data["checker_type"] = next_contract[
+                            "checker_type"
+                        ].upper()
+                        protocol_data["subject_type"] = next_contract["subject_type"]
+                        protocol_data["subject"] = protocol_data["idna_subject"] = (
+                            next_contract["subject"]["subject"]
+                        )
+                        protocol_data["contract"] = copy.deepcopy(next_contract)
+
+                        self.tester_process_manager.add_to_input_queue(
+                            protocol_data, worker_name="main"
+                        )
+
+                    self.ci_stop_in_the_middle_if_time_exceeded()
         return self
 
     def generate_waiting_files(self) -> "SystemLauncher":

PyFunceble/cli/utils/testing.py

@@ -232,9 +232,7 @@ def get_subjects_from_line(
     if cidr2subject is None:
         cidr2subject = CIDR2Subject()
 
-    adblock_inputline2subject.aggressive = (
-        wildcard2subject.aggressive
-    ) = (
+    adblock_inputline2subject.aggressive = wildcard2subject.aggressive = (
         rpz_inputline2subject.aggressive
     ) = inputline2subject.aggressive = url2netloc.aggressive = bool(
         PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive

PyFunceble/config/loader.py

@@ -172,6 +172,16 @@ class ConfigLoader:
             # If timeout is set to a negative digit, switch to the default one.
             config["lookup"]["timeout"] = 5
 
+        if (
+            "cli_testing" in config
+            and "testing_mode" in config["cli_testing"]
+            and "platform_contribution" in config["cli_testing"]["testing_mode"]
+            and config["cli_testing"]["testing_mode"]["platform_contribution"]
+        ):
+            # If we are under a special testing mode. We shouldn't generate
+            # any files
+            config["cli_testing"]["file_generation"]["no_file"] = True
+
         return config
 
     @staticmethod

PyFunceble/data/alembic/mysql/versions/35c79626ecb9_fix_some_columns.py

@@ -5,6 +5,7 @@ Revises: ade87195b0a0
 Create Date: 2020-08-21 11:42:07.044762
 
 """
+
 from alembic import op
 from sqlalchemy.exc import OperationalError
 

PyFunceble/data/alembic/mysql/versions/3a4c55a9320d_add_continue_table.py

@@ -5,6 +5,7 @@ Revises: 3d6f4a33cdb2
 Create Date: 2020-12-13 22:52:56.968513
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/3d6f4a33cdb2_add_inactive_table.py

@@ -5,6 +5,7 @@ Revises: 7bcf7fa64ba1
 Create Date: 2020-12-13 19:45:41.893657
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/45713fea8097_deletion_uneeded_columns_from_whois_.py

@@ -5,6 +5,7 @@ Revises: e04e8301d1a2
 Create Date: 2020-12-07 12:36:04.818466
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql

PyFunceble/data/alembic/mysql/versions/459a0d7b8f09_add_idna_subject_column_into_whois.py

@@ -5,6 +5,7 @@ Revises: 45713fea8097
 Create Date: 2020-12-07 12:37:52.018637
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/6f4729deaf03_delete_inactive_source_column.py

@@ -5,6 +5,7 @@ Revises: 95dc17ddd729
 Create Date: 2021-02-13 12:21:00.493002
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql

PyFunceble/data/alembic/mysql/versions/7bcf7fa64ba1_rename_created_to_created_at_and.py

@@ -5,6 +5,7 @@ Revises: 83ada95132bf
 Create Date: 2020-12-08 17:34:59.349943
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql

PyFunceble/data/alembic/mysql/versions/83ada95132bf_delete_the_file_table.py

@@ -5,6 +5,7 @@ Revises: 459a0d7b8f09
 Create Date: 2020-12-07 12:49:48.797794
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql

PyFunceble/data/alembic/mysql/versions/912bbcb77a6c_add_registrar_column.py

@@ -5,6 +5,7 @@ Revises: 6f4729deaf03
 Create Date: 2021-12-04 23:52:11.861732
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/95dc17ddd729_introduction_of_the_session_id_column.py

@@ -5,6 +5,7 @@ Revises: bef7bcaac3f2
 Create Date: 2020-12-23 02:26:21.647125
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/ade87195b0a0_base.py

@@ -5,6 +5,7 @@ Revises:
 Create Date: 2020-08-15 20:12:26.768419
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 

PyFunceble/data/alembic/mysql/versions/bef7bcaac3f2_make_id_a_bigint.py

@@ -5,6 +5,7 @@ Revises: 3a4c55a9320d
 Create Date: 2020-12-16 19:09:41.212679
 
 """
+
 import sqlalchemy as sa
 from alembic import op
 from sqlalchemy.dialects import mysql