PyFunceble-dev 4.0.0b34__tar.gz → 4.0.0b36__tar.gz

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 1.2
 Name: PyFunceble-dev
-Version: 4.0.0b34
+Version: 4.0.0b36
 Summary: The tool to check the availability or syntax of domain, IP or URL.
 Home-page: https://github.com/funilrys/PyFunceble
 Author: funilrys

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/checker/availability/base.py

@@ -841,6 +841,8 @@ class AvailabilityCheckerBase(CheckerBase):
             lookup_result = self.addressinfo_query_tool.get_info()
         elif self.status.ip_syntax:
             lookup_result = self.hostbyaddr_query_tool.get_info()
+        elif self.status.idna_subject.isdigit():
+            lookup_result = None
         else:
             lookup_result = self.addressinfo_query_tool.get_info()
 

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/checker/syntax/base.py

@@ -122,5 +122,7 @@ class SyntaxCheckerBase(CheckerBase):
         raise NotImplementedError()
 
     # pylint: disable=useless-super-delegation
-    def get_status(self) -> Optional[SyntaxCheckerStatus]:
+    def get_status(
+        self,
+    ) -> Optional[SyntaxCheckerStatus]:  # pragma: no cover ## Safety.
         return super().get_status()

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/cli/entry_points/pyfunceble/cli.py

@@ -219,6 +219,15 @@ def get_filtering_group_data() -> List[Tuple[List[str], dict]]:
                 "help": argparse.SUPPRESS,
             },
         ),
+        (
+            ["--cidr"],
+            {
+                "dest": "cli_testing.cidr_expand",
+                "action": "store_true",
+                "help": "Activates or disables the expansion of CIDR formatted\n"
+                "addresses. %s" % get_configured_value("cli_testing.cidr_expand"),
+            },
+        ),
         (
             [
                 "--complements",

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/cli/file_preloader.py

@@ -66,6 +66,7 @@ from PyFunceble.cli.processes.workers.tester import TesterWorker
 from PyFunceble.cli.utils.stdout import print_single_line
 from PyFunceble.cli.utils.testing import get_subjects_from_line
 from PyFunceble.converter.adblock_input_line2subject import AdblockInputLine2Subject
+from PyFunceble.converter.cidr2subject import CIDR2Subject
 from PyFunceble.converter.input_line2subject import InputLine2Subject
 from PyFunceble.converter.rpz_input_line2subject import RPZInputLine2Subject
 from PyFunceble.converter.rpz_policy2subject import RPZPolicy2Subject
@@ -122,6 +123,7 @@ class FilePreloader:
     rpz_policy2subject: Optional[RPZPolicy2Subject] = None
     rpz_inputline2subject: Optional[RPZInputLine2Subject] = None
     url2netloc: Optional[Url2Netloc] = None
+    cidr2subject: Optional[CIDR2Subject] = None
 
     def __init__(
         self,
@@ -139,6 +141,7 @@ class FilePreloader:
         url2netloc: Optional[Url2Netloc] = None,
         continue_dataset: Optional[ContinueDatasetBase] = None,
         inactive_dataset: Optional[InactiveDatasetBase] = None,
+        cidr2subject: Optional[CIDR2Subject] = None,
     ) -> None:
         if authorized is not None:
             self.authorized = authorized
@@ -160,6 +163,7 @@ class FilePreloader:
         self.url2netloc = url2netloc
         self.continue_dataset = continue_dataset
         self.inactive_dataset = inactive_dataset
+        self.cidr2subject = cidr2subject
 
     def execute_if_authorized(default: Any = None):  # pylint: disable=no-self-argument
         """
@@ -394,6 +398,7 @@ class FilePreloader:
                             inputline2subject=self.inputline2subject,
                             subject2complements=self.subject2complements,
                             url2netloc=self.url2netloc,
+                            cidr2subject=self.cidr2subject,
                         ):
 
                             to_send = copy.deepcopy(self.protocol)

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/cli/processes/base.py

@@ -183,6 +183,21 @@ class ProcessesManagerBase:
 
         return wrapper
 
+    def ignore_if_running(func):  # pylint: disable=no-self-argument
+        """
+        Ignore the launching of the decorated method if the workers are
+        running.
+        """
+
+        @functools.wraps(func)
+        def wrapper(self, *args, **kwargs):
+            if not self.is_running():
+                return func(self, *args, **kwargs)  # pylint: disable=not-callable
+
+            return self
+
+        return wrapper
+
     @property
     def max_worker(self) -> Optional[int]:
         """
@@ -434,6 +449,7 @@ class ProcessesManagerBase:
 
     @ensure_worker_obj_is_given
     @create_workers_if_missing
+    # @ignore_if_running
     def start(self) -> "ProcessesManagerBase":
         """
         Starts a defined number of worker.

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/cli/system/launcher.py

@@ -101,6 +101,7 @@ from PyFunceble.cli.utils.testing import (
 )
 from PyFunceble.cli.utils.version import print_central_messages
 from PyFunceble.converter.adblock_input_line2subject import AdblockInputLine2Subject
+from PyFunceble.converter.cidr2subject import CIDR2Subject
 from PyFunceble.converter.input_line2subject import InputLine2Subject
 from PyFunceble.converter.rpz_input_line2subject import RPZInputLine2Subject
 from PyFunceble.converter.rpz_policy2subject import RPZPolicy2Subject
@@ -133,6 +134,7 @@ class SystemLauncher(SystemBase):
     rpz_policy2subject: RPZPolicy2Subject = RPZPolicy2Subject()
     rpz_inputline2subject: RPZInputLine2Subject = RPZInputLine2Subject()
     url2netloc: Url2Netloc = Url2Netloc()
+    cidr2subject: CIDR2Subject = CIDR2Subject()
 
     stdout_printer: StdoutPrinter = StdoutPrinter()
     file_printer: FilePrinter = FilePrinter()
@@ -528,6 +530,7 @@ class SystemLauncher(SystemBase):
                             inputline2subject=self.inputline2subject,
                             subject2complements=self.subject2complements,
                             url2netloc=self.url2netloc,
+                            cidr2subject=self.cidr2subject,
                         ):
                             to_send = copy.deepcopy(protocol)
                             to_send["subject"] = subject
@@ -575,6 +578,7 @@ class SystemLauncher(SystemBase):
                     inputline2subject=self.inputline2subject,
                     subject2complements=self.subject2complements,
                     url2netloc=self.url2netloc,
+                    cidr2subject=self.cidr2subject,
                 ):
                     to_send = copy.deepcopy(protocol)
                     to_send["subject"], to_send["idna_subject"] = (

{PyFunceble-dev-4.0.0b34 → PyFunceble-dev-4.0.0b36}/PyFunceble/cli/utils/testing.py

@@ -57,6 +57,7 @@ from sqlalchemy.orm import Session
 
 import PyFunceble.storage
 from PyFunceble.converter.adblock_input_line2subject import AdblockInputLine2Subject
+from PyFunceble.converter.cidr2subject import CIDR2Subject
 from PyFunceble.converter.input_line2subject import InputLine2Subject
 from PyFunceble.converter.rpz_input_line2subject import RPZInputLine2Subject
 from PyFunceble.converter.rpz_policy2subject import RPZPolicy2Subject
@@ -186,6 +187,7 @@ def get_subjects_from_line(
     inputline2subject: Optional[InputLine2Subject] = None,
     subject2complements: Optional[Subject2Complements] = None,
     url2netloc: Optional[Url2Netloc] = None,
+    cidr2subject: Optional[CIDR2Subject] = None,
 ) -> List[str]:
     """
     Provides the list of subject to test.
@@ -214,6 +216,9 @@ def get_subjects_from_line(
     if url2netloc is None:
         url2netloc = Url2Netloc()
 
+    if cidr2subject is None:
+        cidr2subject = CIDR2Subject()
+
     if PyFunceble.storage.CONFIGURATION.cli_decoding.adblock:
         result.extend(
             # pylint: disable=line-too-long
@@ -244,6 +249,13 @@ def get_subjects_from_line(
             ]
         )
 
+    if PyFunceble.storage.CONFIGURATION.cli_testing.cidr_expand:
+        result = [
+            y
+            for x in result
+            for y in cidr2subject.set_data_to_convert(x).get_converted()
+        ]
+
     if checker_type.lower() != "syntax":
         for index, subject in enumerate(result):
             if not subject:

PyFunceble-dev-4.0.0b36/PyFunceble/converter/adblock_input_line2subject.py

@@ -0,0 +1,434 @@
+# pylint:disable=line-too-long
+"""
+The tool to check the availability or syntax of domain, IP or URL.
+
+::
+
+
+    ██████╗ ██╗   ██╗███████╗██╗   ██╗███╗   ██╗ ██████╗███████╗██████╗ ██╗     ███████╗
+    ██╔══██╗╚██╗ ██╔╝██╔════╝██║   ██║████╗  ██║██╔════╝██╔════╝██╔══██╗██║     ██╔════╝
+    ██████╔╝ ╚████╔╝ █████╗  ██║   ██║██╔██╗ ██║██║     █████╗  ██████╔╝██║     █████╗
+    ██╔═══╝   ╚██╔╝  ██╔══╝  ██║   ██║██║╚██╗██║██║     ██╔══╝  ██╔══██╗██║     ██╔══╝
+    ██║        ██║   ██║     ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
+    ╚═╝        ╚═╝   ╚═╝      ╚═════╝ ╚═╝  ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝
+
+Provides the conversion of an AdBlock input line into testable subjests.
+
+Author:
+    Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom
+
+Special thanks:
+    https://pyfunceble.github.io/#/special-thanks
+
+Contributors:
+    https://pyfunceble.github.io/#/contributors
+
+Project link:
+    https://github.com/funilrys/PyFunceble
+
+Project documentation:
+    https://pyfunceble.readthedocs.io/en/dev/
+
+Project homepage:
+    https://pyfunceble.github.io/
+
+License:
+::
+
+
+    Copyright 2017, 2018, 2019, 2020, 2021 Nissar Chababy
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+        http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+"""
+
+from typing import Any, List, Optional, Set, Union
+
+from PyFunceble.converter.base import ConverterBase
+from PyFunceble.converter.url2netloc import Url2Netloc
+from PyFunceble.helpers.list import ListHelper
+from PyFunceble.helpers.regex import RegexHelper
+
+
+class AdblockInputLine2Subject(ConverterBase):
+    """
+    Provides an interface for the conversion or extraction of valuable subjects
+    from an inputted AdBlock line.
+    """
+
+    _aggressive: bool = False
+
+    __regex_helper: Optional[RegexHelper] = None
+
+    def __init__(
+        self, data_to_convert: Optional[Any] = None, aggressive: bool = False
+    ) -> None:
+        if aggressive is not None:
+            self.aggressive = aggressive
+
+        self.__regex_helper = RegexHelper()
+
+        super().__init__(data_to_convert=data_to_convert)
+
+    @ConverterBase.data_to_convert.setter
+    def data_to_convert(self, value: Any) -> None:
+        """
+        Overrites the default behavior.
+
+        :raise TypeError:
+            When the given data to convert is not :py:class:`str`
+        """
+
+        if not isinstance(value, str):
+            raise TypeError(f"<value> should be {str}, {type(value)} given.")
+
+        # pylint: disable=no-member
+        super(AdblockInputLine2Subject, self.__class__).data_to_convert.fset(
+            self, value
+        )
+
+    @property
+    def aggressive(self) -> bool:
+        """
+        Provides the state of the :code:`_aggressive` attribute.
+        """
+
+        return self._aggressive
+
+    @aggressive.setter
+    def aggressive(self, value: bool) -> None:
+        """
+        Provides a way to activate/deactivate the aggressive decoding.
+
+        :raise TypeError:
+            When the given data to convert is not :py:class:`str`
+        """
+
+        if not isinstance(value, bool):
+            raise TypeError(f"<value> should be {bool}, {type(value)} given.")
+
+        self._aggressive = value
+
+    def set_aggressive(self, value: bool) -> "AdblockInputLine2Subject":
+        """
+        Provides a way to activate/deactivate the aggressive decoding.
+        """
+
+        self.aggressive = value
+
+        return self
+
+    @staticmethod
+    def should_be_ignored(line: str) -> bool:
+        """
+        Checks if we should ignore the given line.
+        """
+
+        starting_chars = ["!", "@@", "/", "[", ".", "-", "_", "?", "&"]
+
+        return any(line.startswith(x) for x in starting_chars)
+
+    @staticmethod
+    def extract_base(subject: Union[str, List[str]]) -> Union[str, List[str]]:
+        """
+        Extracts the base of the given subject (supposely URL).
+
+        :param subject:
+            The subject to work with.
+
+        Example:
+
+            Giving :code:`"hello.world/?is=beautiful"` returns :code:`"hello.world"`
+        """
+
+        if isinstance(subject, list):
+            return [AdblockInputLine2Subject.extract_base(x) for x in subject]
+
+        subject = subject.replace("*", "").replace("~", "")
+
+        try:
+            return Url2Netloc(subject).get_converted()
+        except ValueError:
+            return subject
+
+    def _decode_multiple_subject(self, decoded: str) -> Set[str]:
+        """
+        Implementation of the decoding of the case that multiple
+        subjects are possible in the given :py:class:`str`.
+
+        :param decoded:
+            The decoded part to split.
+        """
+
+        result = set()
+
+        rematch = self.__regex_helper.set_regex(r"((?:[^~\*,]+))").match(
+            decoded, rematch=True, return_match=True
+        )
+
+        if rematch:
+            result.update({self.extract_base(x) for x in rematch})
+
+        return result
+
+    def _decode_options(self, decoded_options: List[str]) -> Set[str]:
+        """
+        Handle the decoding of the options.
+
+        What it does:
+
+            - It extracts all :code:`domain=` component - when found.
+            - It extracts all :code:`href` URL base - when found.
+
+
+        :param decoded_options:
+            The splitted list of options.
+        """
+
+        result = set()
+
+        for rule in decoded_options:
+            if "domain=" in rule:
+                rule = rule.replace("domain=", "").replace("|", ",")
+
+                result.update(self._decode_multiple_subject(rule))
+                continue
+
+            if "href" in rule:
+                matched = self.__regex_helper.set_regex(
+                    r"((?:\"|\')(.*)(?:\"|\'))"
+                ).match(rule, return_match=True, rematch=True, group=1)
+
+                result.add(self.extract_base(matched))
+                continue
+
+        return result
+
+    def _decode_v1(self, line: str) -> Set[str]:
+        """
+        Implementation of our first decoding mode.
+
+        In this mode we try to decode the simple:
+
+            ||ads.example.com^
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        result = set()
+
+        local_line = line.strip()
+
+        if local_line.startswith("||") and (
+            local_line.endswith("^") or local_line.endswith("$")
+        ):
+            local_line = local_line.replace("||", "", 1)
+
+            if local_line.endswith("^"):
+                local_line = "".join(local_line.rsplit("^", 1))
+            elif local_line.endswith("$"):
+                local_line = "".join(local_line.rsplit("$", 1))
+
+            result.update(self._decode_multiple_subject(local_line))
+
+        return {x for x in result if "." in x}
+
+    def _decode_v2(self, line: str) -> Set[str]:
+        """
+        Implementation of our second decoding mode.
+
+        In this mode, we try to decode the simple:
+
+            |https://ads.example.com|
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        result = set()
+
+        local_line = line.strip()
+
+        if local_line.startswith("|") and local_line.endswith("|"):
+            local_line = local_line.replace("|", "", 1)
+            local_line = "".join(local_line.rsplit("|", 1))
+
+            result.add(self.extract_base(local_line))
+
+        return {x for x in result if "." in x}
+
+    def _decode_v3(self, line: str) -> Set[str]:
+        """
+        Implementation of our third decoding mode.
+
+        In this mode, we try to decode the simple:
+
+            ||ads.example.com^$script,image,domain=example.com|~foo.example.info
+            ||ads.example.com$script,image,domain=example.com|~foo.example.info
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        result = set()
+
+        local_line = line.strip()
+
+        if not local_line.startswith("||"):
+            return result
+
+        if "$" in local_line:
+            v1_mode, options = local_line.split("$", 1)
+
+            if not v1_mode.endswith("^"):
+                v1_mode += "^"
+
+            result.update(self._decode_v1(v1_mode))
+
+            if self.aggressive:
+                result.update(self._decode_options(options.split(",")))
+        elif "^" not in local_line:
+            result.update(self._decode_v1(f"{local_line}^"))
+        else:
+            result.update(self._decode_v1(local_line[: local_line.find("^") + 1]))
+
+        return {x for x in result if "." in x}
+
+    def _decode_v4(self, line: str) -> Set[str]:
+        """
+        Implementation of our fourth decoding mode.
+
+        In this mode, we try to decode the simple:
+
+            @@||ads.example.com/notbanner^$~script
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        result = set()
+        local_line = line.strip()
+
+        if (
+            not self.aggressive
+            or not local_line.startswith("@@||")
+            or "^$" not in local_line
+        ):
+            return result
+
+        print(not local_line.startswith("@@||"), "^$" not in local_line)
+
+        v1_mode, options = local_line.split("$", 1)
+
+        result.update(
+            {self.extract_base(x) for x in self._decode_v1(v1_mode.replace("@@", ""))}
+        )
+
+        result.update(self._decode_options(options.split(",")))
+
+        return {x for x in result if "." in x}
+
+    def _decode_v5(self, line: str) -> Set[str]:
+        """
+        Implementation of our fifth decoding mode.
+
+        In this mode, we try to decode the simple:
+
+            example.com,example.net##.advert
+            exception.example.com#@#.advert
+            example.com,example.net#?#div:-abp-has(> div > img.advert)
+            exception.example.com#@#div:-abp-has(> div > img.advert)
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        local_line = line.strip()
+        result = set()
+
+        if not self.aggressive:
+            return result
+
+        separators = ["##", "#@#", "#?#"]
+
+        obj_of_interest, options = "", ""
+
+        for separator in separators:
+            if separator in local_line:
+                obj_of_interest, options = local_line.split(separator, 1)
+                break
+
+        result.update(self._decode_multiple_subject(obj_of_interest))
+        result.update(self._decode_options(options.split(",")))
+
+        return {x for x in result if "." in x}
+
+    def _decode_v6(self, line: str) -> Set[str]:
+        """
+        Implementation of our sixth decoding mode.
+
+        In this mode we try to decode the simple:
+
+            $domain=exam.pl|elpmaxe.pl|example.pl
+
+        rule.
+
+        :param line:
+            The line to decode.
+        """
+
+        local_line = line.strip()
+        result = set()
+
+        if not self.aggressive:
+            return result
+
+        separators = ["$"]
+
+        for separator in separators:
+            if not local_line.startswith(separator):
+                continue
+
+            options = local_line[len(separator) :]
+
+            result.update(self._decode_options(options.split(",")))
+
+        return {x for x in result if "." in x}
+
+    def get_converted(self) -> List[str]:
+        """
+        Provides the converted data.
+        """
+
+        result = set()
+
+        if not self.should_be_ignored(self.data_to_convert.strip()):
+            result.update(self._decode_v1(self.data_to_convert))
+            result.update(self._decode_v2(self.data_to_convert))
+            result.update(self._decode_v3(self.data_to_convert))
+            result.update(self._decode_v5(self.data_to_convert))
+            result.update(self._decode_v6(self.data_to_convert))
+
+        result.update(self._decode_v4(self.data_to_convert))
+
+        return ListHelper(list(result)).sort().subject