PyFunceble-dev 4.3.0a5__py3-none-any.whl → 4.3.0a10__py3-none-any.whl

PyFunceble/config/loader.py

@@ -66,6 +66,7 @@ from yaml.error import MarkedYAMLError
 import PyFunceble.cli.storage
 import PyFunceble.storage
 from PyFunceble.config.compare import ConfigComparison
+from PyFunceble.dataset.user_agent import UserAgentDataset
 from PyFunceble.downloader.iana import IANADownloader
 from PyFunceble.downloader.public_suffix import PublicSuffixDownloader
 from PyFunceble.downloader.user_agents import UserAgentsDownloader
@@ -438,19 +439,20 @@ class ConfigLoader:
                 else:
                     destination = dest
 
-                DownloadHelper(src).download_text(destination=destination)
-
-        if not self.is_already_loaded():
-            self.install_missing_infrastructure_files()
-            self.download_dynamic_infrastructure_files()
-            download_remote_config(
-                self.remote_config_location, self.path_to_remote_config
-            )
-            download_remote_config(self.path_to_config)
+                DownloadHelper(
+                    src,
+                    certificate_validation=(
+                        PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                        if PyFunceble.storage.CONFIGURATION
+                        else True
+                    ),
+                    own_proxy_handler=True,
+                    proxies=config["proxies"],
+                ).download_text(destination=destination)
 
         try:
             config = self.dict_helper.from_yaml_file(self.path_to_config)
-        except MarkedYAMLError:
+        except (MarkedYAMLError, FileNotFoundError):
             self.file_helper.set_path(self.path_to_default_config).copy(
                 self.path_to_config
             )
@@ -474,6 +476,32 @@ class ConfigLoader:
 
             self.dict_helper.set_subject(config).to_yaml_file(self.path_to_config)
 
+        if self.file_helper.set_path(self.path_to_overwrite_config).exists():
+            # Early load of the overwrite configuration to allow usage of defined
+            # proxy settings.
+            overwrite_data = self.dict_helper.from_yaml_file(
+                self.path_to_overwrite_config
+            )
+
+            if isinstance(overwrite_data, dict):
+                config = Merge(overwrite_data).into(config)
+        else:  # pragma: no cover  ## Just make it visible to end-user.
+            self.file_helper.write("")
+
+        # Now we preset the storage to enforce the usage of the configuration
+        # in any downloads.
+        PyFunceble.storage.CONFIGURATION = Box(
+            config,
+        )
+
+        if not self.is_already_loaded():
+            self.install_missing_infrastructure_files()
+            self.download_dynamic_infrastructure_files()
+            download_remote_config(
+                self.remote_config_location, self.path_to_remote_config
+            )
+            download_remote_config(self.path_to_config)
+
         if (
             self.path_to_remote_config
             and self.file_helper.set_path(self.path_to_remote_config).exists()
@@ -484,14 +512,14 @@ class ConfigLoader:
                 config = Merge(remote_data).into(config)
 
         if self.file_helper.set_path(self.path_to_overwrite_config).exists():
+            # Load the overwrite configuration again to ensure that user defined
+            # settings are always applied - last one wins.
             overwrite_data = self.dict_helper.from_yaml_file(
                 self.path_to_overwrite_config
             )
 
             if isinstance(overwrite_data, dict):
                 config = Merge(overwrite_data).into(config)
-        else:  # pragma: no cover  ## Just make it visible to end-user.
-            self.file_helper.write("")
 
         return config
 
@@ -562,6 +590,9 @@ class ConfigLoader:
         if "proxy" in config and config["proxy"]:
             PyFunceble.storage.PROXY = Box(config["proxy"])
 
+        # Early load user agents to allow usage of defined user agents.
+        UserAgentDataset().get_latest()
+
         return self
 
     def destroy(self, keep_custom: bool = False) -> "ConfigLoader":

PyFunceble/downloader/base.py

@@ -107,6 +107,7 @@ class DownloaderBase:
     _config_dir: Optional[str] = None
     _destination: Optional[str] = None
     _download_link: Optional[str] = None
+    _download_helper: Optional[DownloadHelper] = None
 
     dict_helper: Optional[DictHelper] = None
 
@@ -130,6 +131,17 @@ class DownloaderBase:
         if self.DEFAULT_FILENAME is not None:
             self.destination = os.path.join(self.config_dir, self.DEFAULT_FILENAME)
 
+        self._download_helper = DownloadHelper(
+            self.download_link,
+            own_proxy_handler=True,
+            proxies=PyFunceble.storage.PROXY,
+            certificate_validation=(
+                PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                if PyFunceble.storage.CONFIGURATION
+                else True
+            ),
+        )
+
     @property
     def authorized(self) -> bool:
         """
@@ -343,8 +355,6 @@ class DownloaderBase:
             if not hasattr(self, "download_link") or not self.download_link:
                 raise PyFunceble.downloader.exceptions.NoDownloadLinkGiven()
 
-            if DownloadHelper(self.download_link).download_text(
-                destination=self.destination
-            ):
+            if self._download_helper.download_text(destination=self.destination):
                 self.set_current_downtime()
                 self.save_all_downtimes()

PyFunceble/helpers/download.py

@@ -65,13 +65,29 @@ class DownloadHelper:
     Simplification of the downloads.
 
     :param str url:
-    :param int retry:
+    :param int retries:
         The number of time we have to retry before raising an exception.
+    :param bool certificate_validation:
+        The state of the certificate validation.
+    :param bool own_proxy_handler:
+        Whether we should use our own proxy handler or not.
+    :param dict proxies:
+        The proxy to use.
+
+        When :code:`own_proxy_handler` is set to :code:`True`, the proxies
+        are expected to come from the global configuration.
+        Otherwise, the proxies are expected to be a dictionary as defined
+        by the requests library.
     """
 
     _url: Optional[str] = None
     _certificate_validation: bool = True
     _retries: int = 3
+    _proxies: Optional[dict] = None
+
+    _session = None
+    _own_proxy_handler: Optional[bool] = True
+    _proxies: Optional[dict] = None
 
     def __init__(
         self,
@@ -79,15 +95,42 @@ class DownloadHelper:
         *,
         certificate_validation: bool = True,
         retries: int = 3,
+        own_proxy_handler: Optional[bool] = True,
+        proxies: Optional[dict] = None,
     ) -> None:
-        if url:
+        if url is not None:
             self.url = url
 
-        if certificate_validation:
-            self.certificate_validation = certificate_validation
+        if proxies is not None:
+            self.proxies = proxies
 
-        if retries:
-            self.retries = retries
+        self.retries = retries
+        self.certificate_validation = bool(certificate_validation)
+        self.own_proxy_handler = own_proxy_handler
+
+    @property
+    def session(self) -> requests.Session:
+        """
+        Provides the current state of the :code:`_session` attribute.
+        """
+
+        if not self._session:
+            if self.own_proxy_handler:
+                # pylint: disable=import-outside-toplevel
+                from PyFunceble.query.requests.requester import Requester
+
+                self._session = Requester(proxy_pattern=self.proxies)
+            else:
+                self._session = requests.Session()
+                self._session.proxies = self.proxies
+
+                retries = Retry(total=self.retries, backoff_factor=3)
+                adapter = HTTPAdapter(max_retries=retries)
+
+                self._session.mount("http://", adapter)
+                self._session.mount("https://", adapter)
+
+        return self._session
 
     @property
     def url(self) -> Optional[str]:
@@ -208,7 +251,103 @@ class DownloadHelper:
 
         return self
 
-    def download_text(self, *, destination: Optional[str] = None) -> str:
+    @property
+    def own_proxy_handler(self) -> Optional[bool]:
+        """
+        Provides the current state of the :code:`own_proxy_handler` attribute.
+        """
+
+        return self._own_proxy_handler
+
+    @own_proxy_handler.setter
+    def own_proxy_handler(self, value: bool) -> None:
+        """
+        Sets the state of the own proxy handler.
+
+        :param value:
+            The value to set.
+
+        :raise TypeError:
+            When :code:`value` is not a :py:class:`bool`.
+        """
+
+        if not isinstance(value, bool):
+            raise TypeError(f"<value> should be {bool}, {type(value)} given.")
+
+        self._own_proxy_handler = value
+
+        if value:
+            # We force the recreation of the session.
+            self._session = None
+
+    def set_own_proxy_handler(self, value: bool) -> "DownloadHelper":
+        """
+        Sets the state of the own proxy handler.
+
+        :param value:
+            The value to set.
+        """
+
+        self.own_proxy_handler = value
+
+        return self
+
+    @property
+    def proxies(self) -> Optional[dict]:
+        """
+        Provides the current state of the :code:`_proxies` attribute.
+        """
+
+        return self._proxies
+
+    @proxies.setter
+    def proxies(self, value: Optional[dict]) -> None:
+        """
+        Sets the proxy to use.
+
+        :param value:
+            The proxy to use.
+
+            When :code:`own_proxy_handler` is set to :code:`True`, the proxies
+            are expected to come from the global configuration.
+            Otherwise, the proxies are expected to be a dictionary as defined
+            by the requests library.
+
+        :raise TypeError:
+            When :code:`value` is not a :py:class:`dict`.
+        """
+
+        if not isinstance(value, dict):
+            raise TypeError(f"<value> should be {dict}, {type(value)} given.")
+
+        self._proxies = value
+
+        if value:
+            # We force the recreation of the session.
+            self._session = None
+
+    def set_proxies(self, value: Optional[dict]) -> "DownloadHelper":
+        """
+        Sets the proxy to use.
+
+        :param value:
+            The proxy to use.
+
+            When :code:`own_proxy_handler` is set to :code:`True`, the proxies
+            are expected to come from the global configuration.
+            Otherwise, the proxies are expected to be a dictionary as defined
+            by the requests library.
+        """
+
+        self.proxies = value
+
+        return self
+
+    def download_text(
+        self,
+        *,
+        destination: Optional[str] = None,
+    ) -> str:
         """
         Download the body of the set url.
 
@@ -224,15 +363,7 @@ class DownloadHelper:
         :raise UnableToDownload: When could not unable to download the URL.
         """
 
-        session = requests.Session()
-
-        retries = Retry(total=self.retries, backoff_factor=3)
-        adapter = HTTPAdapter(max_retries=retries)
-
-        session.mount("http://", adapter)
-        session.mount("https://", adapter)
-
-        req = session.get(self.url, verify=self.certificate_validation)
+        req = self.session.get(self.url, verify=self.certificate_validation)
 
         if req.status_code == 200:
             response = req.text
@@ -240,12 +371,8 @@ class DownloadHelper:
             if destination and isinstance(destination, str):
                 FileHelper(destination).write(req.text, overwrite=True)
 
-            adapter.close()
-            req.close()
             return response
 
-        adapter.close()
-        session.close()
         raise PyFunceble.helpers.exceptions.UnableToDownload(
             f"{req.url} (retries: {self.retries} | status code: {req.status_code})"
         )

PyFunceble/helpers/hash.py

@@ -50,11 +50,9 @@ License:
     limitations under the License.
 """
 
+import hashlib
 from typing import Optional, Union
 
-from cryptography.hazmat.backends import default_backend
-from cryptography.hazmat.primitives import hashes
-
 from PyFunceble.helpers.file import FileHelper
 
 
@@ -68,7 +66,7 @@ class HashHelper:
     :raise ValueError: When the given algo is not known.
     """
 
-    _algo: str = "SHA512_224"
+    _algo: str = "sha512_224"
 
     def __init__(self, algo: Optional[str] = None):
         if algo is not None:
@@ -97,11 +95,12 @@ class HashHelper:
         if not isinstance(value, str):
             raise TypeError(f"<value> should be {str}, {type(value)} given.")
 
-        value = value.upper()
+        value = value.lower()
 
-        if not hasattr(hashes, value):
+        if value not in hashlib.algorithms_available:
             raise ValueError(
-                f"<value> ({value!r}) in an unknown algorithm ({self.algo!r})."
+                f"<value> ({value!r}) in an unknown algorithm "
+                f"({hashlib.algorithms_available})."
             )
 
         self._algo = value
@@ -118,13 +117,6 @@ class HashHelper:
 
         return self
 
-    def __get_hash(self) -> hashes.Hash:
-        """
-        Provides the Hash to use.
-        """
-
-        return hashes.Hash(getattr(hashes, self.algo)(), backend=default_backend())
-
     def hash_file(self, file_path: str) -> str:
         """
         Hashes the content of the given file.
@@ -135,7 +127,7 @@ class HashHelper:
 
         block_size = 4096
 
-        digest = self.__get_hash()
+        digest = hashlib.new(self.algo)
 
         with FileHelper(file_path).open("rb") as file_stream:
             block = file_stream.read(block_size)
@@ -144,7 +136,7 @@ class HashHelper:
                 digest.update(block)
                 block = file_stream.read(block_size)
 
-        return digest.finalize().hex()
+        return digest.hexdigest()
 
     def hash_data(self, data: Union[str, bytes]) -> str:
         """
@@ -163,7 +155,7 @@ class HashHelper:
         if isinstance(data, str):
             data = data.encode()
 
-        digest = self.__get_hash()
+        digest = hashlib.new(self.algo)
         digest.update(data)
 
-        return digest.finalize().hex()
+        return digest.hexdigest()

PyFunceble/query/dns/nameserver.py

@@ -50,6 +50,7 @@ License:
     limitations under the License.
 """
 
+import ipaddress
 from typing import List, Optional, Tuple
 
 import dns.exception
@@ -79,15 +80,19 @@ class Nameservers:
 
     protocol: Optional[str] = None
 
-    domain_syntax_checker: DomainSyntaxChecker = DomainSyntaxChecker()
-    url_syntax_checker: URLSyntaxChecker = URLSyntaxChecker()
-    url2netloc: Url2Netloc = Url2Netloc()
+    domain_syntax_checker: Optional[DomainSyntaxChecker] = None
+    url_syntax_checker: Optional[URLSyntaxChecker] = None
+    url2netloc: Optional[Url2Netloc] = None
 
     def __init__(
         self, nameserver: Optional[List[str]] = None, protocol: str = "TCP"
     ) -> None:
         self.protocol = protocol
 
+        self.domain_syntax_checker = DomainSyntaxChecker()
+        self.url_syntax_checker = URLSyntaxChecker()
+        self.url2netloc = Url2Netloc()
+
         if nameserver is not None:
             self.set_nameservers(nameserver)
 
@@ -141,7 +146,10 @@ class Nameservers:
 
         result = []
 
-        if cls.domain_syntax_checker.set_subject(nameserver).is_valid():
+        try:
+            _ = ipaddress.ip_address(nameserver)
+            result.append(nameserver)
+        except ValueError:
             try:
                 result.extend(
                     [
@@ -161,8 +169,6 @@ class Nameservers:
                 )
             except dns.exception.DNSException:
                 pass
-        else:
-            result.append(nameserver)
 
         PyFunceble.facility.Logger.debug(
             "IP from nameserver (%r):\n%r", nameserver, result

PyFunceble/query/dns/query_tool.py

@@ -96,7 +96,7 @@ class DNSQueryTool:
         x.name: x.value for x in dns.rdatatype.RdataType
     }
 
-    nameservers: Nameservers = Nameservers()
+    nameservers: Optional[Nameservers] = None
     _query_record_type: int = dns.rdatatype.RdataType.ANY
 
     _subject: Optional[str] = None
@@ -120,6 +120,8 @@ class DNSQueryTool:
         trust_server: Optional[bool] = None,
         delay: Optional[bool] = None,
     ) -> None:
+        self.nameservers = Nameservers()
+
         if nameservers is not None:
             self.nameservers.set_nameservers(nameservers)
         else:  # pragma: no cover ## I'm not playing with system resolver.

PyFunceble/query/http_status_code.py

@@ -55,9 +55,9 @@ import socket
 from typing import Optional, Union
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.converter.url2netloc import Url2Netloc
+from PyFunceble.query.requests.requester import Requester
 
 
 class HTTPStatusCode:
@@ -75,6 +75,7 @@ class HTTPStatusCode:
     _verify_certificate: bool = True
     _allow_redirects: bool = False
     _url2netloc: Optional[Url2Netloc] = None
+    requester: Optional[Requester] = None
 
     def __init__(
         self,
@@ -103,6 +104,7 @@ class HTTPStatusCode:
             self.allow_redirects = self.STD_ALLOW_REDIRECTS
 
         self._url2netloc = Url2Netloc()
+        self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
 
     def ensure_subject_is_given(func):  # pylint: disable=no-self-argument
         """
@@ -334,7 +336,7 @@ class HTTPStatusCode:
         """  # pylint: disable=line-too-long
 
         try:
-            req = PyFunceble.factory.Requester.get(
+            req = self.requester.get(
                 self.subject,
                 timeout=self.timeout,
                 verify=self.verify_certificate,
@@ -363,12 +365,12 @@ class HTTPStatusCode:
 
             return req.status_code
         except (
-            PyFunceble.factory.Requester.exceptions.RequestException,
-            PyFunceble.factory.Requester.exceptions.InvalidSchema,
-            PyFunceble.factory.Requester.exceptions.InvalidURL,
-            PyFunceble.factory.Requester.exceptions.MissingSchema,
+            self.requester.exceptions.RequestException,
+            self.requester.exceptions.InvalidSchema,
+            self.requester.exceptions.InvalidURL,
+            self.requester.exceptions.MissingSchema,
             socket.timeout,
-            PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
+            self.requester.urllib3_exceptions.InvalidHeader,
         ):
             pass
 

PyFunceble/query/platform.py

@@ -702,9 +702,7 @@ class PlatformQueryTool:
             "shuffle": True,
         }
 
-        if "none" in self.checker_priority:
-            params["shuffle"] = True
-        else:
+        if "none" not in self.checker_priority:
             params["checker_type_priority"] = ",".join(self.checker_priority)
 
         if "none" not in self.checker_exclude:
@@ -717,9 +715,9 @@ class PlatformQueryTool:
                 timeout=self.timeout * 10,
             )
 
-            response_json = response.json()
-
             if response.status_code == 200:
+                response_json = response.json()
+
                 PyFunceble.facility.Logger.debug(
                     "Successfully pulled next %r contracts. Response: %r", response_json
                 )
@@ -772,9 +770,9 @@ class PlatformQueryTool:
                 timeout=self.timeout * 10,
             )
 
-            response_json = response.json()
+            if response.status_code in (202, 200):
+                response_json = response.json()
 
-            if response.status_code == 200:
                 PyFunceble.facility.Logger.debug(
                     "Successfully delivered contract: %r. Response: %r",
                     contract_data,
@@ -786,6 +784,7 @@ class PlatformQueryTool:
                 )
 
                 return response_json
+            response_json = {}
         except (requests.RequestException, json.decoder.JSONDecodeError):
             response_json = {}
 
@@ -796,7 +795,7 @@ class PlatformQueryTool:
             "Finished to deliver contract: %r", contract_data
         )
 
-        return None
+        return response_json
 
     @ensure_modern_api
     def push(

PyFunceble/query/requests/adapter/base.py

@@ -50,12 +50,13 @@ License:
     limitations under the License.
 """
 
+import secrets
 from typing import Optional
 
 import requests.adapters
+import requests.exceptions
 import requests.models
 
-import PyFunceble.storage
 from PyFunceble.checker.syntax.ip import IPSyntaxChecker
 from PyFunceble.query.dns.query_tool import DNSQueryTool
 
@@ -66,10 +67,15 @@ class RequestAdapterBase(requests.adapters.HTTPAdapter):
     adapter.
     """
 
+    NOT_RESOLVED_STD_HOSTNAME: str = (
+        f"{secrets.token_hex(12)}.mock-resolver.pyfunceble.com"
+    )
+
     resolving_cache: dict = {}
     resolving_use_cache: bool = False
     timeout: float = 5.0
     proxy_pattern: dict = {}
+    ssl_context: Optional[dict] = None
 
     def __init__(self, *args, **kwargs):
         if "timeout" in kwargs:
@@ -93,6 +99,10 @@ class RequestAdapterBase(requests.adapters.HTTPAdapter):
         else:
             self.proxy_pattern = {}
 
+        if "ssl_context" in kwargs:
+            self.ssl_context = kwargs["ssl_context"]
+            del kwargs["ssl_context"]
+
         super().__init__(*args, **kwargs)
 
     @staticmethod
@@ -102,9 +112,7 @@ class RequestAdapterBase(requests.adapters.HTTPAdapter):
         given domain.
         """
 
-        raise PyFunceble.factory.Requester.exceptions.ConnectionError(
-            "Could not resolve."
-        )
+        raise requests.exceptions.ConnectionError("Could not resolve.")
 
     @staticmethod
     def extract_extension(subject: str) -> Optional[str]:
@@ -286,3 +294,27 @@ class RequestAdapterBase(requests.adapters.HTTPAdapter):
                 return self.resolve_with_cache(hostname)
             return self.resolve_without_cache(hostname)
         return None
+
+    def init_poolmanager(self, connections, maxsize, block=False, **pool_kwargs):
+        """
+        Overwrite the upstream :code:`init_poolmanager` method to ensure that we
+        use our own ssl context - when given.
+        """
+
+        _ = pool_kwargs.pop("ssl_context", None)
+
+        return super().init_poolmanager(
+            connections, maxsize, block, ssl_context=self.ssl_context, **pool_kwargs
+        )
+
+    def proxy_manager_for(self, proxy, **proxy_kwargs):
+        """
+        Overwrite the upstream :code:`proxy_manager_for` method to ensure that we
+        use our own ssl context - when given.
+        """
+
+        _ = proxy_kwargs.pop("ssl_context", None)
+
+        return super().proxy_manager_for(
+            proxy, ssl_context=self.ssl_context, **proxy_kwargs
+        )