PyFunceble-dev 4.3.0a5__py3-none-any.whl → 4.3.0a10__py3-none-any.whl

PyFunceble/checker/availability/base.py

@@ -60,7 +60,6 @@ from sqlalchemy.orm import Session
 
 import PyFunceble.checker.utils.whois
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
 from PyFunceble.checker.availability.extras.dns import DNSRulesHandler

PyFunceble/checker/availability/domain.py

@@ -51,7 +51,6 @@ License:
 """
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.base import AvailabilityCheckerBase
 from PyFunceble.checker.reputation.domain import DomainReputationChecker

PyFunceble/checker/availability/extras/base.py

@@ -56,10 +56,11 @@ from typing import Callable, Dict, List, Optional, Union
 
 import requests
 
-import PyFunceble.factory
+import PyFunceble.storage
 from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
 from PyFunceble.helpers.regex import RegexHelper
 from PyFunceble.query.dns.query_tool import DNSQueryTool
+from PyFunceble.query.requests.requester import Requester
 
 
 class ExtraRuleHandlerBase:
@@ -76,13 +77,14 @@ class ExtraRuleHandlerBase:
     req: Optional[requests.Response] = None
     dns_query_tool: Optional[DNSQueryTool] = None
     regex_helper: Optional[RegexHelper] = None
+    requester: Optional[Requester] = None
 
     def __init__(self, status: Optional[AvailabilityCheckerStatus] = None) -> None:
         if status is not None:
             self.status = status
 
-        # Be sure that all settings are loaded proprely!!
-        PyFunceble.factory.Requester.guess_all_settings()
+        self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
+
         self.dns_query_tool = DNSQueryTool()
         self.regex_helper = RegexHelper()
 
@@ -227,9 +229,7 @@ class ExtraRuleHandlerBase:
             Whether we shoold follow the redirection - or not.
         """
 
-        self.req = PyFunceble.factory.Requester.get(
-            self.req_url, allow_redirects=allow_redirects
-        )
+        self.req = self.requester.get(self.req_url, allow_redirects=allow_redirects)
 
         return self
 
@@ -272,18 +272,18 @@ class ExtraRuleHandlerBase:
                 method()
 
         try:
-            req = PyFunceble.factory.Requester.get(url, allow_redirects=allow_redirects)
+            req = self.requester.get(url, allow_redirects=allow_redirects)
 
             if match_mode == "regex":
                 handle_regex_match_mode(req)
             else:
                 handle_string_match_mode(req)
         except (
-            PyFunceble.factory.Requester.exceptions.RequestException,
-            PyFunceble.factory.Requester.exceptions.InvalidURL,
-            PyFunceble.factory.Requester.exceptions.Timeout,
-            PyFunceble.factory.Requester.exceptions.ConnectionError,
-            PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
+            self.requester.exceptions.RequestException,
+            self.requester.exceptions.InvalidURL,
+            self.requester.exceptions.Timeout,
+            self.requester.exceptions.ConnectionError,
+            self.requester.urllib3_exceptions.InvalidHeader,
             socket.timeout,
         ):
             pass
@@ -363,18 +363,18 @@ class ExtraRuleHandlerBase:
                 method()
 
         try:
-            req = PyFunceble.factory.Requester.get(url, allow_redirects=allow_redirects)
+            req = self.requester.get(url, allow_redirects=allow_redirects)
 
             if match_mode == "regex":
                 handle_regex_match_mode(req)
             else:
                 handle_string_match_mode(req)
         except (
-            PyFunceble.factory.Requester.exceptions.RequestException,
-            PyFunceble.factory.Requester.exceptions.InvalidURL,
-            PyFunceble.factory.Requester.exceptions.Timeout,
-            PyFunceble.factory.Requester.exceptions.ConnectionError,
-            PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
+            self.requester.exceptions.RequestException,
+            self.requester.exceptions.InvalidURL,
+            self.requester.exceptions.Timeout,
+            self.requester.exceptions.ConnectionError,
+            self.requester.urllib3_exceptions.InvalidHeader,
             socket.timeout,
         ):
             pass

PyFunceble/checker/availability/extras/etoxic.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
 

PyFunceble/checker/availability/extras/parked.py

@@ -50,7 +50,6 @@ License:
     limitations under the License.
 """
 
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
 
@@ -127,7 +126,7 @@ class ParkedRulesHandler(ExtraRuleHandlerBase):
                 "Finished to check %r against our own set of parked rules.",
                 self.status.idna_subject,
             )
-        except PyFunceble.factory.Requester.exceptions.RequestException:
+        except self.requester.exceptions.RequestException:
             pass
 
         return self

PyFunceble/checker/availability/extras/rules.py

@@ -55,7 +55,6 @@ from typing import Optional
 from box import Box
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
 from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
@@ -81,6 +80,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
                 (self.switch_to_down_if_status_code, {410, 424}),
             ],
             r"\.24\.eu$": [(self.switch_to_down_if_status_code, 503)],
+            r"\.3x\.ro$": [(self.switch_to_down_if_status_code, 404)],
             r"\.altervista\.org$": [(self.switch_to_down_if_status_code, 403)],
             r"\.angelfire\.com$": [(self.switch_to_down_if_status_code, 404)],
             r"\.blogspot\.": [self.handle_blogspot],
@@ -90,14 +90,25 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
             r"\.github\.io$": [(self.switch_to_down_if_status_code, 404)],
             r"\.glitchz\.me$": [(self.switch_to_down_if_status_code, 403)],
             r"\.godaddysites\.com$": [(self.switch_to_down_if_status_code, 404)],
-            r"\.hpg.com.br$": [(self.switch_to_down_if_status_code, 404)],
+            r"\.hpg\.com\.br$": [(self.switch_to_down_if_status_code, 404)],
+            r"^heylink\.me$": [(self.switch_to_down_if_status_code, 410)],
+            r"^heyl\.ink$": [(self.switch_to_down_if_status_code, 410)],
             r"\.imgur\.com$": [self.handle_imgur_dot_com],
             r"\.liveadvert\.com$": [(self.switch_to_down_if_status_code, 404)],
             r"\.myhuaweicloudz\.com$": [(self.switch_to_down_if_status_code, 403)],
+            r"^scnv\.io$": [(self.switch_to_down_if_status_code, 404)],
             r"\.skyrock\.com$": [(self.switch_to_down_if_status_code, 404)],
+            r"\.squarespace.com$": [
+                (self.switch_to_down_if_status_code, 404),
+                self.handle_squarespace_com,
+            ],
             r"\.sz.id$": [(self.switch_to_down_if_status_code, 302)],
             r"\.translate\.goog$": [(self.switch_to_down_if_status_code, 403)],
             r"\.tumblr\.com$": [(self.switch_to_down_if_status_code, 404)],
+            r"\.vercel\.app$": [
+                (self.switch_to_down_if_status_code, "451"),
+                self.handle_vercel_dot_app,
+            ],
             r"\.web\.app$": [(self.switch_to_down_if_status_code, 404)],
             r"\.wix\.com$": [(self.switch_to_down_if_status_code, 404)],
             r"^s3\.ap-south-1\.amazonaws\.com$": [
@@ -216,9 +227,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
             This method are assuming we are handling a imgur.com subdomain.
         """
 
-        req = PyFunceble.factory.Requester.get(
-            self.req_url_https, allow_redirects=False
-        )
+        req = self.requester.get(self.req_url_https, allow_redirects=False)
         username = self.status.netloc.replace(".imgur.com", "")
 
         if "Location" in req.headers:
@@ -227,6 +236,56 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
 
         return self
 
+    def handle_vercel_dot_app(self) -> "ExtraRulesHandler":
+        """
+        Handles the :code:`vercel.app` case.
+
+        .. warning::
+            This method assume that we know that we are handling a vercel.app domain.
+        """
+
+        regex_vercel = [r"This%20Deployment%20has%20been%20disabled"]
+
+        self.do_on_header_match(
+            self.req_url,
+            {"x-vercel-error": ["DEPLOYMENT_DISABLED"]},
+            method=self.switch_to_down,
+            match_mode="std",
+            strict=True,
+            allow_redirects=False,
+        )
+
+        self.do_on_body_match(
+            self.req_url,
+            regex_vercel,
+            method=self.switch_to_down,
+            allow_redirects=False,
+        )
+
+        return self
+
+    def handle_squarespace_com(self) -> "ExtraRulesHandler":
+        """
+        Handles the :code:`wordpress.com` case.
+
+        .. warning::
+            This method assume that we know that we are handling a blogspot domain.
+        """
+
+        regex_squarespace = [
+            r"This site has been deleted"
+            r" by the owner\."
+        ]
+
+        self.do_on_body_match(
+            self.req_url,
+            regex_squarespace,
+            method=self.switch_to_down,
+            allow_redirects=True,
+        )
+
+        return self
+
     def __handle_active2inactive(self) -> "ExtraRulesHandler":
         """
         Handles the status deescalation.

PyFunceble/checker/availability/extras/subject_switch.py

@@ -146,7 +146,7 @@ class SubjectSwitchRulesHandler(ExtraRuleHandlerBase):
 
                 if not self.status.status_after_extra_rules:
                     self._switch_down_by_history()
-        except PyFunceble.factory.Requester.exceptions.RequestException:
+        except self.requester.exceptions.RequestException:
             pass
 
         PyFunceble.facility.Logger.info(

PyFunceble/checker/availability/ip.py

@@ -51,7 +51,6 @@ License:
 """
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.base import AvailabilityCheckerBase
 from PyFunceble.checker.reputation.ip import IPReputationChecker

PyFunceble/checker/availability/url.py

@@ -51,7 +51,6 @@ License:
 """
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.availability.base import AvailabilityCheckerBase
 from PyFunceble.checker.availability.status import AvailabilityCheckerStatus

PyFunceble/checker/reputation/base.py

@@ -55,7 +55,6 @@ from typing import List, Optional
 from sqlalchemy.orm import Session
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.base import CheckerBase
 from PyFunceble.checker.reputation.params import ReputationCheckerParams

PyFunceble/cli/processes/migrator.py

@@ -59,7 +59,6 @@ import PyFunceble.checker.utils.whois
 import PyFunceble.cli.utils.stdout
 import PyFunceble.cli.utils.testing
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
 from PyFunceble.cli.migrators.alembic import Alembic

PyFunceble/cli/processes/workers/base.py

@@ -62,7 +62,9 @@ import PyFunceble.cli.facility
 import PyFunceble.cli.factory
 import PyFunceble.facility
 import PyFunceble.sessions
+import PyFunceble.storage
 from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
+from PyFunceble.query.requests.requester import Requester
 
 
 class WorkerBase(multiprocessing.Process):
@@ -99,6 +101,7 @@ class WorkerBase(multiprocessing.Process):
     _exception: Optional[multiprocessing.Pipe] = None
 
     _params: Optional[dict] = {}
+    requester: Optional[Requester] = None
 
     def __init__(
         self,
@@ -149,6 +152,8 @@ class WorkerBase(multiprocessing.Process):
         A method which will be executed after the :code:`__init__` method.
         """
 
+        self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
+
     @property
     def exception(self):
         """
@@ -281,9 +286,6 @@ class WorkerBase(multiprocessing.Process):
             PyFunceble.cli.facility.CredentialLoader.start()
             PyFunceble.cli.factory.DBSession.init_db_sessions()
 
-        # Be sure that all settings are loaded proprely!!
-        PyFunceble.factory.Requester = PyFunceble.factory.requester()
-
         wait_for_stop = (
             bool(PyFunceble.storage.CONFIGURATION.cli_testing.mining) is True
         )

PyFunceble/cli/processes/workers/dir_files_sorter.py

@@ -58,7 +58,6 @@ from typing import Any, List, Optional, Tuple
 
 import PyFunceble.cli.storage
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.cli.processes.workers.file_sorter_base import FileSorterWorkerBase
 

PyFunceble/cli/processes/workers/file_sorter.py

@@ -54,7 +54,6 @@ from typing import Any, Optional, Tuple
 
 import PyFunceble.cli.storage
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.cli.processes.workers.file_sorter_base import FileSorterWorkerBase
 

PyFunceble/cli/processes/workers/file_sorter_base.py

@@ -60,7 +60,6 @@ from typing import Any, Generator, List, Tuple
 
 import PyFunceble.cli.storage
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.cli.filesystem.printer.file import FilePrinter
 from PyFunceble.cli.processes.workers.base import WorkerBase

PyFunceble/cli/processes/workers/migrator.py

@@ -53,7 +53,6 @@ License:
 import traceback
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.cli.processes.workers.base import WorkerBase
 

PyFunceble/cli/processes/workers/miner.py

@@ -57,7 +57,6 @@ from typing import List, Optional, Tuple
 from domain2idna import domain2idna
 
 import PyFunceble.facility
-import PyFunceble.factory
 import PyFunceble.storage
 from PyFunceble.checker.status_base import CheckerStatusBase
 from PyFunceble.cli.processes.workers.base import WorkerBase
@@ -86,8 +85,7 @@ class MinerWorker(WorkerBase):
 
         return super().__post_init__()
 
-    @staticmethod
-    def mine_from(subject: str) -> Optional[List[str]]:
+    def mine_from(self, subject: str) -> Optional[List[str]]:
         """
         Given the subject to work from, try to get the related subjects.
 
@@ -99,7 +97,7 @@ class MinerWorker(WorkerBase):
         result = []
 
         try:
-            req = PyFunceble.factory.Requester.get(subject, allow_redirects=True)
+            req = self.requester.get(subject, allow_redirects=True)
 
             for element in req.history:
                 if "location" in element.headers:
@@ -107,11 +105,11 @@ class MinerWorker(WorkerBase):
 
             result.extend([x for x in req.history if isinstance(x, str)])
         except (
-            PyFunceble.factory.Requester.exceptions.RequestException,
-            PyFunceble.factory.Requester.exceptions.ConnectionError,
-            PyFunceble.factory.Requester.exceptions.Timeout,
-            PyFunceble.factory.Requester.exceptions.InvalidURL,
-            PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
+            self.requester.exceptions.RequestException,
+            self.requester.exceptions.ConnectionError,
+            self.requester.exceptions.Timeout,
+            self.requester.exceptions.InvalidURL,
+            self.requester.urllib3_exceptions.InvalidHeader,
             socket.timeout,
         ):
             PyFunceble.facility.Logger.error(

PyFunceble/cli/processes/workers/tester.py

@@ -55,7 +55,6 @@ from typing import Any, Optional, Tuple
 
 import PyFunceble.cli.utils.testing
 import PyFunceble.facility
-import PyFunceble.factory
 from PyFunceble.checker.availability.domain_and_ip import DomainAndIPAvailabilityChecker
 from PyFunceble.checker.availability.url import URLAvailabilityChecker
 from PyFunceble.checker.base import CheckerBase
@@ -209,13 +208,10 @@ class TesterWorker(WorkerBase):
         queue.
         """
 
-        if (
-            PyFunceble.factory.Requester.session
-            and "Connection" not in PyFunceble.factory.Requester.session.headers
-        ):
+        if "Connection" not in self.requester.headers:
             # Just close the connection immediately. This prevent potential infinite
             # streams.
-            PyFunceble.factory.Requester.session.headers["Connection"] = "close"
+            self.requester.session.headers["Connection"] = "close"
 
         if not isinstance(consumed, dict):
             PyFunceble.facility.Logger.debug(

PyFunceble/cli/scripts/iana.py

@@ -54,6 +54,7 @@ import concurrent.futures
 from typing import Dict, Optional, Tuple
 
 import PyFunceble.facility
+import PyFunceble.storage
 from PyFunceble.dataset.iana import IanaDataset
 from PyFunceble.helpers.dict import DictHelper
 from PyFunceble.helpers.download import DownloadHelper
@@ -265,7 +266,16 @@ class IanaDBGenerator:
         """
 
         raw_data = (
-            DownloadHelper(self.UPSTREAM_LINK)
+            DownloadHelper(
+                self.UPSTREAM_LINK,
+                certificate_validation=(
+                    PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                    if PyFunceble.storage.CONFIGURATION
+                    else True
+                ),
+                own_proxy_handler=True,
+                proxies=PyFunceble.storage.PROXY,
+            )
             .download_text()
             .split('<span class="domain tld">')
         )

PyFunceble/cli/scripts/public_suffix.py

@@ -165,7 +165,20 @@ class PublicSuffixGenerator:
         Starts the generation of the dataset file.
         """
 
-        raw_data = DownloadHelper(self.UPSTREAM_LINK).download_text().split("\n")
+        raw_data = (
+            DownloadHelper(
+                self.UPSTREAM_LINK,
+                certificate_validation=(
+                    PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                    if PyFunceble.storage.CONFIGURATION
+                    else True
+                ),
+                own_proxy_handler=True,
+                proxies=PyFunceble.storage.PROXY,
+            )
+            .download_text()
+            .split("\n")
+        )
 
         with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
             for result in executor.map(self.parse_line, raw_data):

PyFunceble/cli/system/launcher.py

@@ -455,7 +455,16 @@ class SystemLauncher(SystemBase):
             """
 
             if URLSyntaxChecker(file).is_valid():
-                DownloadHelper(file).download_text(destination=destination)
+                DownloadHelper(
+                    file,
+                    certificate_validation=(
+                        PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                        if PyFunceble.storage.CONFIGURATION
+                        else True
+                    ),
+                    own_proxy_handler=True,
+                    proxies=PyFunceble.storage.PROXY,
+                ).download_text(destination=destination)
                 return True
             return False
 
@@ -651,18 +660,14 @@ class SystemLauncher(SystemBase):
 
             max_breakoff = 120.0
 
-            initial_breakoff = (
-                max_breakoff / PyFunceble.storage.CONFIGURATION.cli_testing.max_workers
-            )
+            initial_breakoff = 0.1 * self.tester_process_manager.max_worker
             breakoff = initial_breakoff
 
             while True:
                 protocol_data = {}
 
                 for next_contract in next(
-                    query_tool.pull_contract(
-                        PyFunceble.storage.CONFIGURATION.cli_testing.max_workers
-                    )
+                    query_tool.pull_contract(self.tester_process_manager.max_worker)
                 ):
                     if "subject" not in next_contract or not next_contract["subject"]:
                         continue
@@ -684,16 +689,24 @@ class SystemLauncher(SystemBase):
 
                 self.ci_stop_in_the_middle_if_time_exceeded()
 
-                if PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.dots:
-                    PyFunceble.cli.utils.stdout.print_single_line("S")
+                if (
+                    self.tester_process_manager.input_queue.qsize()
+                    >= self.tester_process_manager.max_worker
+                ):
+                    breakoff_multiplier = (
+                        self.tester_process_manager.input_queue.qsize() * 2
+                    )
 
-                if protocol_data:
-                    breakoff = initial_breakoff
-                elif breakoff < max_breakoff:
-                    breakoff += 0.02
-                else:
+                    if breakoff < max_breakoff:
+                        breakoff += 0.1 * breakoff_multiplier
+                    else:
+                        breakoff = initial_breakoff
+                elif breakoff >= max_breakoff:
                     breakoff = initial_breakoff
 
+                if PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.dots:
+                    PyFunceble.cli.utils.stdout.print_single_line("S")
+
                 time.sleep(breakoff)
 
         for protocol in self.testing_protocol:

PyFunceble/cli/utils/version.py

@@ -55,6 +55,7 @@ import sys
 from datetime import datetime, timezone
 
 import colorama
+import requests
 from box import Box
 
 import PyFunceble.cli.storage
@@ -72,14 +73,24 @@ def get_upstream_version() -> Box:
     Provides the state of the upstream version.
     """
 
+    try:
+        response = DownloadHelper(
+            InternalUrlConverter(
+                PyFunceble.cli.storage.VERSION_DUMP_LINK,
+            ).get_converted(),
+            own_proxy_handler=True,
+            proxies=PyFunceble.storage.PROXY,
+            certificate_validation=(
+                PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
+                if PyFunceble.storage.CONFIGURATION
+                else True
+            ),
+        ).download_text()
+    except requests.exceptions.RequestException:
+        response = "{}"
+
     return Box(
-        DictHelper().from_yaml(
-            DownloadHelper(
-                InternalUrlConverter(
-                    PyFunceble.cli.storage.VERSION_DUMP_LINK
-                ).get_converted()
-            ).download_text()
-        ),
+        DictHelper().from_yaml(response),
         frozen_box=True,
     )
 
@@ -346,16 +357,17 @@ def print_central_messages(check_force_update: bool = False) -> None:
 
     upstream_version = get_upstream_version()
 
-    if check_force_update:
-        handle_force_update(upstream_version)
+    if upstream_version:
+        if check_force_update:
+            handle_force_update(upstream_version)
 
-    _ = (
-        not handle_deprecated_version(upstream_version)
-        and not handle_greater_version(upstream_version)
-        and not handle_older_version(upstream_version)
-    )
+        _ = (
+            not handle_deprecated_version(upstream_version)
+            and not handle_greater_version(upstream_version)
+            and not handle_older_version(upstream_version)
+        )
 
-    handle_messages(upstream_version)
+        handle_messages(upstream_version)
 
     prefix = " - " if len(PyFunceble.cli.storage.EXTRA_MESSAGES) > 1 else ""