PyFunceble-dev 4.3.0a9__py3-none-any.whl → 4.3.0a11__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. PyFunceble/checker/availability/base.py +0 -1
  2. PyFunceble/checker/availability/domain.py +0 -1
  3. PyFunceble/checker/availability/extras/base.py +18 -18
  4. PyFunceble/checker/availability/extras/etoxic.py +0 -1
  5. PyFunceble/checker/availability/extras/parked.py +1 -2
  6. PyFunceble/checker/availability/extras/rules.py +64 -5
  7. PyFunceble/checker/availability/extras/subject_switch.py +1 -1
  8. PyFunceble/checker/availability/ip.py +0 -1
  9. PyFunceble/checker/availability/url.py +0 -1
  10. PyFunceble/checker/reputation/base.py +0 -1
  11. PyFunceble/cli/entry_points/pyfunceble/cli.py +19 -0
  12. PyFunceble/cli/filesystem/printer/base.py +19 -6
  13. PyFunceble/cli/processes/migrator.py +0 -1
  14. PyFunceble/cli/processes/workers/base.py +5 -3
  15. PyFunceble/cli/processes/workers/dir_files_sorter.py +0 -1
  16. PyFunceble/cli/processes/workers/file_sorter.py +0 -1
  17. PyFunceble/cli/processes/workers/file_sorter_base.py +0 -1
  18. PyFunceble/cli/processes/workers/migrator.py +0 -1
  19. PyFunceble/cli/processes/workers/miner.py +7 -9
  20. PyFunceble/cli/processes/workers/producer.py +15 -2
  21. PyFunceble/cli/processes/workers/tester.py +2 -6
  22. PyFunceble/cli/scripts/iana.py +11 -1
  23. PyFunceble/cli/scripts/public_suffix.py +14 -1
  24. PyFunceble/cli/system/launcher.py +10 -1
  25. PyFunceble/cli/utils/version.py +27 -15
  26. PyFunceble/config/loader.py +43 -12
  27. PyFunceble/data/infrastructure/.PyFunceble_production.yaml +14 -0
  28. PyFunceble/downloader/base.py +13 -3
  29. PyFunceble/helpers/download.py +147 -20
  30. PyFunceble/helpers/hash.py +10 -18
  31. PyFunceble/query/dns/nameserver.py +12 -6
  32. PyFunceble/query/dns/query_tool.py +3 -1
  33. PyFunceble/query/http_status_code.py +9 -7
  34. PyFunceble/query/requests/adapter/base.py +36 -4
  35. PyFunceble/query/requests/adapter/http.py +2 -3
  36. PyFunceble/query/requests/adapter/https.py +2 -2
  37. PyFunceble/query/requests/requester.py +70 -41
  38. PyFunceble/storage.py +1 -4
  39. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/METADATA +143 -150
  40. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/RECORD +44 -44
  41. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/WHEEL +1 -1
  42. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/LICENSE +0 -0
  43. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/entry_points.txt +0 -0
  44. {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/top_level.txt +0 -0
@@ -60,7 +60,6 @@ from sqlalchemy.orm import Session
60
60
 
61
61
  import PyFunceble.checker.utils.whois
62
62
  import PyFunceble.facility
63
- import PyFunceble.factory
64
63
  import PyFunceble.storage
65
64
  from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
66
65
  from PyFunceble.checker.availability.extras.dns import DNSRulesHandler
@@ -51,7 +51,6 @@ License:
51
51
  """
52
52
 
53
53
  import PyFunceble.facility
54
- import PyFunceble.factory
55
54
  import PyFunceble.storage
56
55
  from PyFunceble.checker.availability.base import AvailabilityCheckerBase
57
56
  from PyFunceble.checker.reputation.domain import DomainReputationChecker
@@ -56,10 +56,11 @@ from typing import Callable, Dict, List, Optional, Union
56
56
 
57
57
  import requests
58
58
 
59
- import PyFunceble.factory
59
+ import PyFunceble.storage
60
60
  from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
61
61
  from PyFunceble.helpers.regex import RegexHelper
62
62
  from PyFunceble.query.dns.query_tool import DNSQueryTool
63
+ from PyFunceble.query.requests.requester import Requester
63
64
 
64
65
 
65
66
  class ExtraRuleHandlerBase:
@@ -76,13 +77,14 @@ class ExtraRuleHandlerBase:
76
77
  req: Optional[requests.Response] = None
77
78
  dns_query_tool: Optional[DNSQueryTool] = None
78
79
  regex_helper: Optional[RegexHelper] = None
80
+ requester: Optional[Requester] = None
79
81
 
80
82
  def __init__(self, status: Optional[AvailabilityCheckerStatus] = None) -> None:
81
83
  if status is not None:
82
84
  self.status = status
83
85
 
84
- # Be sure that all settings are loaded proprely!!
85
- PyFunceble.factory.Requester.guess_all_settings()
86
+ self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
87
+
86
88
  self.dns_query_tool = DNSQueryTool()
87
89
  self.regex_helper = RegexHelper()
88
90
 
@@ -227,9 +229,7 @@ class ExtraRuleHandlerBase:
227
229
  Whether we shoold follow the redirection - or not.
228
230
  """
229
231
 
230
- self.req = PyFunceble.factory.Requester.get(
231
- self.req_url, allow_redirects=allow_redirects
232
- )
232
+ self.req = self.requester.get(self.req_url, allow_redirects=allow_redirects)
233
233
 
234
234
  return self
235
235
 
@@ -272,18 +272,18 @@ class ExtraRuleHandlerBase:
272
272
  method()
273
273
 
274
274
  try:
275
- req = PyFunceble.factory.Requester.get(url, allow_redirects=allow_redirects)
275
+ req = self.requester.get(url, allow_redirects=allow_redirects)
276
276
 
277
277
  if match_mode == "regex":
278
278
  handle_regex_match_mode(req)
279
279
  else:
280
280
  handle_string_match_mode(req)
281
281
  except (
282
- PyFunceble.factory.Requester.exceptions.RequestException,
283
- PyFunceble.factory.Requester.exceptions.InvalidURL,
284
- PyFunceble.factory.Requester.exceptions.Timeout,
285
- PyFunceble.factory.Requester.exceptions.ConnectionError,
286
- PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
282
+ self.requester.exceptions.RequestException,
283
+ self.requester.exceptions.InvalidURL,
284
+ self.requester.exceptions.Timeout,
285
+ self.requester.exceptions.ConnectionError,
286
+ self.requester.urllib3_exceptions.InvalidHeader,
287
287
  socket.timeout,
288
288
  ):
289
289
  pass
@@ -363,18 +363,18 @@ class ExtraRuleHandlerBase:
363
363
  method()
364
364
 
365
365
  try:
366
- req = PyFunceble.factory.Requester.get(url, allow_redirects=allow_redirects)
366
+ req = self.requester.get(url, allow_redirects=allow_redirects)
367
367
 
368
368
  if match_mode == "regex":
369
369
  handle_regex_match_mode(req)
370
370
  else:
371
371
  handle_string_match_mode(req)
372
372
  except (
373
- PyFunceble.factory.Requester.exceptions.RequestException,
374
- PyFunceble.factory.Requester.exceptions.InvalidURL,
375
- PyFunceble.factory.Requester.exceptions.Timeout,
376
- PyFunceble.factory.Requester.exceptions.ConnectionError,
377
- PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
373
+ self.requester.exceptions.RequestException,
374
+ self.requester.exceptions.InvalidURL,
375
+ self.requester.exceptions.Timeout,
376
+ self.requester.exceptions.ConnectionError,
377
+ self.requester.urllib3_exceptions.InvalidHeader,
378
378
  socket.timeout,
379
379
  ):
380
380
  pass
@@ -50,7 +50,6 @@ License:
50
50
  limitations under the License.
51
51
  """
52
52
 
53
- import PyFunceble.factory
54
53
  import PyFunceble.storage
55
54
  from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
56
55
 
@@ -50,7 +50,6 @@ License:
50
50
  limitations under the License.
51
51
  """
52
52
 
53
- import PyFunceble.factory
54
53
  import PyFunceble.storage
55
54
  from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
56
55
 
@@ -127,7 +126,7 @@ class ParkedRulesHandler(ExtraRuleHandlerBase):
127
126
  "Finished to check %r against our own set of parked rules.",
128
127
  self.status.idna_subject,
129
128
  )
130
- except PyFunceble.factory.Requester.exceptions.RequestException:
129
+ except self.requester.exceptions.RequestException:
131
130
  pass
132
131
 
133
132
  return self
@@ -55,7 +55,6 @@ from typing import Optional
55
55
  from box import Box
56
56
 
57
57
  import PyFunceble.facility
58
- import PyFunceble.factory
59
58
  import PyFunceble.storage
60
59
  from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
61
60
  from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
@@ -81,6 +80,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
81
80
  (self.switch_to_down_if_status_code, {410, 424}),
82
81
  ],
83
82
  r"\.24\.eu$": [(self.switch_to_down_if_status_code, 503)],
83
+ r"\.3x\.ro$": [(self.switch_to_down_if_status_code, 404)],
84
84
  r"\.altervista\.org$": [(self.switch_to_down_if_status_code, 403)],
85
85
  r"\.angelfire\.com$": [(self.switch_to_down_if_status_code, 404)],
86
86
  r"\.blogspot\.": [self.handle_blogspot],
@@ -90,14 +90,25 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
90
90
  r"\.github\.io$": [(self.switch_to_down_if_status_code, 404)],
91
91
  r"\.glitchz\.me$": [(self.switch_to_down_if_status_code, 403)],
92
92
  r"\.godaddysites\.com$": [(self.switch_to_down_if_status_code, 404)],
93
- r"\.hpg.com.br$": [(self.switch_to_down_if_status_code, 404)],
93
+ r"\.hpg\.com\.br$": [(self.switch_to_down_if_status_code, 404)],
94
+ r"^heylink\.me$": [(self.switch_to_down_if_status_code, 410)],
95
+ r"^heyl\.ink$": [(self.switch_to_down_if_status_code, 410)],
94
96
  r"\.imgur\.com$": [self.handle_imgur_dot_com],
95
97
  r"\.liveadvert\.com$": [(self.switch_to_down_if_status_code, 404)],
96
98
  r"\.myhuaweicloudz\.com$": [(self.switch_to_down_if_status_code, 403)],
99
+ r"^scnv\.io$": [(self.switch_to_down_if_status_code, 404)],
97
100
  r"\.skyrock\.com$": [(self.switch_to_down_if_status_code, 404)],
101
+ r"\.squarespace.com$": [
102
+ (self.switch_to_down_if_status_code, 404),
103
+ self.handle_squarespace_com,
104
+ ],
98
105
  r"\.sz.id$": [(self.switch_to_down_if_status_code, 302)],
99
106
  r"\.translate\.goog$": [(self.switch_to_down_if_status_code, 403)],
100
107
  r"\.tumblr\.com$": [(self.switch_to_down_if_status_code, 404)],
108
+ r"\.vercel\.app$": [
109
+ (self.switch_to_down_if_status_code, "451"),
110
+ self.handle_vercel_dot_app,
111
+ ],
101
112
  r"\.web\.app$": [(self.switch_to_down_if_status_code, 404)],
102
113
  r"\.wix\.com$": [(self.switch_to_down_if_status_code, 404)],
103
114
  r"^s3\.ap-south-1\.amazonaws\.com$": [
@@ -216,9 +227,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
216
227
  This method are assuming we are handling a imgur.com subdomain.
217
228
  """
218
229
 
219
- req = PyFunceble.factory.Requester.get(
220
- self.req_url_https, allow_redirects=False
221
- )
230
+ req = self.requester.get(self.req_url_https, allow_redirects=False)
222
231
  username = self.status.netloc.replace(".imgur.com", "")
223
232
 
224
233
  if "Location" in req.headers:
@@ -227,6 +236,56 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
227
236
 
228
237
  return self
229
238
 
239
+ def handle_vercel_dot_app(self) -> "ExtraRulesHandler":
240
+ """
241
+ Handles the :code:`vercel.app` case.
242
+
243
+ .. warning::
244
+ This method assume that we know that we are handling a vercel.app domain.
245
+ """
246
+
247
+ regex_vercel = [r"This%20Deployment%20has%20been%20disabled"]
248
+
249
+ self.do_on_header_match(
250
+ self.req_url,
251
+ {"x-vercel-error": ["DEPLOYMENT_DISABLED"]},
252
+ method=self.switch_to_down,
253
+ match_mode="std",
254
+ strict=True,
255
+ allow_redirects=False,
256
+ )
257
+
258
+ self.do_on_body_match(
259
+ self.req_url,
260
+ regex_vercel,
261
+ method=self.switch_to_down,
262
+ allow_redirects=False,
263
+ )
264
+
265
+ return self
266
+
267
+ def handle_squarespace_com(self) -> "ExtraRulesHandler":
268
+ """
269
+ Handles the :code:`wordpress.com` case.
270
+
271
+ .. warning::
272
+ This method assume that we know that we are handling a blogspot domain.
273
+ """
274
+
275
+ regex_squarespace = [
276
+ r"This site has been deleted"
277
+ r" by the owner\."
278
+ ]
279
+
280
+ self.do_on_body_match(
281
+ self.req_url,
282
+ regex_squarespace,
283
+ method=self.switch_to_down,
284
+ allow_redirects=True,
285
+ )
286
+
287
+ return self
288
+
230
289
  def __handle_active2inactive(self) -> "ExtraRulesHandler":
231
290
  """
232
291
  Handles the status deescalation.
@@ -146,7 +146,7 @@ class SubjectSwitchRulesHandler(ExtraRuleHandlerBase):
146
146
 
147
147
  if not self.status.status_after_extra_rules:
148
148
  self._switch_down_by_history()
149
- except PyFunceble.factory.Requester.exceptions.RequestException:
149
+ except self.requester.exceptions.RequestException:
150
150
  pass
151
151
 
152
152
  PyFunceble.facility.Logger.info(
@@ -51,7 +51,6 @@ License:
51
51
  """
52
52
 
53
53
  import PyFunceble.facility
54
- import PyFunceble.factory
55
54
  import PyFunceble.storage
56
55
  from PyFunceble.checker.availability.base import AvailabilityCheckerBase
57
56
  from PyFunceble.checker.reputation.ip import IPReputationChecker
@@ -51,7 +51,6 @@ License:
51
51
  """
52
52
 
53
53
  import PyFunceble.facility
54
- import PyFunceble.factory
55
54
  import PyFunceble.storage
56
55
  from PyFunceble.checker.availability.base import AvailabilityCheckerBase
57
56
  from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
@@ -55,7 +55,6 @@ from typing import List, Optional
55
55
  from sqlalchemy.orm import Session
56
56
 
57
57
  import PyFunceble.facility
58
- import PyFunceble.factory
59
58
  import PyFunceble.storage
60
59
  from PyFunceble.checker.base import CheckerBase
61
60
  from PyFunceble.checker.reputation.params import ReputationCheckerParams
@@ -833,6 +833,25 @@ def get_output_control_group_data() -> List[Tuple[List[str], dict]]:
833
833
  "default": "all",
834
834
  },
835
835
  ),
836
+ (
837
+ ["--display-datetime"],
838
+ {
839
+ "dest": "cli_testing.display_mode.datetime",
840
+ "action": "store_true",
841
+ "help": "Activates or disables the display of the datetime of the\n"
842
+ "test. %s" % get_configured_value("cli_testing.display_mode.datetime"),
843
+ },
844
+ ),
845
+ (
846
+ ["--display-datetime-fmt"],
847
+ {
848
+ "dest": "cli_testing.display_mode.datetime_format",
849
+ "type": str,
850
+ "help": "Sets the datetime format to use when displaying the\n"
851
+ "datetime of the test. %s"
852
+ % get_configured_value("cli_testing.display_mode.datetime_format"),
853
+ },
854
+ ),
836
855
  (
837
856
  [
838
857
  "--dots",
@@ -53,7 +53,7 @@ License:
53
53
  import copy
54
54
  import functools
55
55
  import string
56
- from typing import Dict, List, Optional
56
+ from typing import Any, Callable, Dict, List, Optional
57
57
 
58
58
 
59
59
  class PrinterBase:
@@ -81,14 +81,15 @@ class PrinterBase:
81
81
  "minutes": 2,
82
82
  "seconds": 6,
83
83
  "registrar": 30,
84
+ "tested_at": 19,
84
85
  }
85
86
 
86
87
  TEMPLATES: Dict[str, string.Template] = {
87
88
  "all": string.Template(
88
89
  "$idna_subject $status $status_source $expiration_date $registrar "
89
- "$http_status_code $checker_type"
90
+ "$http_status_code $checker_type $tested_at"
90
91
  ),
91
- "less": string.Template("$idna_subject $status $status_source"),
92
+ "less": string.Template("$idna_subject $status $status_source $tested_at"),
92
93
  "simple": string.Template("$idna_subject $status"),
93
94
  "percentage": string.Template("$status $percentage $amount"),
94
95
  "hosts": string.Template("$ip $idna_subject"),
@@ -114,8 +115,11 @@ class PrinterBase:
114
115
  "minutes": "Minutes",
115
116
  "seconds": "Seconds",
116
117
  "registrar": "Registrar",
118
+ "tested_at": "Tested At",
117
119
  }
118
120
 
121
+ extra_formatters: Dict[str, Callable[..., Any]] = {}
122
+
119
123
  _template_to_use: Optional[str] = None
120
124
  _dataset: Optional[Dict[str, str]] = None
121
125
  _skip_column: Optional[List[str]] = []
@@ -126,6 +130,7 @@ class PrinterBase:
126
130
  *,
127
131
  dataset: Optional[Dict[str, str]] = None,
128
132
  skip_column: Optional[List[str]] = None,
133
+ extra_formatters: Optional[Dict[str, Callable[..., Any]]] = None,
129
134
  ) -> None:
130
135
  if template_to_use is not None:
131
136
  self.template_to_use = template_to_use
@@ -136,6 +141,9 @@ class PrinterBase:
136
141
  if skip_column is not None:
137
142
  self.skip_column = skip_column
138
143
 
144
+ if extra_formatters is not None:
145
+ self.extra_formatters.update(extra_formatters)
146
+
139
147
  def ensure_template_to_use_is_given(func): # pylint: disable=no-self-argument
140
148
  """
141
149
  Ensures that the template to use is given before launching the
@@ -315,9 +323,11 @@ class PrinterBase:
315
323
  continue
316
324
 
317
325
  if key in self.skip_column:
318
- self.TEMPLATES[self.template_to_use].template = self.TEMPLATES[
319
- self.template_to_use
320
- ].template.replace(f"${key} ", "")
326
+ self.TEMPLATES[self.template_to_use].template = (
327
+ self.TEMPLATES[self.template_to_use]
328
+ .template.replace(f"${key} ", "")
329
+ .replace(f" ${key}", "")
330
+ )
321
331
  continue
322
332
 
323
333
  to_print_data[0][key] = f"{value:<{self.STD_LENGTH[key]}}"
@@ -354,6 +364,9 @@ class PrinterBase:
354
364
  if not value and value != 0:
355
365
  value = self.STD_UNKNOWN
356
366
 
367
+ if key in self.extra_formatters:
368
+ value = self.extra_formatters[key](value)
369
+
357
370
  if self.template_to_use not in ignore_length:
358
371
  to_print[key] = f"{value:<{self.STD_LENGTH[key]}}"
359
372
  else:
@@ -59,7 +59,6 @@ import PyFunceble.checker.utils.whois
59
59
  import PyFunceble.cli.utils.stdout
60
60
  import PyFunceble.cli.utils.testing
61
61
  import PyFunceble.facility
62
- import PyFunceble.factory
63
62
  import PyFunceble.storage
64
63
  from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
65
64
  from PyFunceble.cli.migrators.alembic import Alembic
@@ -62,7 +62,9 @@ import PyFunceble.cli.facility
62
62
  import PyFunceble.cli.factory
63
63
  import PyFunceble.facility
64
64
  import PyFunceble.sessions
65
+ import PyFunceble.storage
65
66
  from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
67
+ from PyFunceble.query.requests.requester import Requester
66
68
 
67
69
 
68
70
  class WorkerBase(multiprocessing.Process):
@@ -99,6 +101,7 @@ class WorkerBase(multiprocessing.Process):
99
101
  _exception: Optional[multiprocessing.Pipe] = None
100
102
 
101
103
  _params: Optional[dict] = {}
104
+ requester: Optional[Requester] = None
102
105
 
103
106
  def __init__(
104
107
  self,
@@ -149,6 +152,8 @@ class WorkerBase(multiprocessing.Process):
149
152
  A method which will be executed after the :code:`__init__` method.
150
153
  """
151
154
 
155
+ self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
156
+
152
157
  @property
153
158
  def exception(self):
154
159
  """
@@ -281,9 +286,6 @@ class WorkerBase(multiprocessing.Process):
281
286
  PyFunceble.cli.facility.CredentialLoader.start()
282
287
  PyFunceble.cli.factory.DBSession.init_db_sessions()
283
288
 
284
- # Be sure that all settings are loaded proprely!!
285
- PyFunceble.factory.Requester = PyFunceble.factory.requester()
286
-
287
289
  wait_for_stop = (
288
290
  bool(PyFunceble.storage.CONFIGURATION.cli_testing.mining) is True
289
291
  )
@@ -58,7 +58,6 @@ from typing import Any, List, Optional, Tuple
58
58
 
59
59
  import PyFunceble.cli.storage
60
60
  import PyFunceble.facility
61
- import PyFunceble.factory
62
61
  import PyFunceble.storage
63
62
  from PyFunceble.cli.processes.workers.file_sorter_base import FileSorterWorkerBase
64
63
 
@@ -54,7 +54,6 @@ from typing import Any, Optional, Tuple
54
54
 
55
55
  import PyFunceble.cli.storage
56
56
  import PyFunceble.facility
57
- import PyFunceble.factory
58
57
  import PyFunceble.storage
59
58
  from PyFunceble.cli.processes.workers.file_sorter_base import FileSorterWorkerBase
60
59
 
@@ -60,7 +60,6 @@ from typing import Any, Generator, List, Tuple
60
60
 
61
61
  import PyFunceble.cli.storage
62
62
  import PyFunceble.facility
63
- import PyFunceble.factory
64
63
  import PyFunceble.storage
65
64
  from PyFunceble.cli.filesystem.printer.file import FilePrinter
66
65
  from PyFunceble.cli.processes.workers.base import WorkerBase
@@ -53,7 +53,6 @@ License:
53
53
  import traceback
54
54
 
55
55
  import PyFunceble.facility
56
- import PyFunceble.factory
57
56
  import PyFunceble.storage
58
57
  from PyFunceble.cli.processes.workers.base import WorkerBase
59
58
 
@@ -57,7 +57,6 @@ from typing import List, Optional, Tuple
57
57
  from domain2idna import domain2idna
58
58
 
59
59
  import PyFunceble.facility
60
- import PyFunceble.factory
61
60
  import PyFunceble.storage
62
61
  from PyFunceble.checker.status_base import CheckerStatusBase
63
62
  from PyFunceble.cli.processes.workers.base import WorkerBase
@@ -86,8 +85,7 @@ class MinerWorker(WorkerBase):
86
85
 
87
86
  return super().__post_init__()
88
87
 
89
- @staticmethod
90
- def mine_from(subject: str) -> Optional[List[str]]:
88
+ def mine_from(self, subject: str) -> Optional[List[str]]:
91
89
  """
92
90
  Given the subject to work from, try to get the related subjects.
93
91
 
@@ -99,7 +97,7 @@ class MinerWorker(WorkerBase):
99
97
  result = []
100
98
 
101
99
  try:
102
- req = PyFunceble.factory.Requester.get(subject, allow_redirects=True)
100
+ req = self.requester.get(subject, allow_redirects=True)
103
101
 
104
102
  for element in req.history:
105
103
  if "location" in element.headers:
@@ -107,11 +105,11 @@ class MinerWorker(WorkerBase):
107
105
 
108
106
  result.extend([x for x in req.history if isinstance(x, str)])
109
107
  except (
110
- PyFunceble.factory.Requester.exceptions.RequestException,
111
- PyFunceble.factory.Requester.exceptions.ConnectionError,
112
- PyFunceble.factory.Requester.exceptions.Timeout,
113
- PyFunceble.factory.Requester.exceptions.InvalidURL,
114
- PyFunceble.factory.Requester.urllib3_exceptions.InvalidHeader,
108
+ self.requester.exceptions.RequestException,
109
+ self.requester.exceptions.ConnectionError,
110
+ self.requester.exceptions.Timeout,
111
+ self.requester.exceptions.InvalidURL,
112
+ self.requester.urllib3_exceptions.InvalidHeader,
115
113
  socket.timeout,
116
114
  ):
117
115
  PyFunceble.facility.Logger.error(
@@ -106,12 +106,25 @@ class ProducerWorker(WorkerBase):
106
106
 
107
107
  def __post_init__(self) -> None:
108
108
  skip_columns = []
109
+ extra_formatters = {}
109
110
 
110
111
  if not PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.registrar:
111
112
  skip_columns.append("registrar")
112
113
 
113
- self.stdout_printer = StdoutPrinter(skip_column=skip_columns)
114
- self.file_printer = FilePrinter(skip_column=skip_columns)
114
+ if not PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.datetime:
115
+ skip_columns.append("tested_at")
116
+ else:
117
+ # pylint: disable=line-too-long
118
+ extra_formatters["tested_at"] = lambda x: x.strftime(
119
+ PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.datetime_format
120
+ )
121
+
122
+ self.stdout_printer = StdoutPrinter(
123
+ skip_column=skip_columns, extra_formatters=extra_formatters
124
+ )
125
+ self.file_printer = FilePrinter(
126
+ skip_column=skip_columns, extra_formatters=extra_formatters
127
+ )
115
128
  self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
116
129
  self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
117
130
  self.continue_dataset = get_continue_databaset_object(
@@ -55,7 +55,6 @@ from typing import Any, Optional, Tuple
55
55
 
56
56
  import PyFunceble.cli.utils.testing
57
57
  import PyFunceble.facility
58
- import PyFunceble.factory
59
58
  from PyFunceble.checker.availability.domain_and_ip import DomainAndIPAvailabilityChecker
60
59
  from PyFunceble.checker.availability.url import URLAvailabilityChecker
61
60
  from PyFunceble.checker.base import CheckerBase
@@ -209,13 +208,10 @@ class TesterWorker(WorkerBase):
209
208
  queue.
210
209
  """
211
210
 
212
- if (
213
- PyFunceble.factory.Requester.session
214
- and "Connection" not in PyFunceble.factory.Requester.session.headers
215
- ):
211
+ if "Connection" not in self.requester.headers:
216
212
  # Just close the connection immediately. This prevent potential infinite
217
213
  # streams.
218
- PyFunceble.factory.Requester.session.headers["Connection"] = "close"
214
+ self.requester.session.headers["Connection"] = "close"
219
215
 
220
216
  if not isinstance(consumed, dict):
221
217
  PyFunceble.facility.Logger.debug(
@@ -54,6 +54,7 @@ import concurrent.futures
54
54
  from typing import Dict, Optional, Tuple
55
55
 
56
56
  import PyFunceble.facility
57
+ import PyFunceble.storage
57
58
  from PyFunceble.dataset.iana import IanaDataset
58
59
  from PyFunceble.helpers.dict import DictHelper
59
60
  from PyFunceble.helpers.download import DownloadHelper
@@ -265,7 +266,16 @@ class IanaDBGenerator:
265
266
  """
266
267
 
267
268
  raw_data = (
268
- DownloadHelper(self.UPSTREAM_LINK)
269
+ DownloadHelper(
270
+ self.UPSTREAM_LINK,
271
+ certificate_validation=(
272
+ PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
273
+ if PyFunceble.storage.CONFIGURATION
274
+ else True
275
+ ),
276
+ own_proxy_handler=True,
277
+ proxies=PyFunceble.storage.PROXY,
278
+ )
269
279
  .download_text()
270
280
  .split('<span class="domain tld">')
271
281
  )
@@ -165,7 +165,20 @@ class PublicSuffixGenerator:
165
165
  Starts the generation of the dataset file.
166
166
  """
167
167
 
168
- raw_data = DownloadHelper(self.UPSTREAM_LINK).download_text().split("\n")
168
+ raw_data = (
169
+ DownloadHelper(
170
+ self.UPSTREAM_LINK,
171
+ certificate_validation=(
172
+ PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
173
+ if PyFunceble.storage.CONFIGURATION
174
+ else True
175
+ ),
176
+ own_proxy_handler=True,
177
+ proxies=PyFunceble.storage.PROXY,
178
+ )
179
+ .download_text()
180
+ .split("\n")
181
+ )
169
182
 
170
183
  with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
171
184
  for result in executor.map(self.parse_line, raw_data):
@@ -455,7 +455,16 @@ class SystemLauncher(SystemBase):
455
455
  """
456
456
 
457
457
  if URLSyntaxChecker(file).is_valid():
458
- DownloadHelper(file).download_text(destination=destination)
458
+ DownloadHelper(
459
+ file,
460
+ certificate_validation=(
461
+ PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
462
+ if PyFunceble.storage.CONFIGURATION
463
+ else True
464
+ ),
465
+ own_proxy_handler=True,
466
+ proxies=PyFunceble.storage.PROXY,
467
+ ).download_text(destination=destination)
459
468
  return True
460
469
  return False
461
470