PyFunceble-dev 4.3.0a9__py3-none-any.whl → 4.3.0a11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyFunceble/checker/availability/base.py +0 -1
- PyFunceble/checker/availability/domain.py +0 -1
- PyFunceble/checker/availability/extras/base.py +18 -18
- PyFunceble/checker/availability/extras/etoxic.py +0 -1
- PyFunceble/checker/availability/extras/parked.py +1 -2
- PyFunceble/checker/availability/extras/rules.py +64 -5
- PyFunceble/checker/availability/extras/subject_switch.py +1 -1
- PyFunceble/checker/availability/ip.py +0 -1
- PyFunceble/checker/availability/url.py +0 -1
- PyFunceble/checker/reputation/base.py +0 -1
- PyFunceble/cli/entry_points/pyfunceble/cli.py +19 -0
- PyFunceble/cli/filesystem/printer/base.py +19 -6
- PyFunceble/cli/processes/migrator.py +0 -1
- PyFunceble/cli/processes/workers/base.py +5 -3
- PyFunceble/cli/processes/workers/dir_files_sorter.py +0 -1
- PyFunceble/cli/processes/workers/file_sorter.py +0 -1
- PyFunceble/cli/processes/workers/file_sorter_base.py +0 -1
- PyFunceble/cli/processes/workers/migrator.py +0 -1
- PyFunceble/cli/processes/workers/miner.py +7 -9
- PyFunceble/cli/processes/workers/producer.py +15 -2
- PyFunceble/cli/processes/workers/tester.py +2 -6
- PyFunceble/cli/scripts/iana.py +11 -1
- PyFunceble/cli/scripts/public_suffix.py +14 -1
- PyFunceble/cli/system/launcher.py +10 -1
- PyFunceble/cli/utils/version.py +27 -15
- PyFunceble/config/loader.py +43 -12
- PyFunceble/data/infrastructure/.PyFunceble_production.yaml +14 -0
- PyFunceble/downloader/base.py +13 -3
- PyFunceble/helpers/download.py +147 -20
- PyFunceble/helpers/hash.py +10 -18
- PyFunceble/query/dns/nameserver.py +12 -6
- PyFunceble/query/dns/query_tool.py +3 -1
- PyFunceble/query/http_status_code.py +9 -7
- PyFunceble/query/requests/adapter/base.py +36 -4
- PyFunceble/query/requests/adapter/http.py +2 -3
- PyFunceble/query/requests/adapter/https.py +2 -2
- PyFunceble/query/requests/requester.py +70 -41
- PyFunceble/storage.py +1 -4
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/METADATA +143 -150
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/RECORD +44 -44
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/WHEEL +1 -1
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/LICENSE +0 -0
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/entry_points.txt +0 -0
- {PyFunceble_dev-4.3.0a9.dist-info → PyFunceble_dev-4.3.0a11.dist-info}/top_level.txt +0 -0
@@ -60,7 +60,6 @@ from sqlalchemy.orm import Session
|
|
60
60
|
|
61
61
|
import PyFunceble.checker.utils.whois
|
62
62
|
import PyFunceble.facility
|
63
|
-
import PyFunceble.factory
|
64
63
|
import PyFunceble.storage
|
65
64
|
from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
|
66
65
|
from PyFunceble.checker.availability.extras.dns import DNSRulesHandler
|
@@ -56,10 +56,11 @@ from typing import Callable, Dict, List, Optional, Union
|
|
56
56
|
|
57
57
|
import requests
|
58
58
|
|
59
|
-
import PyFunceble.
|
59
|
+
import PyFunceble.storage
|
60
60
|
from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
|
61
61
|
from PyFunceble.helpers.regex import RegexHelper
|
62
62
|
from PyFunceble.query.dns.query_tool import DNSQueryTool
|
63
|
+
from PyFunceble.query.requests.requester import Requester
|
63
64
|
|
64
65
|
|
65
66
|
class ExtraRuleHandlerBase:
|
@@ -76,13 +77,14 @@ class ExtraRuleHandlerBase:
|
|
76
77
|
req: Optional[requests.Response] = None
|
77
78
|
dns_query_tool: Optional[DNSQueryTool] = None
|
78
79
|
regex_helper: Optional[RegexHelper] = None
|
80
|
+
requester: Optional[Requester] = None
|
79
81
|
|
80
82
|
def __init__(self, status: Optional[AvailabilityCheckerStatus] = None) -> None:
|
81
83
|
if status is not None:
|
82
84
|
self.status = status
|
83
85
|
|
84
|
-
|
85
|
-
|
86
|
+
self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
|
87
|
+
|
86
88
|
self.dns_query_tool = DNSQueryTool()
|
87
89
|
self.regex_helper = RegexHelper()
|
88
90
|
|
@@ -227,9 +229,7 @@ class ExtraRuleHandlerBase:
|
|
227
229
|
Whether we shoold follow the redirection - or not.
|
228
230
|
"""
|
229
231
|
|
230
|
-
self.req =
|
231
|
-
self.req_url, allow_redirects=allow_redirects
|
232
|
-
)
|
232
|
+
self.req = self.requester.get(self.req_url, allow_redirects=allow_redirects)
|
233
233
|
|
234
234
|
return self
|
235
235
|
|
@@ -272,18 +272,18 @@ class ExtraRuleHandlerBase:
|
|
272
272
|
method()
|
273
273
|
|
274
274
|
try:
|
275
|
-
req =
|
275
|
+
req = self.requester.get(url, allow_redirects=allow_redirects)
|
276
276
|
|
277
277
|
if match_mode == "regex":
|
278
278
|
handle_regex_match_mode(req)
|
279
279
|
else:
|
280
280
|
handle_string_match_mode(req)
|
281
281
|
except (
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
282
|
+
self.requester.exceptions.RequestException,
|
283
|
+
self.requester.exceptions.InvalidURL,
|
284
|
+
self.requester.exceptions.Timeout,
|
285
|
+
self.requester.exceptions.ConnectionError,
|
286
|
+
self.requester.urllib3_exceptions.InvalidHeader,
|
287
287
|
socket.timeout,
|
288
288
|
):
|
289
289
|
pass
|
@@ -363,18 +363,18 @@ class ExtraRuleHandlerBase:
|
|
363
363
|
method()
|
364
364
|
|
365
365
|
try:
|
366
|
-
req =
|
366
|
+
req = self.requester.get(url, allow_redirects=allow_redirects)
|
367
367
|
|
368
368
|
if match_mode == "regex":
|
369
369
|
handle_regex_match_mode(req)
|
370
370
|
else:
|
371
371
|
handle_string_match_mode(req)
|
372
372
|
except (
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
|
377
|
-
|
373
|
+
self.requester.exceptions.RequestException,
|
374
|
+
self.requester.exceptions.InvalidURL,
|
375
|
+
self.requester.exceptions.Timeout,
|
376
|
+
self.requester.exceptions.ConnectionError,
|
377
|
+
self.requester.urllib3_exceptions.InvalidHeader,
|
378
378
|
socket.timeout,
|
379
379
|
):
|
380
380
|
pass
|
@@ -50,7 +50,6 @@ License:
|
|
50
50
|
limitations under the License.
|
51
51
|
"""
|
52
52
|
|
53
|
-
import PyFunceble.factory
|
54
53
|
import PyFunceble.storage
|
55
54
|
from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
|
56
55
|
|
@@ -127,7 +126,7 @@ class ParkedRulesHandler(ExtraRuleHandlerBase):
|
|
127
126
|
"Finished to check %r against our own set of parked rules.",
|
128
127
|
self.status.idna_subject,
|
129
128
|
)
|
130
|
-
except
|
129
|
+
except self.requester.exceptions.RequestException:
|
131
130
|
pass
|
132
131
|
|
133
132
|
return self
|
@@ -55,7 +55,6 @@ from typing import Optional
|
|
55
55
|
from box import Box
|
56
56
|
|
57
57
|
import PyFunceble.facility
|
58
|
-
import PyFunceble.factory
|
59
58
|
import PyFunceble.storage
|
60
59
|
from PyFunceble.checker.availability.extras.base import ExtraRuleHandlerBase
|
61
60
|
from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
|
@@ -81,6 +80,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
|
|
81
80
|
(self.switch_to_down_if_status_code, {410, 424}),
|
82
81
|
],
|
83
82
|
r"\.24\.eu$": [(self.switch_to_down_if_status_code, 503)],
|
83
|
+
r"\.3x\.ro$": [(self.switch_to_down_if_status_code, 404)],
|
84
84
|
r"\.altervista\.org$": [(self.switch_to_down_if_status_code, 403)],
|
85
85
|
r"\.angelfire\.com$": [(self.switch_to_down_if_status_code, 404)],
|
86
86
|
r"\.blogspot\.": [self.handle_blogspot],
|
@@ -90,14 +90,25 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
|
|
90
90
|
r"\.github\.io$": [(self.switch_to_down_if_status_code, 404)],
|
91
91
|
r"\.glitchz\.me$": [(self.switch_to_down_if_status_code, 403)],
|
92
92
|
r"\.godaddysites\.com$": [(self.switch_to_down_if_status_code, 404)],
|
93
|
-
r"\.hpg
|
93
|
+
r"\.hpg\.com\.br$": [(self.switch_to_down_if_status_code, 404)],
|
94
|
+
r"^heylink\.me$": [(self.switch_to_down_if_status_code, 410)],
|
95
|
+
r"^heyl\.ink$": [(self.switch_to_down_if_status_code, 410)],
|
94
96
|
r"\.imgur\.com$": [self.handle_imgur_dot_com],
|
95
97
|
r"\.liveadvert\.com$": [(self.switch_to_down_if_status_code, 404)],
|
96
98
|
r"\.myhuaweicloudz\.com$": [(self.switch_to_down_if_status_code, 403)],
|
99
|
+
r"^scnv\.io$": [(self.switch_to_down_if_status_code, 404)],
|
97
100
|
r"\.skyrock\.com$": [(self.switch_to_down_if_status_code, 404)],
|
101
|
+
r"\.squarespace.com$": [
|
102
|
+
(self.switch_to_down_if_status_code, 404),
|
103
|
+
self.handle_squarespace_com,
|
104
|
+
],
|
98
105
|
r"\.sz.id$": [(self.switch_to_down_if_status_code, 302)],
|
99
106
|
r"\.translate\.goog$": [(self.switch_to_down_if_status_code, 403)],
|
100
107
|
r"\.tumblr\.com$": [(self.switch_to_down_if_status_code, 404)],
|
108
|
+
r"\.vercel\.app$": [
|
109
|
+
(self.switch_to_down_if_status_code, "451"),
|
110
|
+
self.handle_vercel_dot_app,
|
111
|
+
],
|
101
112
|
r"\.web\.app$": [(self.switch_to_down_if_status_code, 404)],
|
102
113
|
r"\.wix\.com$": [(self.switch_to_down_if_status_code, 404)],
|
103
114
|
r"^s3\.ap-south-1\.amazonaws\.com$": [
|
@@ -216,9 +227,7 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
|
|
216
227
|
This method are assuming we are handling a imgur.com subdomain.
|
217
228
|
"""
|
218
229
|
|
219
|
-
req =
|
220
|
-
self.req_url_https, allow_redirects=False
|
221
|
-
)
|
230
|
+
req = self.requester.get(self.req_url_https, allow_redirects=False)
|
222
231
|
username = self.status.netloc.replace(".imgur.com", "")
|
223
232
|
|
224
233
|
if "Location" in req.headers:
|
@@ -227,6 +236,56 @@ class ExtraRulesHandler(ExtraRuleHandlerBase):
|
|
227
236
|
|
228
237
|
return self
|
229
238
|
|
239
|
+
def handle_vercel_dot_app(self) -> "ExtraRulesHandler":
|
240
|
+
"""
|
241
|
+
Handles the :code:`vercel.app` case.
|
242
|
+
|
243
|
+
.. warning::
|
244
|
+
This method assume that we know that we are handling a vercel.app domain.
|
245
|
+
"""
|
246
|
+
|
247
|
+
regex_vercel = [r"This%20Deployment%20has%20been%20disabled"]
|
248
|
+
|
249
|
+
self.do_on_header_match(
|
250
|
+
self.req_url,
|
251
|
+
{"x-vercel-error": ["DEPLOYMENT_DISABLED"]},
|
252
|
+
method=self.switch_to_down,
|
253
|
+
match_mode="std",
|
254
|
+
strict=True,
|
255
|
+
allow_redirects=False,
|
256
|
+
)
|
257
|
+
|
258
|
+
self.do_on_body_match(
|
259
|
+
self.req_url,
|
260
|
+
regex_vercel,
|
261
|
+
method=self.switch_to_down,
|
262
|
+
allow_redirects=False,
|
263
|
+
)
|
264
|
+
|
265
|
+
return self
|
266
|
+
|
267
|
+
def handle_squarespace_com(self) -> "ExtraRulesHandler":
|
268
|
+
"""
|
269
|
+
Handles the :code:`wordpress.com` case.
|
270
|
+
|
271
|
+
.. warning::
|
272
|
+
This method assume that we know that we are handling a blogspot domain.
|
273
|
+
"""
|
274
|
+
|
275
|
+
regex_squarespace = [
|
276
|
+
r"This site has been deleted"
|
277
|
+
r" by the owner\."
|
278
|
+
]
|
279
|
+
|
280
|
+
self.do_on_body_match(
|
281
|
+
self.req_url,
|
282
|
+
regex_squarespace,
|
283
|
+
method=self.switch_to_down,
|
284
|
+
allow_redirects=True,
|
285
|
+
)
|
286
|
+
|
287
|
+
return self
|
288
|
+
|
230
289
|
def __handle_active2inactive(self) -> "ExtraRulesHandler":
|
231
290
|
"""
|
232
291
|
Handles the status deescalation.
|
@@ -146,7 +146,7 @@ class SubjectSwitchRulesHandler(ExtraRuleHandlerBase):
|
|
146
146
|
|
147
147
|
if not self.status.status_after_extra_rules:
|
148
148
|
self._switch_down_by_history()
|
149
|
-
except
|
149
|
+
except self.requester.exceptions.RequestException:
|
150
150
|
pass
|
151
151
|
|
152
152
|
PyFunceble.facility.Logger.info(
|
@@ -51,7 +51,6 @@ License:
|
|
51
51
|
"""
|
52
52
|
|
53
53
|
import PyFunceble.facility
|
54
|
-
import PyFunceble.factory
|
55
54
|
import PyFunceble.storage
|
56
55
|
from PyFunceble.checker.availability.base import AvailabilityCheckerBase
|
57
56
|
from PyFunceble.checker.availability.status import AvailabilityCheckerStatus
|
@@ -55,7 +55,6 @@ from typing import List, Optional
|
|
55
55
|
from sqlalchemy.orm import Session
|
56
56
|
|
57
57
|
import PyFunceble.facility
|
58
|
-
import PyFunceble.factory
|
59
58
|
import PyFunceble.storage
|
60
59
|
from PyFunceble.checker.base import CheckerBase
|
61
60
|
from PyFunceble.checker.reputation.params import ReputationCheckerParams
|
@@ -833,6 +833,25 @@ def get_output_control_group_data() -> List[Tuple[List[str], dict]]:
|
|
833
833
|
"default": "all",
|
834
834
|
},
|
835
835
|
),
|
836
|
+
(
|
837
|
+
["--display-datetime"],
|
838
|
+
{
|
839
|
+
"dest": "cli_testing.display_mode.datetime",
|
840
|
+
"action": "store_true",
|
841
|
+
"help": "Activates or disables the display of the datetime of the\n"
|
842
|
+
"test. %s" % get_configured_value("cli_testing.display_mode.datetime"),
|
843
|
+
},
|
844
|
+
),
|
845
|
+
(
|
846
|
+
["--display-datetime-fmt"],
|
847
|
+
{
|
848
|
+
"dest": "cli_testing.display_mode.datetime_format",
|
849
|
+
"type": str,
|
850
|
+
"help": "Sets the datetime format to use when displaying the\n"
|
851
|
+
"datetime of the test. %s"
|
852
|
+
% get_configured_value("cli_testing.display_mode.datetime_format"),
|
853
|
+
},
|
854
|
+
),
|
836
855
|
(
|
837
856
|
[
|
838
857
|
"--dots",
|
@@ -53,7 +53,7 @@ License:
|
|
53
53
|
import copy
|
54
54
|
import functools
|
55
55
|
import string
|
56
|
-
from typing import Dict, List, Optional
|
56
|
+
from typing import Any, Callable, Dict, List, Optional
|
57
57
|
|
58
58
|
|
59
59
|
class PrinterBase:
|
@@ -81,14 +81,15 @@ class PrinterBase:
|
|
81
81
|
"minutes": 2,
|
82
82
|
"seconds": 6,
|
83
83
|
"registrar": 30,
|
84
|
+
"tested_at": 19,
|
84
85
|
}
|
85
86
|
|
86
87
|
TEMPLATES: Dict[str, string.Template] = {
|
87
88
|
"all": string.Template(
|
88
89
|
"$idna_subject $status $status_source $expiration_date $registrar "
|
89
|
-
"$http_status_code $checker_type"
|
90
|
+
"$http_status_code $checker_type $tested_at"
|
90
91
|
),
|
91
|
-
"less": string.Template("$idna_subject $status $status_source"),
|
92
|
+
"less": string.Template("$idna_subject $status $status_source $tested_at"),
|
92
93
|
"simple": string.Template("$idna_subject $status"),
|
93
94
|
"percentage": string.Template("$status $percentage $amount"),
|
94
95
|
"hosts": string.Template("$ip $idna_subject"),
|
@@ -114,8 +115,11 @@ class PrinterBase:
|
|
114
115
|
"minutes": "Minutes",
|
115
116
|
"seconds": "Seconds",
|
116
117
|
"registrar": "Registrar",
|
118
|
+
"tested_at": "Tested At",
|
117
119
|
}
|
118
120
|
|
121
|
+
extra_formatters: Dict[str, Callable[..., Any]] = {}
|
122
|
+
|
119
123
|
_template_to_use: Optional[str] = None
|
120
124
|
_dataset: Optional[Dict[str, str]] = None
|
121
125
|
_skip_column: Optional[List[str]] = []
|
@@ -126,6 +130,7 @@ class PrinterBase:
|
|
126
130
|
*,
|
127
131
|
dataset: Optional[Dict[str, str]] = None,
|
128
132
|
skip_column: Optional[List[str]] = None,
|
133
|
+
extra_formatters: Optional[Dict[str, Callable[..., Any]]] = None,
|
129
134
|
) -> None:
|
130
135
|
if template_to_use is not None:
|
131
136
|
self.template_to_use = template_to_use
|
@@ -136,6 +141,9 @@ class PrinterBase:
|
|
136
141
|
if skip_column is not None:
|
137
142
|
self.skip_column = skip_column
|
138
143
|
|
144
|
+
if extra_formatters is not None:
|
145
|
+
self.extra_formatters.update(extra_formatters)
|
146
|
+
|
139
147
|
def ensure_template_to_use_is_given(func): # pylint: disable=no-self-argument
|
140
148
|
"""
|
141
149
|
Ensures that the template to use is given before launching the
|
@@ -315,9 +323,11 @@ class PrinterBase:
|
|
315
323
|
continue
|
316
324
|
|
317
325
|
if key in self.skip_column:
|
318
|
-
self.TEMPLATES[self.template_to_use].template =
|
319
|
-
self.template_to_use
|
320
|
-
|
326
|
+
self.TEMPLATES[self.template_to_use].template = (
|
327
|
+
self.TEMPLATES[self.template_to_use]
|
328
|
+
.template.replace(f"${key} ", "")
|
329
|
+
.replace(f" ${key}", "")
|
330
|
+
)
|
321
331
|
continue
|
322
332
|
|
323
333
|
to_print_data[0][key] = f"{value:<{self.STD_LENGTH[key]}}"
|
@@ -354,6 +364,9 @@ class PrinterBase:
|
|
354
364
|
if not value and value != 0:
|
355
365
|
value = self.STD_UNKNOWN
|
356
366
|
|
367
|
+
if key in self.extra_formatters:
|
368
|
+
value = self.extra_formatters[key](value)
|
369
|
+
|
357
370
|
if self.template_to_use not in ignore_length:
|
358
371
|
to_print[key] = f"{value:<{self.STD_LENGTH[key]}}"
|
359
372
|
else:
|
@@ -59,7 +59,6 @@ import PyFunceble.checker.utils.whois
|
|
59
59
|
import PyFunceble.cli.utils.stdout
|
60
60
|
import PyFunceble.cli.utils.testing
|
61
61
|
import PyFunceble.facility
|
62
|
-
import PyFunceble.factory
|
63
62
|
import PyFunceble.storage
|
64
63
|
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
|
65
64
|
from PyFunceble.cli.migrators.alembic import Alembic
|
@@ -62,7 +62,9 @@ import PyFunceble.cli.facility
|
|
62
62
|
import PyFunceble.cli.factory
|
63
63
|
import PyFunceble.facility
|
64
64
|
import PyFunceble.sessions
|
65
|
+
import PyFunceble.storage
|
65
66
|
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
|
67
|
+
from PyFunceble.query.requests.requester import Requester
|
66
68
|
|
67
69
|
|
68
70
|
class WorkerBase(multiprocessing.Process):
|
@@ -99,6 +101,7 @@ class WorkerBase(multiprocessing.Process):
|
|
99
101
|
_exception: Optional[multiprocessing.Pipe] = None
|
100
102
|
|
101
103
|
_params: Optional[dict] = {}
|
104
|
+
requester: Optional[Requester] = None
|
102
105
|
|
103
106
|
def __init__(
|
104
107
|
self,
|
@@ -149,6 +152,8 @@ class WorkerBase(multiprocessing.Process):
|
|
149
152
|
A method which will be executed after the :code:`__init__` method.
|
150
153
|
"""
|
151
154
|
|
155
|
+
self.requester = Requester(config=PyFunceble.storage.CONFIGURATION)
|
156
|
+
|
152
157
|
@property
|
153
158
|
def exception(self):
|
154
159
|
"""
|
@@ -281,9 +286,6 @@ class WorkerBase(multiprocessing.Process):
|
|
281
286
|
PyFunceble.cli.facility.CredentialLoader.start()
|
282
287
|
PyFunceble.cli.factory.DBSession.init_db_sessions()
|
283
288
|
|
284
|
-
# Be sure that all settings are loaded proprely!!
|
285
|
-
PyFunceble.factory.Requester = PyFunceble.factory.requester()
|
286
|
-
|
287
289
|
wait_for_stop = (
|
288
290
|
bool(PyFunceble.storage.CONFIGURATION.cli_testing.mining) is True
|
289
291
|
)
|
@@ -60,7 +60,6 @@ from typing import Any, Generator, List, Tuple
|
|
60
60
|
|
61
61
|
import PyFunceble.cli.storage
|
62
62
|
import PyFunceble.facility
|
63
|
-
import PyFunceble.factory
|
64
63
|
import PyFunceble.storage
|
65
64
|
from PyFunceble.cli.filesystem.printer.file import FilePrinter
|
66
65
|
from PyFunceble.cli.processes.workers.base import WorkerBase
|
@@ -57,7 +57,6 @@ from typing import List, Optional, Tuple
|
|
57
57
|
from domain2idna import domain2idna
|
58
58
|
|
59
59
|
import PyFunceble.facility
|
60
|
-
import PyFunceble.factory
|
61
60
|
import PyFunceble.storage
|
62
61
|
from PyFunceble.checker.status_base import CheckerStatusBase
|
63
62
|
from PyFunceble.cli.processes.workers.base import WorkerBase
|
@@ -86,8 +85,7 @@ class MinerWorker(WorkerBase):
|
|
86
85
|
|
87
86
|
return super().__post_init__()
|
88
87
|
|
89
|
-
|
90
|
-
def mine_from(subject: str) -> Optional[List[str]]:
|
88
|
+
def mine_from(self, subject: str) -> Optional[List[str]]:
|
91
89
|
"""
|
92
90
|
Given the subject to work from, try to get the related subjects.
|
93
91
|
|
@@ -99,7 +97,7 @@ class MinerWorker(WorkerBase):
|
|
99
97
|
result = []
|
100
98
|
|
101
99
|
try:
|
102
|
-
req =
|
100
|
+
req = self.requester.get(subject, allow_redirects=True)
|
103
101
|
|
104
102
|
for element in req.history:
|
105
103
|
if "location" in element.headers:
|
@@ -107,11 +105,11 @@ class MinerWorker(WorkerBase):
|
|
107
105
|
|
108
106
|
result.extend([x for x in req.history if isinstance(x, str)])
|
109
107
|
except (
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
108
|
+
self.requester.exceptions.RequestException,
|
109
|
+
self.requester.exceptions.ConnectionError,
|
110
|
+
self.requester.exceptions.Timeout,
|
111
|
+
self.requester.exceptions.InvalidURL,
|
112
|
+
self.requester.urllib3_exceptions.InvalidHeader,
|
115
113
|
socket.timeout,
|
116
114
|
):
|
117
115
|
PyFunceble.facility.Logger.error(
|
@@ -106,12 +106,25 @@ class ProducerWorker(WorkerBase):
|
|
106
106
|
|
107
107
|
def __post_init__(self) -> None:
|
108
108
|
skip_columns = []
|
109
|
+
extra_formatters = {}
|
109
110
|
|
110
111
|
if not PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.registrar:
|
111
112
|
skip_columns.append("registrar")
|
112
113
|
|
113
|
-
|
114
|
-
|
114
|
+
if not PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.datetime:
|
115
|
+
skip_columns.append("tested_at")
|
116
|
+
else:
|
117
|
+
# pylint: disable=line-too-long
|
118
|
+
extra_formatters["tested_at"] = lambda x: x.strftime(
|
119
|
+
PyFunceble.storage.CONFIGURATION.cli_testing.display_mode.datetime_format
|
120
|
+
)
|
121
|
+
|
122
|
+
self.stdout_printer = StdoutPrinter(
|
123
|
+
skip_column=skip_columns, extra_formatters=extra_formatters
|
124
|
+
)
|
125
|
+
self.file_printer = FilePrinter(
|
126
|
+
skip_column=skip_columns, extra_formatters=extra_formatters
|
127
|
+
)
|
115
128
|
self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
|
116
129
|
self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
|
117
130
|
self.continue_dataset = get_continue_databaset_object(
|
@@ -55,7 +55,6 @@ from typing import Any, Optional, Tuple
|
|
55
55
|
|
56
56
|
import PyFunceble.cli.utils.testing
|
57
57
|
import PyFunceble.facility
|
58
|
-
import PyFunceble.factory
|
59
58
|
from PyFunceble.checker.availability.domain_and_ip import DomainAndIPAvailabilityChecker
|
60
59
|
from PyFunceble.checker.availability.url import URLAvailabilityChecker
|
61
60
|
from PyFunceble.checker.base import CheckerBase
|
@@ -209,13 +208,10 @@ class TesterWorker(WorkerBase):
|
|
209
208
|
queue.
|
210
209
|
"""
|
211
210
|
|
212
|
-
if
|
213
|
-
PyFunceble.factory.Requester.session
|
214
|
-
and "Connection" not in PyFunceble.factory.Requester.session.headers
|
215
|
-
):
|
211
|
+
if "Connection" not in self.requester.headers:
|
216
212
|
# Just close the connection immediately. This prevent potential infinite
|
217
213
|
# streams.
|
218
|
-
|
214
|
+
self.requester.session.headers["Connection"] = "close"
|
219
215
|
|
220
216
|
if not isinstance(consumed, dict):
|
221
217
|
PyFunceble.facility.Logger.debug(
|
PyFunceble/cli/scripts/iana.py
CHANGED
@@ -54,6 +54,7 @@ import concurrent.futures
|
|
54
54
|
from typing import Dict, Optional, Tuple
|
55
55
|
|
56
56
|
import PyFunceble.facility
|
57
|
+
import PyFunceble.storage
|
57
58
|
from PyFunceble.dataset.iana import IanaDataset
|
58
59
|
from PyFunceble.helpers.dict import DictHelper
|
59
60
|
from PyFunceble.helpers.download import DownloadHelper
|
@@ -265,7 +266,16 @@ class IanaDBGenerator:
|
|
265
266
|
"""
|
266
267
|
|
267
268
|
raw_data = (
|
268
|
-
DownloadHelper(
|
269
|
+
DownloadHelper(
|
270
|
+
self.UPSTREAM_LINK,
|
271
|
+
certificate_validation=(
|
272
|
+
PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
|
273
|
+
if PyFunceble.storage.CONFIGURATION
|
274
|
+
else True
|
275
|
+
),
|
276
|
+
own_proxy_handler=True,
|
277
|
+
proxies=PyFunceble.storage.PROXY,
|
278
|
+
)
|
269
279
|
.download_text()
|
270
280
|
.split('<span class="domain tld">')
|
271
281
|
)
|
@@ -165,7 +165,20 @@ class PublicSuffixGenerator:
|
|
165
165
|
Starts the generation of the dataset file.
|
166
166
|
"""
|
167
167
|
|
168
|
-
raw_data =
|
168
|
+
raw_data = (
|
169
|
+
DownloadHelper(
|
170
|
+
self.UPSTREAM_LINK,
|
171
|
+
certificate_validation=(
|
172
|
+
PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
|
173
|
+
if PyFunceble.storage.CONFIGURATION
|
174
|
+
else True
|
175
|
+
),
|
176
|
+
own_proxy_handler=True,
|
177
|
+
proxies=PyFunceble.storage.PROXY,
|
178
|
+
)
|
179
|
+
.download_text()
|
180
|
+
.split("\n")
|
181
|
+
)
|
169
182
|
|
170
183
|
with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:
|
171
184
|
for result in executor.map(self.parse_line, raw_data):
|
@@ -455,7 +455,16 @@ class SystemLauncher(SystemBase):
|
|
455
455
|
"""
|
456
456
|
|
457
457
|
if URLSyntaxChecker(file).is_valid():
|
458
|
-
DownloadHelper(
|
458
|
+
DownloadHelper(
|
459
|
+
file,
|
460
|
+
certificate_validation=(
|
461
|
+
PyFunceble.storage.CONFIGURATION.verify_ssl_certificate
|
462
|
+
if PyFunceble.storage.CONFIGURATION
|
463
|
+
else True
|
464
|
+
),
|
465
|
+
own_proxy_handler=True,
|
466
|
+
proxies=PyFunceble.storage.PROXY,
|
467
|
+
).download_text(destination=destination)
|
459
468
|
return True
|
460
469
|
return False
|
461
470
|
|