PyFunceble-dev 4.3.0a23__py3-none-any.whl → 4.3.0a24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyFunceble/checker/availability/base.py +5 -2
- PyFunceble/checker/availability/domain.py +18 -0
- PyFunceble/checker/availability/extras/base.py +4 -4
- PyFunceble/checker/availability/extras/dns.py +1 -1
- PyFunceble/checker/availability/extras/etoxic.py +3 -3
- PyFunceble/checker/availability/extras/parked.py +22 -20
- PyFunceble/checker/availability/extras/rules.py +5 -13
- PyFunceble/checker/availability/extras/subject_switch.py +4 -5
- PyFunceble/checker/availability/ip.py +18 -0
- PyFunceble/checker/availability/params.py +1 -1
- PyFunceble/checker/availability/status.py +9 -4
- PyFunceble/checker/availability/url.py +6 -0
- PyFunceble/checker/base.py +4 -0
- PyFunceble/checker/reputation/base.py +5 -2
- PyFunceble/checker/reputation/domain.py +11 -0
- PyFunceble/checker/reputation/ip.py +11 -0
- PyFunceble/checker/reputation/status.py +1 -1
- PyFunceble/checker/reputation/url.py +12 -1
- PyFunceble/checker/syntax/base.py +4 -1
- PyFunceble/checker/syntax/domain.py +4 -0
- PyFunceble/checker/syntax/ip.py +4 -0
- PyFunceble/checker/syntax/ipv4.py +10 -13
- PyFunceble/checker/syntax/ipv6.py +9 -11
- PyFunceble/checker/syntax/params.py +1 -1
- PyFunceble/checker/syntax/second_lvl_domain.py +3 -4
- PyFunceble/checker/syntax/subdomain.py +3 -4
- PyFunceble/checker/syntax/url.py +9 -6
- PyFunceble/checker/utils/whois.py +7 -11
- PyFunceble/cli/continuous_integration/base.py +14 -14
- PyFunceble/cli/credential_loader.py +4 -6
- PyFunceble/cli/entry_points/clean.py +0 -1
- PyFunceble/cli/entry_points/production.py +1 -1
- PyFunceble/cli/entry_points/pyfunceble/cli.py +1 -1
- PyFunceble/cli/execution_time.py +2 -2
- PyFunceble/cli/filesystem/cleanup.py +1 -3
- PyFunceble/cli/filesystem/dir_base.py +5 -1
- PyFunceble/cli/filesystem/dir_structure/backup.py +1 -2
- PyFunceble/cli/filesystem/dir_structure/base.py +0 -1
- PyFunceble/cli/filesystem/dir_structure/restore.py +5 -7
- PyFunceble/cli/filesystem/status_file.py +2 -3
- PyFunceble/cli/migrators/alembic.py +2 -4
- PyFunceble/cli/migrators/csv_file/inactive_source_delete.py +0 -1
- PyFunceble/cli/migrators/csv_file/whois_registrar_add.py +0 -1
- PyFunceble/cli/migrators/db_base.py +2 -2
- PyFunceble/cli/migrators/file_cleanup/base.py +3 -3
- PyFunceble/cli/migrators/file_cleanup/hashes_file.py +2 -3
- PyFunceble/cli/migrators/file_cleanup/mining_file.py +2 -3
- PyFunceble/cli/migrators/file_cleanup/production_config_file.py +2 -3
- PyFunceble/cli/migrators/mariadb/base.py +2 -4
- PyFunceble/cli/migrators/mariadb/file_and_status.py +1 -1
- PyFunceble/cli/migrators/mariadb/whois_record_idna_subject.py +0 -2
- PyFunceble/cli/processes/workers/base.py +0 -1
- PyFunceble/cli/processes/workers/chancy_producer.py +1 -2
- PyFunceble/cli/processes/workers/dir_files_sorter.py +6 -14
- PyFunceble/cli/processes/workers/file_sorter.py +2 -9
- PyFunceble/cli/processes/workers/migrator.py +0 -1
- PyFunceble/cli/processes/workers/producer.py +2 -4
- PyFunceble/cli/processes/workers/tester.py +2 -3
- PyFunceble/cli/scripts/iana.py +14 -10
- PyFunceble/cli/scripts/production.py +2 -2
- PyFunceble/cli/scripts/public_suffix.py +2 -2
- PyFunceble/cli/system/integrator.py +1 -1
- PyFunceble/cli/system/launcher.py +9 -10
- PyFunceble/cli/utils/stdout.py +1 -1
- PyFunceble/cli/utils/testing.py +6 -16
- PyFunceble/config/compare.py +10 -9
- PyFunceble/config/loader.py +7 -4
- PyFunceble/converter/input_line2subject.py +2 -2
- PyFunceble/converter/internal_url.py +1 -1
- PyFunceble/converter/rpz_input_line2subject.py +2 -4
- PyFunceble/converter/subject2complements.py +4 -1
- PyFunceble/converter/url2netloc.py +3 -3
- PyFunceble/converter/wildcard2subject.py +3 -3
- PyFunceble/helpers/command.py +1 -1
- PyFunceble/helpers/dict.py +4 -4
- PyFunceble/helpers/file.py +2 -2
- PyFunceble/helpers/list.py +10 -7
- PyFunceble/helpers/merge.py +2 -2
- PyFunceble/helpers/regex.py +9 -11
- PyFunceble/query/dns/nameserver.py +9 -19
- PyFunceble/query/dns/query_tool.py +64 -205
- PyFunceble/query/dns/resolver.py +64 -51
- PyFunceble/query/netinfo/address.py +4 -6
- PyFunceble/query/netinfo/base.py +1 -1
- PyFunceble/query/netinfo/hostbyaddr.py +5 -8
- PyFunceble/query/requests/adapter/base.py +3 -3
- PyFunceble/query/requests/adapter/http.py +0 -1
- PyFunceble/query/requests/requester.py +6 -6
- PyFunceble/query/whois/converter/expiration_date.py +19 -8
- PyFunceble/query/whois/converter/month2unified.py +4 -6
- PyFunceble/query/whois/converter/registrar.py +18 -5
- PyFunceble/query/whois/query_tool.py +15 -18
- PyFunceble/storage.py +1 -1
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/METADATA +107 -106
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/RECORD +99 -99
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/WHEEL +0 -0
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/entry_points.txt +0 -0
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/licenses/LICENSE +0 -0
- {pyfunceble_dev-4.3.0a23.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/top_level.txt +0 -0
@@ -53,7 +53,6 @@ License:
|
|
53
53
|
from typing import Any, Optional, Tuple
|
54
54
|
|
55
55
|
import PyFunceble.facility
|
56
|
-
import PyFunceble.storage
|
57
56
|
from PyFunceble.checker.status_base import CheckerStatusBase
|
58
57
|
from PyFunceble.cli.processes.workers.producer import ProducerWorker
|
59
58
|
|
@@ -81,7 +80,7 @@ class ChancyProducerWorker(ProducerWorker):
|
|
81
80
|
|
82
81
|
if not isinstance(test_dataset, dict):
|
83
82
|
PyFunceble.facility.Logger.info(
|
84
|
-
"Skipping because test dataset is not a
|
83
|
+
"Skipping because test dataset is not a dictionary."
|
85
84
|
)
|
86
85
|
return None
|
87
86
|
|
@@ -149,26 +149,18 @@ class DireFileSorterWorker(FileSorterWorkerBase):
|
|
149
149
|
# Just for human brain :-)
|
150
150
|
directory = consumed["directory"]
|
151
151
|
|
152
|
-
|
153
|
-
|
154
|
-
else:
|
155
|
-
remove_duplicates = True
|
156
|
-
|
157
|
-
if "write_header" in consumed:
|
158
|
-
write_header = consumed["write_header"]
|
159
|
-
else:
|
160
|
-
write_header = True
|
152
|
+
remove_duplicates = consumed.get("remove_duplicates", True)
|
153
|
+
write_header = consumed.get("write_header", True)
|
161
154
|
|
162
155
|
with concurrent.futures.ThreadPoolExecutor(
|
163
156
|
max_workers=PyFunceble.storage.CONFIGURATION.cli_testing.max_workers,
|
164
157
|
) as executor:
|
165
|
-
submitted_list = [
|
166
|
-
|
167
|
-
for file in self.get_files_to_sort(directory):
|
168
|
-
submitted = executor.submit(
|
158
|
+
submitted_list = [
|
159
|
+
executor.submit(
|
169
160
|
self.process_file_sorting, file, remove_duplicates, write_header
|
170
161
|
)
|
171
|
-
|
162
|
+
for file in self.get_files_to_sort(directory)
|
163
|
+
]
|
172
164
|
|
173
165
|
for submitted in concurrent.futures.as_completed(submitted_list):
|
174
166
|
# Ensure that everything is finished
|
@@ -103,15 +103,8 @@ class FileSorterWorker(FileSorterWorkerBase):
|
|
103
103
|
# Just for human brain :-)
|
104
104
|
file = consumed["file"]
|
105
105
|
|
106
|
-
|
107
|
-
|
108
|
-
else:
|
109
|
-
remove_duplicates = True
|
110
|
-
|
111
|
-
if "write_header" in consumed:
|
112
|
-
write_header = consumed["write_header"]
|
113
|
-
else:
|
114
|
-
write_header = True
|
106
|
+
remove_duplicates = consumed.get("remove_duplicates", True)
|
107
|
+
write_header = consumed.get("write_header", True)
|
115
108
|
|
116
109
|
self.process_file_sorting(
|
117
110
|
file, remove_duplicates=remove_duplicates, write_header=write_header
|
@@ -68,7 +68,7 @@ from PyFunceble.cli.filesystem.status_file import StatusFileGenerator
|
|
68
68
|
from PyFunceble.cli.processes.workers.base import WorkerBase
|
69
69
|
from PyFunceble.cli.utils.stdout import get_template_to_use, print_single_line
|
70
70
|
from PyFunceble.cli.utils.testing import (
|
71
|
-
|
71
|
+
get_continue_dataset_object,
|
72
72
|
get_inactive_dataset_object,
|
73
73
|
)
|
74
74
|
from PyFunceble.dataset.autocontinue.base import ContinueDatasetBase
|
@@ -127,9 +127,7 @@ class ProducerWorker(WorkerBase):
|
|
127
127
|
)
|
128
128
|
self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
|
129
129
|
self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
|
130
|
-
self.continue_dataset =
|
131
|
-
db_session=self.db_session
|
132
|
-
)
|
130
|
+
self.continue_dataset = get_continue_dataset_object(db_session=self.db_session)
|
133
131
|
self.status_file_generator = StatusFileGenerator().guess_all_settings()
|
134
132
|
self.counter = FilesystemCounter()
|
135
133
|
self.registrar_counter = RegistrarCounter()
|
@@ -89,7 +89,7 @@ class TesterWorker(WorkerBase):
|
|
89
89
|
result = super().perform_external_poweron_checks()
|
90
90
|
|
91
91
|
self.continue_dataset = (
|
92
|
-
PyFunceble.cli.utils.testing.
|
92
|
+
PyFunceble.cli.utils.testing.get_continue_dataset_object(
|
93
93
|
db_session=self.db_session
|
94
94
|
)
|
95
95
|
)
|
@@ -215,8 +215,7 @@ class TesterWorker(WorkerBase):
|
|
215
215
|
|
216
216
|
if not isinstance(consumed, dict):
|
217
217
|
PyFunceble.facility.Logger.debug(
|
218
|
-
"Skipping latest dataset because consumed data was not "
|
219
|
-
"a dictionnary."
|
218
|
+
"Skipping latest dataset because consumed data was not a dictionary."
|
220
219
|
)
|
221
220
|
return None
|
222
221
|
|
PyFunceble/cli/scripts/iana.py
CHANGED
@@ -143,12 +143,18 @@ class IanaDBGenerator:
|
|
143
143
|
An internal storage map.
|
144
144
|
"""
|
145
145
|
|
146
|
+
whois_query_tool: Optional[WhoisQueryTool] = None
|
147
|
+
regex_helper: Optional[RegexHelper] = None
|
148
|
+
|
146
149
|
def __init__(self, destination: Optional[str] = None) -> None:
|
147
150
|
if destination is not None:
|
148
151
|
self.destination = destination
|
149
152
|
else:
|
150
153
|
self.destination = IanaDataset().source_file
|
151
154
|
|
155
|
+
self.whois_query_tool = WhoisQueryTool()
|
156
|
+
self.regex_helper = RegexHelper()
|
157
|
+
|
152
158
|
@property
|
153
159
|
def destination(self) -> Optional[str]:
|
154
160
|
"""
|
@@ -196,11 +202,9 @@ class IanaDBGenerator:
|
|
196
202
|
Given an extension, tries to get or guess its extension.
|
197
203
|
"""
|
198
204
|
|
199
|
-
whois_query_tool = WhoisQueryTool()
|
200
|
-
|
201
205
|
dummy_domain = f"hello.{extension}"
|
202
206
|
iana_record = (
|
203
|
-
whois_query_tool.set_server(self.IANA_WHOIS_SERVER)
|
207
|
+
self.whois_query_tool.set_server(self.IANA_WHOIS_SERVER)
|
204
208
|
.set_subject(dummy_domain)
|
205
209
|
.record
|
206
210
|
)
|
@@ -208,7 +212,7 @@ class IanaDBGenerator:
|
|
208
212
|
if iana_record and "refer" in iana_record:
|
209
213
|
regex_referrer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n"
|
210
214
|
|
211
|
-
matched =
|
215
|
+
matched = self.regex_helper.set_regex(regex_referrer).match(
|
212
216
|
iana_record, return_match=True, group=1
|
213
217
|
)
|
214
218
|
|
@@ -216,7 +220,7 @@ class IanaDBGenerator:
|
|
216
220
|
return matched
|
217
221
|
|
218
222
|
possible_server = f"whois.nic.{extension}"
|
219
|
-
response = whois_query_tool.set_server(possible_server).record
|
223
|
+
response = self.whois_query_tool.set_server(possible_server).record
|
220
224
|
|
221
225
|
if response:
|
222
226
|
return possible_server
|
@@ -224,7 +228,7 @@ class IanaDBGenerator:
|
|
224
228
|
if extension in self.MANUAL_SERVER:
|
225
229
|
possible_server = self.MANUAL_SERVER[extension]
|
226
230
|
|
227
|
-
response = whois_query_tool.set_server(possible_server).record
|
231
|
+
response = self.whois_query_tool.set_server(possible_server).record
|
228
232
|
|
229
233
|
if response:
|
230
234
|
return possible_server
|
@@ -247,10 +251,10 @@ class IanaDBGenerator:
|
|
247
251
|
|
248
252
|
regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"
|
249
253
|
|
250
|
-
regex_helper
|
251
|
-
|
252
|
-
|
253
|
-
extension = regex_helper.match(block, return_match=True, group=2)
|
254
|
+
if self.regex_helper.set_regex(regex_valid_extension).match(
|
255
|
+
block, return_match=False
|
256
|
+
):
|
257
|
+
extension = self.regex_helper.match(block, return_match=True, group=2)
|
254
258
|
|
255
259
|
if extension:
|
256
260
|
return extension, self.get_referrer_from_extension(extension)
|
@@ -86,7 +86,7 @@ class PublicSuffixGenerator:
|
|
86
86
|
An internal storage of our map.
|
87
87
|
"""
|
88
88
|
|
89
|
-
|
89
|
+
wildcard2subject: Wildcard2Subject = Wildcard2Subject()
|
90
90
|
|
91
91
|
def __init__(self, destination: Optional[str] = None) -> None:
|
92
92
|
if destination is not None:
|
@@ -147,7 +147,7 @@ class PublicSuffixGenerator:
|
|
147
147
|
if not any(line.startswith(x) for x in self.COMMENT_SIGN) and "." in line:
|
148
148
|
lines = [line, line.encode("idna").decode("utf-8")]
|
149
149
|
lines = [
|
150
|
-
self.
|
150
|
+
self.wildcard2subject.set_data_to_convert(x).get_converted()
|
151
151
|
for x in lines
|
152
152
|
]
|
153
153
|
extension = lines[0].rsplit(".", 1)[-1]
|
@@ -73,7 +73,7 @@ class SystemIntegrator(SystemBase):
|
|
73
73
|
Provides our system integrator. The idea is that we given an argparse
|
74
74
|
Namespace, we should be able to parse it into our system.
|
75
75
|
|
76
|
-
To simplify the trick and headache in the CLI management, I
|
76
|
+
To simplify the trick and headache in the CLI management, I explicitly
|
77
77
|
mapped the :code:`dest` argument to what we are supposed to have in the
|
78
78
|
flatten version of the configuration. That way, we only need to compare
|
79
79
|
against the flatten version instead of looping over all possible levels
|
@@ -101,7 +101,7 @@ from PyFunceble.cli.processes.producer import ProducerProcessesManager
|
|
101
101
|
from PyFunceble.cli.processes.tester import TesterProcessesManager
|
102
102
|
from PyFunceble.cli.system.base import SystemBase
|
103
103
|
from PyFunceble.cli.utils.testing import (
|
104
|
-
|
104
|
+
get_continue_dataset_object,
|
105
105
|
get_destination_from_origin,
|
106
106
|
get_inactive_dataset_object,
|
107
107
|
get_subjects_from_line,
|
@@ -187,9 +187,7 @@ class SystemLauncher(SystemBase):
|
|
187
187
|
|
188
188
|
self.execution_time_holder = ExecutionTime().set_start_time()
|
189
189
|
self.checker_type = get_testing_mode()
|
190
|
-
self.continue_dataset =
|
191
|
-
db_session=self.db_session
|
192
|
-
)
|
190
|
+
self.continue_dataset = get_continue_dataset_object(db_session=self.db_session)
|
193
191
|
self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
|
194
192
|
self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
|
195
193
|
self.continuous_integration = ci_object()
|
@@ -775,7 +773,7 @@ class SystemLauncher(SystemBase):
|
|
775
773
|
def generate_waiting_files(self) -> "SystemLauncher":
|
776
774
|
"""
|
777
775
|
Generates all the files that needs to be generated when all status
|
778
|
-
are
|
776
|
+
are processes.
|
779
777
|
"""
|
780
778
|
|
781
779
|
def generate_percentage_file(parent_dirname: Union[str, None]) -> None:
|
@@ -1001,18 +999,19 @@ class SystemLauncher(SystemBase):
|
|
1001
999
|
"""
|
1002
1000
|
Remove the inline destination - when necessary.
|
1003
1001
|
|
1004
|
-
:param
|
1002
|
+
:param protocol:
|
1005
1003
|
The protocol to work with.
|
1006
1004
|
"""
|
1007
1005
|
|
1008
1006
|
if not protocol["destination"]:
|
1009
|
-
|
1007
|
+
directory_helper.set_path(
|
1010
1008
|
self.counter.set_differ_to_inline(True)
|
1011
1009
|
.set_parent_dirname(protocol["destination"])
|
1012
1010
|
.get_output_basedir()
|
1013
1011
|
).delete()
|
1014
1012
|
|
1015
1013
|
file_helper = FileHelper()
|
1014
|
+
directory_helper = DirectoryHelper()
|
1016
1015
|
|
1017
1016
|
for protocol in self.testing_protocol:
|
1018
1017
|
if "destination" in protocol or "output_dir" in protocol:
|
@@ -1028,7 +1027,7 @@ class SystemLauncher(SystemBase):
|
|
1028
1027
|
|
1029
1028
|
def run_standard_end_instructions(self) -> "SystemLauncher":
|
1030
1029
|
"""
|
1031
|
-
|
1030
|
+
Runs our standard "end" instructions.
|
1032
1031
|
|
1033
1032
|
The instructions executed by this method are the one we execute normally.
|
1034
1033
|
|
@@ -1049,7 +1048,7 @@ class SystemLauncher(SystemBase):
|
|
1049
1048
|
|
1050
1049
|
def run_ci_saving_instructions(self) -> "SystemLauncher":
|
1051
1050
|
"""
|
1052
|
-
|
1051
|
+
Runs our CI "saving" instructions.
|
1053
1052
|
|
1054
1053
|
The instructions executed by this method are the one we execute
|
1055
1054
|
before ending a testing session under one of the supported CI engines.
|
@@ -1071,7 +1070,7 @@ class SystemLauncher(SystemBase):
|
|
1071
1070
|
|
1072
1071
|
def run_ci_end_saving_instructions(self) -> "SystemLauncher":
|
1073
1072
|
"""
|
1074
|
-
|
1073
|
+
Runs our CI END "saving" instructions.
|
1075
1074
|
|
1076
1075
|
The instructions executed by this method are the one we execute
|
1077
1076
|
before ending a testing session under one of the supported CI engines.
|
PyFunceble/cli/utils/stdout.py
CHANGED
PyFunceble/cli/utils/testing.py
CHANGED
@@ -90,7 +90,7 @@ def get_testing_mode() -> str:
|
|
90
90
|
return "UNKNOWN"
|
91
91
|
|
92
92
|
|
93
|
-
def
|
93
|
+
def get_continue_dataset_object(
|
94
94
|
db_session: Optional[Session] = None,
|
95
95
|
) -> Union[DatasetBase, CSVDatasetBase, DBDatasetBase]:
|
96
96
|
"""
|
@@ -199,35 +199,25 @@ def get_subjects_from_line(
|
|
199
199
|
result = []
|
200
200
|
|
201
201
|
if adblock_inputline2subject is None:
|
202
|
-
adblock_inputline2subject = AdblockInputLine2Subject(
|
203
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
204
|
-
)
|
202
|
+
adblock_inputline2subject = AdblockInputLine2Subject()
|
205
203
|
|
206
204
|
if wildcard2subject is None:
|
207
|
-
wildcard2subject = Wildcard2Subject(
|
208
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
209
|
-
)
|
205
|
+
wildcard2subject = Wildcard2Subject()
|
210
206
|
|
211
207
|
if rpz_policy2subject is None:
|
212
208
|
rpz_policy2subject = RPZPolicy2Subject()
|
213
209
|
|
214
210
|
if rpz_inputline2subject is None:
|
215
|
-
rpz_inputline2subject = RPZInputLine2Subject(
|
216
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
217
|
-
)
|
211
|
+
rpz_inputline2subject = RPZInputLine2Subject()
|
218
212
|
|
219
213
|
if inputline2subject is None:
|
220
|
-
inputline2subject = InputLine2Subject(
|
221
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
222
|
-
)
|
214
|
+
inputline2subject = InputLine2Subject()
|
223
215
|
|
224
216
|
if subject2complements is None:
|
225
217
|
subject2complements = Subject2Complements()
|
226
218
|
|
227
219
|
if url2netloc is None:
|
228
|
-
url2netloc = Url2Netloc(
|
229
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
230
|
-
)
|
220
|
+
url2netloc = Url2Netloc()
|
231
221
|
|
232
222
|
if cidr2subject is None:
|
233
223
|
cidr2subject = CIDR2Subject()
|
PyFunceble/config/compare.py
CHANGED
@@ -11,7 +11,7 @@ The tool to check the availability or syntax of domain, IP or URL.
|
|
11
11
|
██║ ██║ ██║ ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
|
12
12
|
╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝
|
13
13
|
|
14
|
-
Provides the configuration
|
14
|
+
Provides the configuration comparison interface.
|
15
15
|
|
16
16
|
Author:
|
17
17
|
Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom
|
@@ -175,7 +175,7 @@ class ConfigComparison:
|
|
175
175
|
}
|
176
176
|
|
177
177
|
_local_config: dict = {}
|
178
|
-
|
178
|
+
_upstream_config: dict = {}
|
179
179
|
|
180
180
|
dict_helper: DictHelper = DictHelper()
|
181
181
|
|
@@ -228,12 +228,12 @@ class ConfigComparison:
|
|
228
228
|
Provides the current state of the :code:`_upstream_config`.
|
229
229
|
"""
|
230
230
|
|
231
|
-
return self.
|
231
|
+
return self._upstream_config
|
232
232
|
|
233
233
|
@upstream_config.setter
|
234
234
|
def upstream_config(self, value: dict) -> None:
|
235
235
|
"""
|
236
|
-
Sets the
|
236
|
+
Sets the upstream configuration to work with.
|
237
237
|
|
238
238
|
:raise TypeError:
|
239
239
|
When :code:`value` is not a :py:class:`dict`
|
@@ -242,11 +242,11 @@ class ConfigComparison:
|
|
242
242
|
if not isinstance(value, dict):
|
243
243
|
raise TypeError(f"<value> should be {dict}, {type(value)} given.")
|
244
244
|
|
245
|
-
self.
|
245
|
+
self._upstream_config = copy.deepcopy(value)
|
246
246
|
|
247
247
|
def set_upstream_config(self, value: dict) -> "ConfigComparison":
|
248
248
|
"""
|
249
|
-
Sets the
|
249
|
+
Sets the upstream configuration to work with.
|
250
250
|
"""
|
251
251
|
|
252
252
|
self.upstream_config = value
|
@@ -265,6 +265,7 @@ class ConfigComparison:
|
|
265
265
|
)
|
266
266
|
or "user_agent" not in self.local_config
|
267
267
|
or not isinstance(self.local_config["user_agent"], dict)
|
268
|
+
or "http_codes" not in self.local_config
|
268
269
|
or "active" in self.local_config["http_codes"]
|
269
270
|
or "not_found_default" in self.local_config["http_codes"]
|
270
271
|
or "self_managed" not in self.local_config["http_codes"]
|
@@ -281,7 +282,7 @@ class ConfigComparison:
|
|
281
282
|
if index in self.DELETED_CORE:
|
282
283
|
return False
|
283
284
|
|
284
|
-
for index in self.local_config
|
285
|
+
for index in self.local_config.get("links", {}):
|
285
286
|
if index in self.DELETED_LINKS:
|
286
287
|
return False
|
287
288
|
|
@@ -369,10 +370,10 @@ class ConfigComparison:
|
|
369
370
|
del merged[index]
|
370
371
|
|
371
372
|
for index in self.DELETED_LINKS:
|
372
|
-
if index in merged
|
373
|
+
if index in merged.get("links", {}):
|
373
374
|
del merged["links"][index]
|
374
375
|
|
375
|
-
if not bool(merged["http_codes"]["self_managed"]):
|
376
|
+
if "http_codes" in merged and not bool(merged["http_codes"]["self_managed"]):
|
376
377
|
for index, values in PyFunceble.storage.STD_HTTP_CODES.list.items():
|
377
378
|
merged["http_codes"]["list"][index] = list(values)
|
378
379
|
|
PyFunceble/config/loader.py
CHANGED
@@ -99,8 +99,8 @@ class ConfigLoader:
|
|
99
99
|
_config_dir: Optional[str] = None
|
100
100
|
__config_loaded: bool = False
|
101
101
|
|
102
|
-
file_helper: FileHelper =
|
103
|
-
dict_helper: DictHelper =
|
102
|
+
file_helper: Optional[FileHelper] = None
|
103
|
+
dict_helper: Optional[DictHelper] = None
|
104
104
|
|
105
105
|
def __init__(
|
106
106
|
self, merge_upstream: Optional[bool] = None, *, config_dir: Optional[str] = None
|
@@ -123,6 +123,9 @@ class ConfigLoader:
|
|
123
123
|
elif EnvironmentVariableHelper("PYFUNCEBLE_AUTO_CONFIGURATION").exists():
|
124
124
|
self.merge_upstream = True
|
125
125
|
|
126
|
+
self.file_helper = FileHelper()
|
127
|
+
self.dict_helper = DictHelper()
|
128
|
+
|
126
129
|
def __del__(self) -> None:
|
127
130
|
self.destroy()
|
128
131
|
|
@@ -451,7 +454,7 @@ class ConfigLoader:
|
|
451
454
|
self,
|
452
455
|
) -> "ConfigLoader":
|
453
456
|
"""
|
454
|
-
Downloads all the
|
457
|
+
Downloads all the dynamically (generated) infrastructure files.
|
455
458
|
|
456
459
|
.. note::
|
457
460
|
Downloaded if missing:
|
@@ -532,7 +535,7 @@ class ConfigLoader:
|
|
532
535
|
or self.merge_upstream
|
533
536
|
or is_3_x_version(config)
|
534
537
|
or not config_comparer.is_local_identical()
|
535
|
-
): # pragma: no cover ## Testing the underlying comparison method is
|
538
|
+
): # pragma: no cover ## Testing the underlying comparison method is sufficient
|
536
539
|
config = config_comparer.get_merged()
|
537
540
|
|
538
541
|
self.dict_helper.set_subject(config).to_yaml_file(self.path_to_config)
|
@@ -60,7 +60,7 @@ from PyFunceble.converter.url2netloc import Url2Netloc
|
|
60
60
|
|
61
61
|
class InputLine2Subject(ConverterBase):
|
62
62
|
"""
|
63
|
-
Converts/Extract the
|
63
|
+
Converts/Extract the subjects to test from an imputed line.
|
64
64
|
"""
|
65
65
|
|
66
66
|
COMMENT: str = "#"
|
@@ -72,7 +72,7 @@ class InputLine2Subject(ConverterBase):
|
|
72
72
|
@ConverterBase.data_to_convert.setter
|
73
73
|
def data_to_convert(self, value: Any) -> None:
|
74
74
|
"""
|
75
|
-
|
75
|
+
Overrides the default behavior.
|
76
76
|
|
77
77
|
:raise TypeError:
|
78
78
|
When the given data to convert is not :py:class:`str`
|
@@ -71,7 +71,7 @@ class InternalUrlConverter(ConverterBase):
|
|
71
71
|
@ConverterBase.data_to_convert.setter
|
72
72
|
def data_to_convert(self, value: Any) -> None:
|
73
73
|
"""
|
74
|
-
|
74
|
+
Overrides the default behavior.
|
75
75
|
|
76
76
|
:raise TypeError:
|
77
77
|
When the given data to convert is not :py:class:`str`
|
@@ -59,7 +59,7 @@ from PyFunceble.converter.input_line2subject import InputLine2Subject
|
|
59
59
|
|
60
60
|
class RPZInputLine2Subject(InputLine2Subject):
|
61
61
|
"""
|
62
|
-
Converts/Extracts the subject from the given RPZ
|
62
|
+
Converts/Extracts the subject from the given RPZ input line.
|
63
63
|
"""
|
64
64
|
|
65
65
|
COMMENT: list = [";", "//", "#"]
|
@@ -96,9 +96,7 @@ class RPZInputLine2Subject(InputLine2Subject):
|
|
96
96
|
if self.SPACE in subject or self.TAB in subject:
|
97
97
|
subject = subject.split()[0]
|
98
98
|
|
99
|
-
|
100
|
-
result.append(subject)
|
101
|
-
elif not subject.isdigit():
|
99
|
+
if not subject.isdigit():
|
102
100
|
result.append(subject)
|
103
101
|
|
104
102
|
return result
|
@@ -64,6 +64,7 @@ class Subject2Complements(ConverterBase):
|
|
64
64
|
"""
|
65
65
|
|
66
66
|
_include_given: bool = False
|
67
|
+
domain_syntax_checker: Optional[DomainSyntaxChecker] = None
|
67
68
|
|
68
69
|
def __init__(
|
69
70
|
self,
|
@@ -74,6 +75,8 @@ class Subject2Complements(ConverterBase):
|
|
74
75
|
if include_given is not None:
|
75
76
|
self.include_given = include_given
|
76
77
|
|
78
|
+
self.domain_syntax_checker = DomainSyntaxChecker()
|
79
|
+
|
77
80
|
super().__init__(data_to_convert=data_to_convert)
|
78
81
|
|
79
82
|
@ConverterBase.data_to_convert.setter
|
@@ -143,7 +146,7 @@ class Subject2Complements(ConverterBase):
|
|
143
146
|
_ = aggressive
|
144
147
|
result = []
|
145
148
|
|
146
|
-
checker =
|
149
|
+
checker = self.domain_syntax_checker.set_subject(data)
|
147
150
|
|
148
151
|
if self.include_given and data not in result:
|
149
152
|
result.append(data)
|
@@ -58,7 +58,7 @@ from PyFunceble.converter.base import ConverterBase
|
|
58
58
|
|
59
59
|
class Url2Netloc(ConverterBase):
|
60
60
|
"""
|
61
|
-
Provides the interface for the conversion/
|
61
|
+
Provides the interface for the conversion/extraction of the network location
|
62
62
|
of a given URL.
|
63
63
|
"""
|
64
64
|
|
@@ -70,7 +70,7 @@ class Url2Netloc(ConverterBase):
|
|
70
70
|
@ConverterBase.data_to_convert.setter
|
71
71
|
def data_to_convert(self, value: Any) -> None:
|
72
72
|
"""
|
73
|
-
|
73
|
+
Overrides the default behavior.
|
74
74
|
|
75
75
|
:raise TypeError:
|
76
76
|
When the given data to convert is not :py:class:`str`
|
@@ -116,7 +116,7 @@ class Url2Netloc(ConverterBase):
|
|
116
116
|
Provides the converted data (after conversion)
|
117
117
|
"""
|
118
118
|
|
119
|
-
#
|
119
|
+
# Retro-compatibility.
|
120
120
|
self.parse_url()
|
121
121
|
|
122
122
|
return self.convert(self.data_to_convert)
|
@@ -52,7 +52,7 @@ License:
|
|
52
52
|
"""
|
53
53
|
# pylint: enable=line-too-long
|
54
54
|
|
55
|
-
from typing import Any
|
55
|
+
from typing import Any, Optional
|
56
56
|
|
57
57
|
from PyFunceble.converter.base import ConverterBase
|
58
58
|
|
@@ -67,7 +67,7 @@ class Wildcard2Subject(ConverterBase):
|
|
67
67
|
@ConverterBase.data_to_convert.setter
|
68
68
|
def data_to_convert(self, value: Any) -> None:
|
69
69
|
"""
|
70
|
-
|
70
|
+
Overrides the default behavior.
|
71
71
|
|
72
72
|
:raise TypeError:
|
73
73
|
When the given data to convert is not :py:class:`str`
|
@@ -86,7 +86,7 @@ class Wildcard2Subject(ConverterBase):
|
|
86
86
|
|
87
87
|
return self.convert(self.data_to_convert)
|
88
88
|
|
89
|
-
def convert(self, data: Any, *, aggressive: bool = False) -> str:
|
89
|
+
def convert(self, data: Any, *, aggressive: bool = False) -> Optional[str]:
|
90
90
|
"""
|
91
91
|
Converts the given dataset.
|
92
92
|
|
PyFunceble/helpers/command.py
CHANGED
@@ -212,7 +212,7 @@ class CommandHelper:
|
|
212
212
|
is that :func:`~PyFunceble.helpers.Command.execute` wait for the
|
213
213
|
process to end in order to return its output while this method
|
214
214
|
return each line one by one
|
215
|
-
- as they are
|
215
|
+
- as they are outputted.
|
216
216
|
|
217
217
|
:param bool rstrip:
|
218
218
|
Deactivates the rstrip of the output.
|
PyFunceble/helpers/dict.py
CHANGED
@@ -66,7 +66,7 @@ class DictHelper:
|
|
66
66
|
Simplify some :code:`dict` manipulation.
|
67
67
|
|
68
68
|
:param dict main: The main :code:`dict` to work with.
|
69
|
-
:raise TypeError: When :code:`main` is not a dict nor a list (
|
69
|
+
:raise TypeError: When :code:`main` is not a dict nor a list (tolerated).
|
70
70
|
"""
|
71
71
|
|
72
72
|
_subject: Optional[Union[Any, dict]] = None
|
@@ -349,7 +349,7 @@ class DictHelper:
|
|
349
349
|
:param bool default_flow_style: Uses the default flow style.
|
350
350
|
:param int indent: The indentation to apply.
|
351
351
|
:param bool allow_unicode: Allows the decoding of unicode chars.
|
352
|
-
:param bool sort_keys:
|
352
|
+
:param bool sort_keys: Sorts the keys.
|
353
353
|
|
354
354
|
:rtype: dict|list
|
355
355
|
"""
|
@@ -371,7 +371,7 @@ class DictHelper:
|
|
371
371
|
data: Optional[Any] = None,
|
372
372
|
) -> dict:
|
373
373
|
"""
|
374
|
-
Flatten the current
|
374
|
+
Flatten the current dictionary.
|
375
375
|
|
376
376
|
:param separator:
|
377
377
|
The separator to apply.
|
@@ -405,7 +405,7 @@ class DictHelper:
|
|
405
405
|
|
406
406
|
def unflatten(self, *, separator: str = ".", data: Optional[Any] = None):
|
407
407
|
"""
|
408
|
-
Unflatten a previously flatten
|
408
|
+
Unflatten a previously flatten dictionary.
|
409
409
|
|
410
410
|
:param separator:
|
411
411
|
The separator to split.
|