PyFunceble-dev 4.3.0a21__py3-none-any.whl → 4.3.0a24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyFunceble/checker/availability/base.py +7 -6
- PyFunceble/checker/availability/domain.py +18 -0
- PyFunceble/checker/availability/extras/base.py +4 -4
- PyFunceble/checker/availability/extras/dns.py +1 -1
- PyFunceble/checker/availability/extras/etoxic.py +3 -3
- PyFunceble/checker/availability/extras/parked.py +22 -20
- PyFunceble/checker/availability/extras/rules.py +5 -13
- PyFunceble/checker/availability/extras/subject_switch.py +4 -5
- PyFunceble/checker/availability/ip.py +18 -0
- PyFunceble/checker/availability/params.py +1 -1
- PyFunceble/checker/availability/status.py +9 -4
- PyFunceble/checker/availability/url.py +6 -0
- PyFunceble/checker/base.py +4 -0
- PyFunceble/checker/reputation/base.py +5 -2
- PyFunceble/checker/reputation/domain.py +11 -0
- PyFunceble/checker/reputation/ip.py +11 -0
- PyFunceble/checker/reputation/status.py +1 -1
- PyFunceble/checker/reputation/url.py +12 -1
- PyFunceble/checker/syntax/base.py +4 -1
- PyFunceble/checker/syntax/domain.py +4 -0
- PyFunceble/checker/syntax/ip.py +4 -0
- PyFunceble/checker/syntax/ipv4.py +10 -13
- PyFunceble/checker/syntax/ipv6.py +9 -11
- PyFunceble/checker/syntax/params.py +1 -1
- PyFunceble/checker/syntax/second_lvl_domain.py +3 -4
- PyFunceble/checker/syntax/subdomain.py +3 -4
- PyFunceble/checker/syntax/url.py +9 -6
- PyFunceble/checker/utils/whois.py +7 -11
- PyFunceble/cli/continuous_integration/base.py +14 -14
- PyFunceble/cli/credential_loader.py +4 -6
- PyFunceble/cli/entry_points/clean.py +0 -1
- PyFunceble/cli/entry_points/production.py +1 -1
- PyFunceble/cli/entry_points/pyfunceble/cli.py +11 -2
- PyFunceble/cli/execution_time.py +2 -2
- PyFunceble/cli/filesystem/cleanup.py +1 -3
- PyFunceble/cli/filesystem/dir_base.py +5 -1
- PyFunceble/cli/filesystem/dir_structure/backup.py +1 -2
- PyFunceble/cli/filesystem/dir_structure/base.py +0 -1
- PyFunceble/cli/filesystem/dir_structure/restore.py +5 -7
- PyFunceble/cli/filesystem/status_file.py +2 -3
- PyFunceble/cli/migrators/alembic.py +2 -4
- PyFunceble/cli/migrators/csv_file/inactive_source_delete.py +0 -1
- PyFunceble/cli/migrators/csv_file/whois_registrar_add.py +0 -1
- PyFunceble/cli/migrators/db_base.py +2 -2
- PyFunceble/cli/migrators/file_cleanup/base.py +3 -3
- PyFunceble/cli/migrators/file_cleanup/hashes_file.py +2 -3
- PyFunceble/cli/migrators/file_cleanup/mining_file.py +2 -3
- PyFunceble/cli/migrators/file_cleanup/production_config_file.py +2 -3
- PyFunceble/cli/migrators/mariadb/base.py +2 -4
- PyFunceble/cli/migrators/mariadb/file_and_status.py +1 -1
- PyFunceble/cli/migrators/mariadb/whois_record_idna_subject.py +0 -2
- PyFunceble/cli/processes/migrator.py +0 -1
- PyFunceble/cli/processes/workers/base.py +0 -1
- PyFunceble/cli/processes/workers/chancy_producer.py +1 -2
- PyFunceble/cli/processes/workers/dir_files_sorter.py +6 -14
- PyFunceble/cli/processes/workers/file_sorter.py +2 -9
- PyFunceble/cli/processes/workers/migrator.py +0 -1
- PyFunceble/cli/processes/workers/producer.py +2 -4
- PyFunceble/cli/processes/workers/tester.py +2 -3
- PyFunceble/cli/scripts/iana.py +14 -10
- PyFunceble/cli/scripts/production.py +2 -2
- PyFunceble/cli/scripts/public_suffix.py +2 -2
- PyFunceble/cli/system/integrator.py +1 -1
- PyFunceble/cli/system/launcher.py +14 -11
- PyFunceble/cli/utils/stdout.py +1 -1
- PyFunceble/cli/utils/testing.py +6 -16
- PyFunceble/config/compare.py +10 -9
- PyFunceble/config/loader.py +7 -4
- PyFunceble/converter/input_line2subject.py +2 -2
- PyFunceble/converter/internal_url.py +1 -1
- PyFunceble/converter/rpz_input_line2subject.py +2 -4
- PyFunceble/converter/subject2complements.py +4 -1
- PyFunceble/converter/url2netloc.py +3 -3
- PyFunceble/converter/wildcard2subject.py +3 -3
- PyFunceble/helpers/command.py +1 -1
- PyFunceble/helpers/dict.py +4 -4
- PyFunceble/helpers/file.py +2 -2
- PyFunceble/helpers/list.py +10 -7
- PyFunceble/helpers/merge.py +2 -2
- PyFunceble/helpers/regex.py +9 -11
- PyFunceble/query/dns/nameserver.py +9 -19
- PyFunceble/query/dns/query_tool.py +64 -205
- PyFunceble/query/dns/resolver.py +64 -51
- PyFunceble/query/netinfo/address.py +4 -6
- PyFunceble/query/netinfo/base.py +1 -1
- PyFunceble/query/netinfo/hostbyaddr.py +5 -8
- PyFunceble/query/requests/adapter/base.py +3 -3
- PyFunceble/query/requests/adapter/http.py +0 -1
- PyFunceble/query/requests/requester.py +6 -6
- PyFunceble/query/whois/converter/expiration_date.py +19 -8
- PyFunceble/query/whois/converter/month2unified.py +4 -6
- PyFunceble/query/whois/converter/registrar.py +18 -5
- PyFunceble/query/whois/query_tool.py +15 -18
- PyFunceble/storage.py +1 -1
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/METADATA +109 -108
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/RECORD +100 -100
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/WHEEL +1 -1
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/entry_points.txt +0 -0
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/licenses/LICENSE +0 -0
- {pyfunceble_dev-4.3.0a21.dist-info → pyfunceble_dev-4.3.0a24.dist-info}/top_level.txt +0 -0
@@ -56,8 +56,6 @@ from typing import Any, Generator, Tuple
|
|
56
56
|
from sqlalchemy.sql import text
|
57
57
|
|
58
58
|
import PyFunceble.cli.facility
|
59
|
-
import PyFunceble.cli.factory
|
60
|
-
import PyFunceble.sessions
|
61
59
|
from PyFunceble.cli.migrators.db_base import DBMigratorBase
|
62
60
|
|
63
61
|
|
@@ -72,7 +70,7 @@ class MariaDBMigratorBase(DBMigratorBase):
|
|
72
70
|
Otherwise, apply the given :code:`default`.
|
73
71
|
"""
|
74
72
|
|
75
|
-
def
|
73
|
+
def inner_method(func):
|
76
74
|
@functools.wraps(func)
|
77
75
|
def wrapper(self, *args, **kwargs):
|
78
76
|
if self.authorized:
|
@@ -81,7 +79,7 @@ class MariaDBMigratorBase(DBMigratorBase):
|
|
81
79
|
|
82
80
|
return wrapper
|
83
81
|
|
84
|
-
return
|
82
|
+
return inner_method
|
85
83
|
|
86
84
|
def get_rows(
|
87
85
|
self, statement: str, limit: int = 20
|
@@ -92,7 +92,7 @@ class FileAndStatusMigrator(MariaDBMigratorBase):
|
|
92
92
|
)
|
93
93
|
|
94
94
|
inactive_dataset = PyFunceble.cli.utils.testing.get_inactive_dataset_object()
|
95
|
-
continue_dataset = PyFunceble.cli.utils.testing.
|
95
|
+
continue_dataset = PyFunceble.cli.utils.testing.get_continue_dataset_object()
|
96
96
|
|
97
97
|
drop_table = True
|
98
98
|
|
@@ -52,9 +52,7 @@ License:
|
|
52
52
|
|
53
53
|
import domain2idna
|
54
54
|
|
55
|
-
import PyFunceble.cli.factory
|
56
55
|
import PyFunceble.facility
|
57
|
-
import PyFunceble.sessions
|
58
56
|
from PyFunceble.cli.migrators.mariadb.base import MariaDBMigratorBase
|
59
57
|
from PyFunceble.cli.utils.stdout import print_single_line
|
60
58
|
from PyFunceble.database.sqlalchemy.all_schemas import WhoisRecord
|
@@ -53,7 +53,6 @@ License:
|
|
53
53
|
from typing import Any, Optional, Tuple
|
54
54
|
|
55
55
|
import PyFunceble.facility
|
56
|
-
import PyFunceble.storage
|
57
56
|
from PyFunceble.checker.status_base import CheckerStatusBase
|
58
57
|
from PyFunceble.cli.processes.workers.producer import ProducerWorker
|
59
58
|
|
@@ -81,7 +80,7 @@ class ChancyProducerWorker(ProducerWorker):
|
|
81
80
|
|
82
81
|
if not isinstance(test_dataset, dict):
|
83
82
|
PyFunceble.facility.Logger.info(
|
84
|
-
"Skipping because test dataset is not a
|
83
|
+
"Skipping because test dataset is not a dictionary."
|
85
84
|
)
|
86
85
|
return None
|
87
86
|
|
@@ -149,26 +149,18 @@ class DireFileSorterWorker(FileSorterWorkerBase):
|
|
149
149
|
# Just for human brain :-)
|
150
150
|
directory = consumed["directory"]
|
151
151
|
|
152
|
-
|
153
|
-
|
154
|
-
else:
|
155
|
-
remove_duplicates = True
|
156
|
-
|
157
|
-
if "write_header" in consumed:
|
158
|
-
write_header = consumed["write_header"]
|
159
|
-
else:
|
160
|
-
write_header = True
|
152
|
+
remove_duplicates = consumed.get("remove_duplicates", True)
|
153
|
+
write_header = consumed.get("write_header", True)
|
161
154
|
|
162
155
|
with concurrent.futures.ThreadPoolExecutor(
|
163
156
|
max_workers=PyFunceble.storage.CONFIGURATION.cli_testing.max_workers,
|
164
157
|
) as executor:
|
165
|
-
submitted_list = [
|
166
|
-
|
167
|
-
for file in self.get_files_to_sort(directory):
|
168
|
-
submitted = executor.submit(
|
158
|
+
submitted_list = [
|
159
|
+
executor.submit(
|
169
160
|
self.process_file_sorting, file, remove_duplicates, write_header
|
170
161
|
)
|
171
|
-
|
162
|
+
for file in self.get_files_to_sort(directory)
|
163
|
+
]
|
172
164
|
|
173
165
|
for submitted in concurrent.futures.as_completed(submitted_list):
|
174
166
|
# Ensure that everything is finished
|
@@ -103,15 +103,8 @@ class FileSorterWorker(FileSorterWorkerBase):
|
|
103
103
|
# Just for human brain :-)
|
104
104
|
file = consumed["file"]
|
105
105
|
|
106
|
-
|
107
|
-
|
108
|
-
else:
|
109
|
-
remove_duplicates = True
|
110
|
-
|
111
|
-
if "write_header" in consumed:
|
112
|
-
write_header = consumed["write_header"]
|
113
|
-
else:
|
114
|
-
write_header = True
|
106
|
+
remove_duplicates = consumed.get("remove_duplicates", True)
|
107
|
+
write_header = consumed.get("write_header", True)
|
115
108
|
|
116
109
|
self.process_file_sorting(
|
117
110
|
file, remove_duplicates=remove_duplicates, write_header=write_header
|
@@ -68,7 +68,7 @@ from PyFunceble.cli.filesystem.status_file import StatusFileGenerator
|
|
68
68
|
from PyFunceble.cli.processes.workers.base import WorkerBase
|
69
69
|
from PyFunceble.cli.utils.stdout import get_template_to_use, print_single_line
|
70
70
|
from PyFunceble.cli.utils.testing import (
|
71
|
-
|
71
|
+
get_continue_dataset_object,
|
72
72
|
get_inactive_dataset_object,
|
73
73
|
)
|
74
74
|
from PyFunceble.dataset.autocontinue.base import ContinueDatasetBase
|
@@ -127,9 +127,7 @@ class ProducerWorker(WorkerBase):
|
|
127
127
|
)
|
128
128
|
self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
|
129
129
|
self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
|
130
|
-
self.continue_dataset =
|
131
|
-
db_session=self.db_session
|
132
|
-
)
|
130
|
+
self.continue_dataset = get_continue_dataset_object(db_session=self.db_session)
|
133
131
|
self.status_file_generator = StatusFileGenerator().guess_all_settings()
|
134
132
|
self.counter = FilesystemCounter()
|
135
133
|
self.registrar_counter = RegistrarCounter()
|
@@ -89,7 +89,7 @@ class TesterWorker(WorkerBase):
|
|
89
89
|
result = super().perform_external_poweron_checks()
|
90
90
|
|
91
91
|
self.continue_dataset = (
|
92
|
-
PyFunceble.cli.utils.testing.
|
92
|
+
PyFunceble.cli.utils.testing.get_continue_dataset_object(
|
93
93
|
db_session=self.db_session
|
94
94
|
)
|
95
95
|
)
|
@@ -215,8 +215,7 @@ class TesterWorker(WorkerBase):
|
|
215
215
|
|
216
216
|
if not isinstance(consumed, dict):
|
217
217
|
PyFunceble.facility.Logger.debug(
|
218
|
-
"Skipping latest dataset because consumed data was not "
|
219
|
-
"a dictionnary."
|
218
|
+
"Skipping latest dataset because consumed data was not a dictionary."
|
220
219
|
)
|
221
220
|
return None
|
222
221
|
|
PyFunceble/cli/scripts/iana.py
CHANGED
@@ -143,12 +143,18 @@ class IanaDBGenerator:
|
|
143
143
|
An internal storage map.
|
144
144
|
"""
|
145
145
|
|
146
|
+
whois_query_tool: Optional[WhoisQueryTool] = None
|
147
|
+
regex_helper: Optional[RegexHelper] = None
|
148
|
+
|
146
149
|
def __init__(self, destination: Optional[str] = None) -> None:
|
147
150
|
if destination is not None:
|
148
151
|
self.destination = destination
|
149
152
|
else:
|
150
153
|
self.destination = IanaDataset().source_file
|
151
154
|
|
155
|
+
self.whois_query_tool = WhoisQueryTool()
|
156
|
+
self.regex_helper = RegexHelper()
|
157
|
+
|
152
158
|
@property
|
153
159
|
def destination(self) -> Optional[str]:
|
154
160
|
"""
|
@@ -196,11 +202,9 @@ class IanaDBGenerator:
|
|
196
202
|
Given an extension, tries to get or guess its extension.
|
197
203
|
"""
|
198
204
|
|
199
|
-
whois_query_tool = WhoisQueryTool()
|
200
|
-
|
201
205
|
dummy_domain = f"hello.{extension}"
|
202
206
|
iana_record = (
|
203
|
-
whois_query_tool.set_server(self.IANA_WHOIS_SERVER)
|
207
|
+
self.whois_query_tool.set_server(self.IANA_WHOIS_SERVER)
|
204
208
|
.set_subject(dummy_domain)
|
205
209
|
.record
|
206
210
|
)
|
@@ -208,7 +212,7 @@ class IanaDBGenerator:
|
|
208
212
|
if iana_record and "refer" in iana_record:
|
209
213
|
regex_referrer = r"(?s)refer\:\s+([a-zA-Z0-9._-]+)\n"
|
210
214
|
|
211
|
-
matched =
|
215
|
+
matched = self.regex_helper.set_regex(regex_referrer).match(
|
212
216
|
iana_record, return_match=True, group=1
|
213
217
|
)
|
214
218
|
|
@@ -216,7 +220,7 @@ class IanaDBGenerator:
|
|
216
220
|
return matched
|
217
221
|
|
218
222
|
possible_server = f"whois.nic.{extension}"
|
219
|
-
response = whois_query_tool.set_server(possible_server).record
|
223
|
+
response = self.whois_query_tool.set_server(possible_server).record
|
220
224
|
|
221
225
|
if response:
|
222
226
|
return possible_server
|
@@ -224,7 +228,7 @@ class IanaDBGenerator:
|
|
224
228
|
if extension in self.MANUAL_SERVER:
|
225
229
|
possible_server = self.MANUAL_SERVER[extension]
|
226
230
|
|
227
|
-
response = whois_query_tool.set_server(possible_server).record
|
231
|
+
response = self.whois_query_tool.set_server(possible_server).record
|
228
232
|
|
229
233
|
if response:
|
230
234
|
return possible_server
|
@@ -247,10 +251,10 @@ class IanaDBGenerator:
|
|
247
251
|
|
248
252
|
regex_valid_extension = r"(/domains/root/db/)(.*)(\.html)"
|
249
253
|
|
250
|
-
regex_helper
|
251
|
-
|
252
|
-
|
253
|
-
extension = regex_helper.match(block, return_match=True, group=2)
|
254
|
+
if self.regex_helper.set_regex(regex_valid_extension).match(
|
255
|
+
block, return_match=False
|
256
|
+
):
|
257
|
+
extension = self.regex_helper.match(block, return_match=True, group=2)
|
254
258
|
|
255
259
|
if extension:
|
256
260
|
return extension, self.get_referrer_from_extension(extension)
|
@@ -86,7 +86,7 @@ class PublicSuffixGenerator:
|
|
86
86
|
An internal storage of our map.
|
87
87
|
"""
|
88
88
|
|
89
|
-
|
89
|
+
wildcard2subject: Wildcard2Subject = Wildcard2Subject()
|
90
90
|
|
91
91
|
def __init__(self, destination: Optional[str] = None) -> None:
|
92
92
|
if destination is not None:
|
@@ -147,7 +147,7 @@ class PublicSuffixGenerator:
|
|
147
147
|
if not any(line.startswith(x) for x in self.COMMENT_SIGN) and "." in line:
|
148
148
|
lines = [line, line.encode("idna").decode("utf-8")]
|
149
149
|
lines = [
|
150
|
-
self.
|
150
|
+
self.wildcard2subject.set_data_to_convert(x).get_converted()
|
151
151
|
for x in lines
|
152
152
|
]
|
153
153
|
extension = lines[0].rsplit(".", 1)[-1]
|
@@ -73,7 +73,7 @@ class SystemIntegrator(SystemBase):
|
|
73
73
|
Provides our system integrator. The idea is that we given an argparse
|
74
74
|
Namespace, we should be able to parse it into our system.
|
75
75
|
|
76
|
-
To simplify the trick and headache in the CLI management, I
|
76
|
+
To simplify the trick and headache in the CLI management, I explicitly
|
77
77
|
mapped the :code:`dest` argument to what we are supposed to have in the
|
78
78
|
flatten version of the configuration. That way, we only need to compare
|
79
79
|
against the flatten version instead of looping over all possible levels
|
@@ -69,7 +69,6 @@ import domain2idna
|
|
69
69
|
import sqlalchemy.exc
|
70
70
|
from sqlalchemy.orm import Session
|
71
71
|
|
72
|
-
import PyFunceble.checker.utils.whois
|
73
72
|
import PyFunceble.cli.storage
|
74
73
|
import PyFunceble.cli.utils.ascii_logo
|
75
74
|
import PyFunceble.cli.utils.sort
|
@@ -78,6 +77,7 @@ import PyFunceble.facility
|
|
78
77
|
import PyFunceble.helpers.exceptions
|
79
78
|
import PyFunceble.storage
|
80
79
|
from PyFunceble.checker.syntax.url import URLSyntaxChecker
|
80
|
+
from PyFunceble.checker.utils.whois import get_whois_dataset_object
|
81
81
|
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
|
82
82
|
from PyFunceble.cli.continuous_integration.exceptions import StopExecution
|
83
83
|
from PyFunceble.cli.continuous_integration.utils import ci_object
|
@@ -101,7 +101,7 @@ from PyFunceble.cli.processes.producer import ProducerProcessesManager
|
|
101
101
|
from PyFunceble.cli.processes.tester import TesterProcessesManager
|
102
102
|
from PyFunceble.cli.system.base import SystemBase
|
103
103
|
from PyFunceble.cli.utils.testing import (
|
104
|
-
|
104
|
+
get_continue_dataset_object,
|
105
105
|
get_destination_from_origin,
|
106
106
|
get_inactive_dataset_object,
|
107
107
|
get_subjects_from_line,
|
@@ -119,6 +119,7 @@ from PyFunceble.converter.wildcard2subject import Wildcard2Subject
|
|
119
119
|
from PyFunceble.dataset.autocontinue.base import ContinueDatasetBase
|
120
120
|
from PyFunceble.dataset.autocontinue.csv import CSVContinueDataset
|
121
121
|
from PyFunceble.dataset.inactive.base import InactiveDatasetBase
|
122
|
+
from PyFunceble.dataset.whois.base import WhoisDatasetBase
|
122
123
|
from PyFunceble.helpers.directory import DirectoryHelper
|
123
124
|
from PyFunceble.helpers.download import DownloadHelper
|
124
125
|
from PyFunceble.helpers.file import FileHelper
|
@@ -167,6 +168,7 @@ class SystemLauncher(SystemBase):
|
|
167
168
|
|
168
169
|
continue_dataset: Optional[ContinueDatasetBase] = None
|
169
170
|
inactive_dataset: Optional[InactiveDatasetBase] = None
|
171
|
+
whois_dataset: Optional[WhoisDatasetBase] = None
|
170
172
|
continuous_integration: Optional[ContinuousIntegrationBase] = None
|
171
173
|
|
172
174
|
db_session: Optional[Session] = None
|
@@ -185,10 +187,9 @@ class SystemLauncher(SystemBase):
|
|
185
187
|
|
186
188
|
self.execution_time_holder = ExecutionTime().set_start_time()
|
187
189
|
self.checker_type = get_testing_mode()
|
188
|
-
self.continue_dataset =
|
189
|
-
db_session=self.db_session
|
190
|
-
)
|
190
|
+
self.continue_dataset = get_continue_dataset_object(db_session=self.db_session)
|
191
191
|
self.inactive_dataset = get_inactive_dataset_object(db_session=self.db_session)
|
192
|
+
self.whois_dataset = get_whois_dataset_object(db_session=self.db_session)
|
192
193
|
self.continuous_integration = ci_object()
|
193
194
|
|
194
195
|
if self.continuous_integration.authorized:
|
@@ -772,7 +773,7 @@ class SystemLauncher(SystemBase):
|
|
772
773
|
def generate_waiting_files(self) -> "SystemLauncher":
|
773
774
|
"""
|
774
775
|
Generates all the files that needs to be generated when all status
|
775
|
-
are
|
776
|
+
are processes.
|
776
777
|
"""
|
777
778
|
|
778
779
|
def generate_percentage_file(parent_dirname: Union[str, None]) -> None:
|
@@ -998,18 +999,19 @@ class SystemLauncher(SystemBase):
|
|
998
999
|
"""
|
999
1000
|
Remove the inline destination - when necessary.
|
1000
1001
|
|
1001
|
-
:param
|
1002
|
+
:param protocol:
|
1002
1003
|
The protocol to work with.
|
1003
1004
|
"""
|
1004
1005
|
|
1005
1006
|
if not protocol["destination"]:
|
1006
|
-
|
1007
|
+
directory_helper.set_path(
|
1007
1008
|
self.counter.set_differ_to_inline(True)
|
1008
1009
|
.set_parent_dirname(protocol["destination"])
|
1009
1010
|
.get_output_basedir()
|
1010
1011
|
).delete()
|
1011
1012
|
|
1012
1013
|
file_helper = FileHelper()
|
1014
|
+
directory_helper = DirectoryHelper()
|
1013
1015
|
|
1014
1016
|
for protocol in self.testing_protocol:
|
1015
1017
|
if "destination" in protocol or "output_dir" in protocol:
|
@@ -1019,12 +1021,13 @@ class SystemLauncher(SystemBase):
|
|
1019
1021
|
remove_continue_dataset(protocol)
|
1020
1022
|
remove_preload_dataset(protocol)
|
1021
1023
|
remove_inline_dest(protocol)
|
1024
|
+
self.whois_dataset.cleanup()
|
1022
1025
|
|
1023
1026
|
return self
|
1024
1027
|
|
1025
1028
|
def run_standard_end_instructions(self) -> "SystemLauncher":
|
1026
1029
|
"""
|
1027
|
-
|
1030
|
+
Runs our standard "end" instructions.
|
1028
1031
|
|
1029
1032
|
The instructions executed by this method are the one we execute normally.
|
1030
1033
|
|
@@ -1045,7 +1048,7 @@ class SystemLauncher(SystemBase):
|
|
1045
1048
|
|
1046
1049
|
def run_ci_saving_instructions(self) -> "SystemLauncher":
|
1047
1050
|
"""
|
1048
|
-
|
1051
|
+
Runs our CI "saving" instructions.
|
1049
1052
|
|
1050
1053
|
The instructions executed by this method are the one we execute
|
1051
1054
|
before ending a testing session under one of the supported CI engines.
|
@@ -1067,7 +1070,7 @@ class SystemLauncher(SystemBase):
|
|
1067
1070
|
|
1068
1071
|
def run_ci_end_saving_instructions(self) -> "SystemLauncher":
|
1069
1072
|
"""
|
1070
|
-
|
1073
|
+
Runs our CI END "saving" instructions.
|
1071
1074
|
|
1072
1075
|
The instructions executed by this method are the one we execute
|
1073
1076
|
before ending a testing session under one of the supported CI engines.
|
PyFunceble/cli/utils/stdout.py
CHANGED
PyFunceble/cli/utils/testing.py
CHANGED
@@ -90,7 +90,7 @@ def get_testing_mode() -> str:
|
|
90
90
|
return "UNKNOWN"
|
91
91
|
|
92
92
|
|
93
|
-
def
|
93
|
+
def get_continue_dataset_object(
|
94
94
|
db_session: Optional[Session] = None,
|
95
95
|
) -> Union[DatasetBase, CSVDatasetBase, DBDatasetBase]:
|
96
96
|
"""
|
@@ -199,35 +199,25 @@ def get_subjects_from_line(
|
|
199
199
|
result = []
|
200
200
|
|
201
201
|
if adblock_inputline2subject is None:
|
202
|
-
adblock_inputline2subject = AdblockInputLine2Subject(
|
203
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
204
|
-
)
|
202
|
+
adblock_inputline2subject = AdblockInputLine2Subject()
|
205
203
|
|
206
204
|
if wildcard2subject is None:
|
207
|
-
wildcard2subject = Wildcard2Subject(
|
208
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
209
|
-
)
|
205
|
+
wildcard2subject = Wildcard2Subject()
|
210
206
|
|
211
207
|
if rpz_policy2subject is None:
|
212
208
|
rpz_policy2subject = RPZPolicy2Subject()
|
213
209
|
|
214
210
|
if rpz_inputline2subject is None:
|
215
|
-
rpz_inputline2subject = RPZInputLine2Subject(
|
216
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
217
|
-
)
|
211
|
+
rpz_inputline2subject = RPZInputLine2Subject()
|
218
212
|
|
219
213
|
if inputline2subject is None:
|
220
|
-
inputline2subject = InputLine2Subject(
|
221
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
222
|
-
)
|
214
|
+
inputline2subject = InputLine2Subject()
|
223
215
|
|
224
216
|
if subject2complements is None:
|
225
217
|
subject2complements = Subject2Complements()
|
226
218
|
|
227
219
|
if url2netloc is None:
|
228
|
-
url2netloc = Url2Netloc(
|
229
|
-
aggressive=bool(PyFunceble.storage.CONFIGURATION.cli_decoding.aggressive)
|
230
|
-
)
|
220
|
+
url2netloc = Url2Netloc()
|
231
221
|
|
232
222
|
if cidr2subject is None:
|
233
223
|
cidr2subject = CIDR2Subject()
|
PyFunceble/config/compare.py
CHANGED
@@ -11,7 +11,7 @@ The tool to check the availability or syntax of domain, IP or URL.
|
|
11
11
|
██║ ██║ ██║ ╚██████╔╝██║ ╚████║╚██████╗███████╗██████╔╝███████╗███████╗
|
12
12
|
╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═══╝ ╚═════╝╚══════╝╚═════╝ ╚══════╝╚══════╝
|
13
13
|
|
14
|
-
Provides the configuration
|
14
|
+
Provides the configuration comparison interface.
|
15
15
|
|
16
16
|
Author:
|
17
17
|
Nissar Chababy, @funilrys, contactTATAfunilrysTODTODcom
|
@@ -175,7 +175,7 @@ class ConfigComparison:
|
|
175
175
|
}
|
176
176
|
|
177
177
|
_local_config: dict = {}
|
178
|
-
|
178
|
+
_upstream_config: dict = {}
|
179
179
|
|
180
180
|
dict_helper: DictHelper = DictHelper()
|
181
181
|
|
@@ -228,12 +228,12 @@ class ConfigComparison:
|
|
228
228
|
Provides the current state of the :code:`_upstream_config`.
|
229
229
|
"""
|
230
230
|
|
231
|
-
return self.
|
231
|
+
return self._upstream_config
|
232
232
|
|
233
233
|
@upstream_config.setter
|
234
234
|
def upstream_config(self, value: dict) -> None:
|
235
235
|
"""
|
236
|
-
Sets the
|
236
|
+
Sets the upstream configuration to work with.
|
237
237
|
|
238
238
|
:raise TypeError:
|
239
239
|
When :code:`value` is not a :py:class:`dict`
|
@@ -242,11 +242,11 @@ class ConfigComparison:
|
|
242
242
|
if not isinstance(value, dict):
|
243
243
|
raise TypeError(f"<value> should be {dict}, {type(value)} given.")
|
244
244
|
|
245
|
-
self.
|
245
|
+
self._upstream_config = copy.deepcopy(value)
|
246
246
|
|
247
247
|
def set_upstream_config(self, value: dict) -> "ConfigComparison":
|
248
248
|
"""
|
249
|
-
Sets the
|
249
|
+
Sets the upstream configuration to work with.
|
250
250
|
"""
|
251
251
|
|
252
252
|
self.upstream_config = value
|
@@ -265,6 +265,7 @@ class ConfigComparison:
|
|
265
265
|
)
|
266
266
|
or "user_agent" not in self.local_config
|
267
267
|
or not isinstance(self.local_config["user_agent"], dict)
|
268
|
+
or "http_codes" not in self.local_config
|
268
269
|
or "active" in self.local_config["http_codes"]
|
269
270
|
or "not_found_default" in self.local_config["http_codes"]
|
270
271
|
or "self_managed" not in self.local_config["http_codes"]
|
@@ -281,7 +282,7 @@ class ConfigComparison:
|
|
281
282
|
if index in self.DELETED_CORE:
|
282
283
|
return False
|
283
284
|
|
284
|
-
for index in self.local_config
|
285
|
+
for index in self.local_config.get("links", {}):
|
285
286
|
if index in self.DELETED_LINKS:
|
286
287
|
return False
|
287
288
|
|
@@ -369,10 +370,10 @@ class ConfigComparison:
|
|
369
370
|
del merged[index]
|
370
371
|
|
371
372
|
for index in self.DELETED_LINKS:
|
372
|
-
if index in merged
|
373
|
+
if index in merged.get("links", {}):
|
373
374
|
del merged["links"][index]
|
374
375
|
|
375
|
-
if not bool(merged["http_codes"]["self_managed"]):
|
376
|
+
if "http_codes" in merged and not bool(merged["http_codes"]["self_managed"]):
|
376
377
|
for index, values in PyFunceble.storage.STD_HTTP_CODES.list.items():
|
377
378
|
merged["http_codes"]["list"][index] = list(values)
|
378
379
|
|
PyFunceble/config/loader.py
CHANGED
@@ -99,8 +99,8 @@ class ConfigLoader:
|
|
99
99
|
_config_dir: Optional[str] = None
|
100
100
|
__config_loaded: bool = False
|
101
101
|
|
102
|
-
file_helper: FileHelper =
|
103
|
-
dict_helper: DictHelper =
|
102
|
+
file_helper: Optional[FileHelper] = None
|
103
|
+
dict_helper: Optional[DictHelper] = None
|
104
104
|
|
105
105
|
def __init__(
|
106
106
|
self, merge_upstream: Optional[bool] = None, *, config_dir: Optional[str] = None
|
@@ -123,6 +123,9 @@ class ConfigLoader:
|
|
123
123
|
elif EnvironmentVariableHelper("PYFUNCEBLE_AUTO_CONFIGURATION").exists():
|
124
124
|
self.merge_upstream = True
|
125
125
|
|
126
|
+
self.file_helper = FileHelper()
|
127
|
+
self.dict_helper = DictHelper()
|
128
|
+
|
126
129
|
def __del__(self) -> None:
|
127
130
|
self.destroy()
|
128
131
|
|
@@ -451,7 +454,7 @@ class ConfigLoader:
|
|
451
454
|
self,
|
452
455
|
) -> "ConfigLoader":
|
453
456
|
"""
|
454
|
-
Downloads all the
|
457
|
+
Downloads all the dynamically (generated) infrastructure files.
|
455
458
|
|
456
459
|
.. note::
|
457
460
|
Downloaded if missing:
|
@@ -532,7 +535,7 @@ class ConfigLoader:
|
|
532
535
|
or self.merge_upstream
|
533
536
|
or is_3_x_version(config)
|
534
537
|
or not config_comparer.is_local_identical()
|
535
|
-
): # pragma: no cover ## Testing the underlying comparison method is
|
538
|
+
): # pragma: no cover ## Testing the underlying comparison method is sufficient
|
536
539
|
config = config_comparer.get_merged()
|
537
540
|
|
538
541
|
self.dict_helper.set_subject(config).to_yaml_file(self.path_to_config)
|
@@ -60,7 +60,7 @@ from PyFunceble.converter.url2netloc import Url2Netloc
|
|
60
60
|
|
61
61
|
class InputLine2Subject(ConverterBase):
|
62
62
|
"""
|
63
|
-
Converts/Extract the
|
63
|
+
Converts/Extract the subjects to test from an imputed line.
|
64
64
|
"""
|
65
65
|
|
66
66
|
COMMENT: str = "#"
|
@@ -72,7 +72,7 @@ class InputLine2Subject(ConverterBase):
|
|
72
72
|
@ConverterBase.data_to_convert.setter
|
73
73
|
def data_to_convert(self, value: Any) -> None:
|
74
74
|
"""
|
75
|
-
|
75
|
+
Overrides the default behavior.
|
76
76
|
|
77
77
|
:raise TypeError:
|
78
78
|
When the given data to convert is not :py:class:`str`
|
@@ -71,7 +71,7 @@ class InternalUrlConverter(ConverterBase):
|
|
71
71
|
@ConverterBase.data_to_convert.setter
|
72
72
|
def data_to_convert(self, value: Any) -> None:
|
73
73
|
"""
|
74
|
-
|
74
|
+
Overrides the default behavior.
|
75
75
|
|
76
76
|
:raise TypeError:
|
77
77
|
When the given data to convert is not :py:class:`str`
|
@@ -59,7 +59,7 @@ from PyFunceble.converter.input_line2subject import InputLine2Subject
|
|
59
59
|
|
60
60
|
class RPZInputLine2Subject(InputLine2Subject):
|
61
61
|
"""
|
62
|
-
Converts/Extracts the subject from the given RPZ
|
62
|
+
Converts/Extracts the subject from the given RPZ input line.
|
63
63
|
"""
|
64
64
|
|
65
65
|
COMMENT: list = [";", "//", "#"]
|
@@ -96,9 +96,7 @@ class RPZInputLine2Subject(InputLine2Subject):
|
|
96
96
|
if self.SPACE in subject or self.TAB in subject:
|
97
97
|
subject = subject.split()[0]
|
98
98
|
|
99
|
-
|
100
|
-
result.append(subject)
|
101
|
-
elif not subject.isdigit():
|
99
|
+
if not subject.isdigit():
|
102
100
|
result.append(subject)
|
103
101
|
|
104
102
|
return result
|