PyFunceble-dev 4.2.29__py3-none-any.whl → 4.3.0a1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- PyFunceble/cli/entry_points/pyfunceble/cli.py +22 -0
- PyFunceble/cli/filesystem/dir_structure/base.py +1 -2
- PyFunceble/cli/migrators/base.py +47 -1
- PyFunceble/cli/migrators/csv_file/inactive_source_delete.py +1 -1
- PyFunceble/cli/migrators/csv_file/whois_registrar_add.py +1 -1
- PyFunceble/cli/migrators/file_cleanup/hashes_file.py +1 -1
- PyFunceble/cli/migrators/file_cleanup/mining_file.py +1 -1
- PyFunceble/cli/migrators/file_cleanup/production_config_file.py +1 -1
- PyFunceble/cli/migrators/json2csv/inactive.py +1 -1
- PyFunceble/cli/migrators/json2csv/whois.py +1 -1
- PyFunceble/cli/scripts/production.py +67 -26
- PyFunceble/cli/system/integrator.py +51 -14
- PyFunceble/cli/system/launcher.py +14 -5
- PyFunceble/config/loader.py +146 -22
- PyFunceble/database/credential/base.py +46 -3
- PyFunceble/dataset/base.py +3 -3
- PyFunceble/dataset/db_base.py +44 -0
- PyFunceble/dataset/iana.py +3 -6
- PyFunceble/dataset/inactive/csv.py +1 -1
- PyFunceble/dataset/ipv4_reputation.py +5 -9
- PyFunceble/dataset/public_suffix.py +4 -8
- PyFunceble/dataset/user_agent.py +4 -7
- PyFunceble/dataset/whois/csv.py +1 -1
- PyFunceble/downloader/base.py +64 -5
- PyFunceble/downloader/iana.py +4 -11
- PyFunceble/downloader/ipv4_reputation.py +2 -11
- PyFunceble/downloader/public_suffix.py +5 -11
- PyFunceble/downloader/user_agents.py +5 -11
- PyFunceble/query/whois/query_tool.py +1 -0
- PyFunceble/storage.py +2 -19
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/METADATA +58 -58
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/RECORD +36 -36
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/LICENSE +0 -0
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/WHEEL +0 -0
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/entry_points.txt +0 -0
- {PyFunceble_dev-4.2.29.dist-info → PyFunceble_dev-4.3.0a1.dist-info}/top_level.txt +0 -0
@@ -1197,6 +1197,28 @@ def get_default_group_data() -> List[Tuple[List[str], dict]]:
|
|
1197
1197
|
"version": "%(prog)s " + PyFunceble.storage.PROJECT_VERSION,
|
1198
1198
|
},
|
1199
1199
|
),
|
1200
|
+
(
|
1201
|
+
[
|
1202
|
+
"--config-file",
|
1203
|
+
],
|
1204
|
+
{
|
1205
|
+
"dest": "config_file",
|
1206
|
+
"type": str,
|
1207
|
+
"help": "Sets the configuration file to use. It can be a\n"
|
1208
|
+
"local or remote file. Please note that this configuration can be\n"
|
1209
|
+
"overwritten by your overwrite configuration file.",
|
1210
|
+
},
|
1211
|
+
),
|
1212
|
+
(
|
1213
|
+
[
|
1214
|
+
"--config-dir",
|
1215
|
+
],
|
1216
|
+
{
|
1217
|
+
"dest": "config_dir",
|
1218
|
+
"type": os.path.realpath,
|
1219
|
+
"help": "Sets the configuration directory to use.",
|
1220
|
+
},
|
1221
|
+
),
|
1200
1222
|
]
|
1201
1223
|
|
1202
1224
|
|
@@ -59,7 +59,6 @@ except ImportError: # pragma: no cover ## Retro compatibility
|
|
59
59
|
import importlib_resources as package_resources
|
60
60
|
|
61
61
|
|
62
|
-
import PyFunceble.storage
|
63
62
|
from PyFunceble.cli.filesystem.cleanup import FilesystemCleanup
|
64
63
|
from PyFunceble.cli.filesystem.dir_base import FilesystemDirBase
|
65
64
|
|
@@ -78,7 +77,7 @@ class DirectoryStructureBase(FilesystemDirBase):
|
|
78
77
|
) -> None:
|
79
78
|
with package_resources.path(
|
80
79
|
"PyFunceble.data.infrastructure",
|
81
|
-
|
80
|
+
"dir_structure_production.json",
|
82
81
|
) as file_path:
|
83
82
|
self.std_source_file = str(file_path)
|
84
83
|
|
PyFunceble/cli/migrators/base.py
CHANGED
@@ -54,6 +54,7 @@ from typing import Optional
|
|
54
54
|
|
55
55
|
from sqlalchemy.orm import Session
|
56
56
|
|
57
|
+
import PyFunceble.storage
|
57
58
|
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
|
58
59
|
|
59
60
|
|
@@ -66,11 +67,19 @@ class MigratorBase:
|
|
66
67
|
continuous_integration: Optional[ContinuousIntegrationBase] = None
|
67
68
|
db_session: Optional[Session] = None
|
68
69
|
|
70
|
+
_config_dir: Optional[str] = None
|
69
71
|
print_action_to_stdout: bool = False
|
70
72
|
|
71
|
-
def __init__(
|
73
|
+
def __init__(
|
74
|
+
self, print_action_to_stdout: bool = False, *, config_dir: Optional[str] = None
|
75
|
+
) -> None:
|
72
76
|
self.print_action_to_stdout = print_action_to_stdout
|
73
77
|
|
78
|
+
if config_dir is not None:
|
79
|
+
self._config_dir = config_dir
|
80
|
+
else:
|
81
|
+
self._config_dir = PyFunceble.storage.CONFIG_DIRECTORY
|
82
|
+
|
74
83
|
self.__post_init__()
|
75
84
|
|
76
85
|
def __post_init__(self) -> None:
|
@@ -78,6 +87,43 @@ class MigratorBase:
|
|
78
87
|
A method to be called (automatically) after the __init__ execution.
|
79
88
|
"""
|
80
89
|
|
90
|
+
@property
|
91
|
+
def config_dir(self) -> Optional[str]:
|
92
|
+
"""
|
93
|
+
Provides the current state of the :code:`_config_dir` attribute.
|
94
|
+
"""
|
95
|
+
|
96
|
+
return self._config_dir
|
97
|
+
|
98
|
+
@config_dir.setter
|
99
|
+
def config_dir(self, value: str) -> None:
|
100
|
+
"""
|
101
|
+
Sets the configuration directory.
|
102
|
+
|
103
|
+
:param value:
|
104
|
+
The value to set.
|
105
|
+
|
106
|
+
:raise TypeError:
|
107
|
+
When value is not a :py:class:`str`.
|
108
|
+
"""
|
109
|
+
|
110
|
+
if not isinstance(value, str):
|
111
|
+
raise TypeError(f"<value> should be {str}, {type(value)} given.")
|
112
|
+
|
113
|
+
self._config_dir = value
|
114
|
+
|
115
|
+
def set_config_dir(self, value: str) -> "MigratorBase":
|
116
|
+
"""
|
117
|
+
Sets the configuration directory.
|
118
|
+
|
119
|
+
:param value:
|
120
|
+
The value to set.
|
121
|
+
"""
|
122
|
+
|
123
|
+
self.config_dir = value
|
124
|
+
|
125
|
+
return self
|
126
|
+
|
81
127
|
def start(self) -> "MigratorBase":
|
82
128
|
"""
|
83
129
|
Starts the migration.
|
@@ -77,7 +77,7 @@ class InactiveDatasetDeleteSourceColumnMigrator(CSVFileMigratorBase):
|
|
77
77
|
|
78
78
|
def __post_init__(self) -> None:
|
79
79
|
self.source_file = os.path.join(
|
80
|
-
|
80
|
+
self.config_dir, PyFunceble.cli.storage.INACTIVE_DB_FILE
|
81
81
|
)
|
82
82
|
|
83
83
|
return super().__post_init__()
|
@@ -71,7 +71,7 @@ class WhoisDatasetAddRegistrarColumnMigrator(CSVFileMigratorBase):
|
|
71
71
|
|
72
72
|
def __post_init__(self) -> None:
|
73
73
|
self.source_file = os.path.join(
|
74
|
-
|
74
|
+
self.config_dir, PyFunceble.cli.storage.WHOIS_DB_FILE
|
75
75
|
)
|
76
76
|
|
77
77
|
return super().__post_init__()
|
@@ -64,7 +64,7 @@ class HashesFileCleanupMigrator(FileClenupMigratorBase):
|
|
64
64
|
|
65
65
|
def __post_init__(self) -> None:
|
66
66
|
self.source_file = os.path.join(
|
67
|
-
|
67
|
+
self.config_dir, PyFunceble.cli.storage.HASHES_FILENAME
|
68
68
|
)
|
69
69
|
|
70
70
|
return super().__post_init__()
|
@@ -64,7 +64,7 @@ class MiningFileCleanupMigrator(FileClenupMigratorBase):
|
|
64
64
|
|
65
65
|
def __post_init__(self) -> None:
|
66
66
|
self.source_file = os.path.join(
|
67
|
-
|
67
|
+
self.config_dir, PyFunceble.cli.storage.MINING_OLD_FILE
|
68
68
|
)
|
69
69
|
|
70
70
|
return super().__post_init__()
|
@@ -65,7 +65,7 @@ class ProductionConfigFileCleanupMigrator(FileClenupMigratorBase):
|
|
65
65
|
|
66
66
|
def __post_init__(self) -> None:
|
67
67
|
self.source_file = os.path.join(
|
68
|
-
|
68
|
+
self.config_dir,
|
69
69
|
PyFunceble.storage.DISTRIBUTED_CONFIGURATION_FILENAME,
|
70
70
|
)
|
71
71
|
|
@@ -84,10 +84,6 @@ class ProductionPrep:
|
|
84
84
|
you should run this only if your are developing PyFunceble.
|
85
85
|
"""
|
86
86
|
|
87
|
-
VERSION_FILE_PATH: str = os.path.join(
|
88
|
-
PyFunceble.storage.CONFIG_DIRECTORY,
|
89
|
-
PyFunceble.cli.storage.DISTRIBUTED_VERSION_FILENAME,
|
90
|
-
)
|
91
87
|
AVAILABLE_BRANCHES: List[str] = ["dev", "master"]
|
92
88
|
|
93
89
|
regex_helper: RegexHelper = RegexHelper()
|
@@ -101,15 +97,23 @@ class ProductionPrep:
|
|
101
97
|
"""
|
102
98
|
|
103
99
|
_branch: Optional[str] = None
|
100
|
+
_config_dir: Optional[str] = None
|
104
101
|
|
105
102
|
previous_version: Optional[str] = None
|
106
103
|
"""
|
107
104
|
Provides the previous version (from :code:`version_file_content`)
|
108
105
|
"""
|
109
106
|
|
110
|
-
def __init__(
|
107
|
+
def __init__(
|
108
|
+
self, branch: Optional[str] = None, *, config_dir: Optional[str] = None
|
109
|
+
) -> None:
|
110
|
+
if config_dir is not None:
|
111
|
+
self.config_dir = config_dir
|
112
|
+
else:
|
113
|
+
self.config_dir = PyFunceble.storage.CONFIG_DIRECTORY
|
114
|
+
|
111
115
|
self.version_file_content = self.dict_helper.from_yaml_file(
|
112
|
-
self.
|
116
|
+
self.version_file_path
|
113
117
|
)
|
114
118
|
|
115
119
|
self.previous_version = copy.deepcopy(
|
@@ -138,6 +142,54 @@ class ProductionPrep:
|
|
138
142
|
|
139
143
|
return wrapper
|
140
144
|
|
145
|
+
@property
|
146
|
+
def config_dir(self) -> Optional[str]:
|
147
|
+
"""
|
148
|
+
Provides the current state of the :code:`_config_dir` attribute.
|
149
|
+
"""
|
150
|
+
|
151
|
+
return self._config_dir
|
152
|
+
|
153
|
+
@config_dir.setter
|
154
|
+
def config_dir(self, value: str) -> None:
|
155
|
+
"""
|
156
|
+
Sets the configuration directory.
|
157
|
+
|
158
|
+
:param value:
|
159
|
+
The value to set.
|
160
|
+
|
161
|
+
:raise TypeError:
|
162
|
+
When value is not a :py:class:`str`.
|
163
|
+
"""
|
164
|
+
|
165
|
+
if not isinstance(value, str):
|
166
|
+
raise TypeError(f"<value> should be {str}, {type(value)} given.")
|
167
|
+
|
168
|
+
self._config_dir = value
|
169
|
+
|
170
|
+
def set_config_dir(self, value: str) -> "ProductionPrep":
|
171
|
+
"""
|
172
|
+
Sets the configuration directory.
|
173
|
+
|
174
|
+
:param value:
|
175
|
+
The value to set.
|
176
|
+
"""
|
177
|
+
|
178
|
+
self.config_dir = value
|
179
|
+
|
180
|
+
return self
|
181
|
+
|
182
|
+
@property
|
183
|
+
def version_file_path(self) -> str:
|
184
|
+
"""
|
185
|
+
Provides the path to the version file.
|
186
|
+
"""
|
187
|
+
|
188
|
+
return os.path.join(
|
189
|
+
self.config_dir,
|
190
|
+
PyFunceble.cli.storage.DISTRIBUTED_VERSION_FILENAME,
|
191
|
+
)
|
192
|
+
|
141
193
|
@property
|
142
194
|
def branch(self) -> Optional[str]:
|
143
195
|
"""
|
@@ -270,8 +322,7 @@ class ProductionPrep:
|
|
270
322
|
|
271
323
|
return self
|
272
324
|
|
273
|
-
|
274
|
-
def update_code_format() -> "ProductionPrep":
|
325
|
+
def update_code_format(self) -> "ProductionPrep":
|
275
326
|
"""
|
276
327
|
Updates the format of the source code using black.
|
277
328
|
"""
|
@@ -304,16 +355,14 @@ class ProductionPrep:
|
|
304
355
|
isort_config = isort.settings.Config(settings_file="setup.cfg")
|
305
356
|
|
306
357
|
files = [
|
307
|
-
os.path.join(
|
358
|
+
os.path.join(self.config_dir, "setup.py"),
|
308
359
|
]
|
309
360
|
|
310
361
|
for file in files:
|
311
362
|
format_file(file, isort_config)
|
312
363
|
|
313
364
|
for root, _, files in os.walk(
|
314
|
-
os.path.join(
|
315
|
-
PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME
|
316
|
-
)
|
365
|
+
os.path.join(self.config_dir, PyFunceble.storage.PROJECT_NAME)
|
317
366
|
):
|
318
367
|
if "__pycache__" in root:
|
319
368
|
continue
|
@@ -324,9 +373,7 @@ class ProductionPrep:
|
|
324
373
|
|
325
374
|
format_file(os.path.join(root, file), isort_config)
|
326
375
|
|
327
|
-
for root, _, files in os.walk(
|
328
|
-
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests")
|
329
|
-
):
|
376
|
+
for root, _, files in os.walk(os.path.join(self.config_dir, "tests")):
|
330
377
|
if "__pycache__" in root:
|
331
378
|
continue
|
332
379
|
|
@@ -346,12 +393,10 @@ class ProductionPrep:
|
|
346
393
|
".keep",
|
347
394
|
]
|
348
395
|
|
349
|
-
self.update_urls(os.path.join(
|
396
|
+
self.update_urls(os.path.join(self.config_dir, "setup.py"))
|
350
397
|
|
351
398
|
for root, _, files in os.walk(
|
352
|
-
os.path.join(
|
353
|
-
PyFunceble.storage.CONFIG_DIRECTORY, PyFunceble.storage.PROJECT_NAME
|
354
|
-
)
|
399
|
+
os.path.join(self.config_dir, PyFunceble.storage.PROJECT_NAME)
|
355
400
|
):
|
356
401
|
if "__pycache__" in root:
|
357
402
|
continue
|
@@ -362,9 +407,7 @@ class ProductionPrep:
|
|
362
407
|
|
363
408
|
self.update_urls(os.path.join(root, file))
|
364
409
|
|
365
|
-
for root, _, files in os.walk(
|
366
|
-
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "tests")
|
367
|
-
):
|
410
|
+
for root, _, files in os.walk(os.path.join(self.config_dir, "tests")):
|
368
411
|
if "__pycache__" in root:
|
369
412
|
continue
|
370
413
|
|
@@ -409,9 +452,7 @@ class ProductionPrep:
|
|
409
452
|
),
|
410
453
|
]
|
411
454
|
|
412
|
-
self.file_helper.set_path(
|
413
|
-
os.path.join(PyFunceble.storage.CONFIG_DIRECTORY, "setup.py")
|
414
|
-
)
|
455
|
+
self.file_helper.set_path(os.path.join(self.config_dir, "setup.py"))
|
415
456
|
|
416
457
|
if not self.file_helper.exists():
|
417
458
|
raise FileNotFoundError(self.file_helper.path)
|
@@ -453,7 +494,7 @@ class ProductionPrep:
|
|
453
494
|
)
|
454
495
|
|
455
496
|
self.dict_helper.set_subject(self.version_file_content).to_yaml_file(
|
456
|
-
self.
|
497
|
+
self.version_file_path
|
457
498
|
)
|
458
499
|
|
459
500
|
PyFunceble.facility.Logger.info(
|
@@ -52,6 +52,8 @@ License:
|
|
52
52
|
"""
|
53
53
|
|
54
54
|
import os
|
55
|
+
import sys
|
56
|
+
import traceback
|
55
57
|
|
56
58
|
import colorama
|
57
59
|
|
@@ -59,7 +61,9 @@ import PyFunceble.cli.facility
|
|
59
61
|
import PyFunceble.cli.factory
|
60
62
|
import PyFunceble.cli.storage
|
61
63
|
import PyFunceble.facility
|
64
|
+
import PyFunceble.helpers.exceptions
|
62
65
|
import PyFunceble.storage
|
66
|
+
from PyFunceble.cli.continuous_integration.exceptions import StopExecution
|
63
67
|
from PyFunceble.cli.system.base import SystemBase
|
64
68
|
from PyFunceble.helpers.dict import DictHelper
|
65
69
|
|
@@ -250,23 +254,56 @@ class SystemIntegrator(SystemBase):
|
|
250
254
|
Starts a group of actions provided by this interface.
|
251
255
|
"""
|
252
256
|
|
253
|
-
|
254
|
-
|
255
|
-
os.path.join(
|
256
|
-
self.args.output_location,
|
257
|
-
PyFunceble.cli.storage.OUTPUTS.parent_directory,
|
258
|
-
)
|
259
|
-
)
|
257
|
+
try:
|
258
|
+
self.init_logger()
|
260
259
|
|
261
|
-
|
260
|
+
if hasattr(self.args, "output_location") and self.args.output_location:
|
261
|
+
PyFunceble.cli.storage.OUTPUT_DIRECTORY = os.path.realpath(
|
262
|
+
os.path.join(
|
263
|
+
self.args.output_location,
|
264
|
+
PyFunceble.cli.storage.OUTPUTS.parent_directory,
|
265
|
+
)
|
266
|
+
)
|
262
267
|
|
263
|
-
|
268
|
+
if hasattr(self.args, "config_dir") and self.args.config_dir:
|
269
|
+
PyFunceble.facility.ConfigLoader.set_config_dir(self.args.config_dir)
|
270
|
+
PyFunceble.storage.CONFIG_DIRECTORY = self.args.config_dir
|
271
|
+
|
272
|
+
if hasattr(self.args, "config_file") and self.args.config_file:
|
273
|
+
PyFunceble.facility.ConfigLoader.set_remote_config_location(
|
274
|
+
self.args.config_file
|
275
|
+
).reload()
|
276
|
+
|
277
|
+
PyFunceble.facility.Logger.debug("Given arguments:\n%r", self.args)
|
278
|
+
|
279
|
+
self.inject_into_config()
|
280
|
+
self.check_config()
|
281
|
+
self.check_deprecated()
|
282
|
+
|
283
|
+
PyFunceble.cli.facility.CredentialLoader.start()
|
284
|
+
PyFunceble.cli.factory.DBSession.init_db_sessions()
|
285
|
+
except (KeyboardInterrupt, StopExecution):
|
286
|
+
pass
|
287
|
+
except Exception as exception: # pylint: disable=broad-except
|
288
|
+
PyFunceble.facility.Logger.critical(
|
289
|
+
"Fatal error.",
|
290
|
+
exc_info=True,
|
291
|
+
)
|
292
|
+
if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload):
|
293
|
+
message = (
|
294
|
+
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download "
|
295
|
+
f"{exception}"
|
296
|
+
)
|
297
|
+
else:
|
298
|
+
message = (
|
299
|
+
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: "
|
300
|
+
f"{exception}"
|
301
|
+
)
|
302
|
+
print(message)
|
264
303
|
|
265
|
-
|
266
|
-
|
267
|
-
self.check_deprecated()
|
304
|
+
if PyFunceble.facility.Logger.authorized:
|
305
|
+
print(traceback.format_exc())
|
268
306
|
|
269
|
-
|
270
|
-
PyFunceble.cli.factory.DBSession.init_db_sessions()
|
307
|
+
sys.exit(1)
|
271
308
|
|
272
309
|
return self
|
@@ -74,6 +74,7 @@ import PyFunceble.cli.utils.ascii_logo
|
|
74
74
|
import PyFunceble.cli.utils.sort
|
75
75
|
import PyFunceble.cli.utils.stdout
|
76
76
|
import PyFunceble.facility
|
77
|
+
import PyFunceble.helpers.exceptions
|
77
78
|
import PyFunceble.storage
|
78
79
|
from PyFunceble.checker.syntax.url import URLSyntaxChecker
|
79
80
|
from PyFunceble.cli.continuous_integration.base import ContinuousIntegrationBase
|
@@ -1138,12 +1139,20 @@ class SystemLauncher(SystemBase):
|
|
1138
1139
|
"Fatal error.",
|
1139
1140
|
exc_info=True,
|
1140
1141
|
)
|
1141
|
-
|
1142
|
-
|
1143
|
-
|
1144
|
-
|
1142
|
+
if isinstance(exception, PyFunceble.helpers.exceptions.UnableToDownload):
|
1143
|
+
message = (
|
1144
|
+
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Unable to download "
|
1145
|
+
f"{exception}"
|
1146
|
+
)
|
1147
|
+
else:
|
1148
|
+
message = (
|
1149
|
+
f"{colorama.Fore.RED}{colorama.Style.BRIGHT}Fatal Error: "
|
1150
|
+
f"{exception}"
|
1151
|
+
)
|
1152
|
+
print(message)
|
1145
1153
|
|
1146
|
-
|
1154
|
+
if PyFunceble.facility.Logger.authorized:
|
1155
|
+
print(traceback.format_exc())
|
1147
1156
|
sys.exit(1)
|
1148
1157
|
|
1149
1158
|
PyFunceble.cli.utils.stdout.print_thanks()
|