atomicshop 2.16.25__py3-none-any.whl → 2.16.27__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/file_io/csvs.py +20 -0
- atomicshop/mitm/config_static.py +2 -0
- atomicshop/mitm/initialize_engines.py +9 -9
- atomicshop/mitm/mitm_main.py +57 -42
- atomicshop/mitm/recs_files.py +2 -1
- atomicshop/mitm/shared_functions.py +4 -1
- atomicshop/on_exit.py +38 -9
- atomicshop/print_api.py +34 -10
- atomicshop/wrappers/loggingw/loggingw.py +94 -0
- atomicshop/wrappers/socketw/dns_server.py +150 -59
- atomicshop/wrappers/socketw/socket_wrapper.py +12 -0
- atomicshop/wrappers/socketw/statistics_csv.py +3 -5
- {atomicshop-2.16.25.dist-info → atomicshop-2.16.27.dist-info}/METADATA +1 -1
- {atomicshop-2.16.25.dist-info → atomicshop-2.16.27.dist-info}/RECORD +18 -18
- {atomicshop-2.16.25.dist-info → atomicshop-2.16.27.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.16.25.dist-info → atomicshop-2.16.27.dist-info}/WHEEL +0 -0
- {atomicshop-2.16.25.dist-info → atomicshop-2.16.27.dist-info}/top_level.txt +0 -0
atomicshop/__init__.py
CHANGED
atomicshop/file_io/csvs.py
CHANGED
|
@@ -254,3 +254,23 @@ def escape_csv_line_to_list(csv_line: list) -> list:
|
|
|
254
254
|
result_csv_entries.append(escape_csv_value(entry))
|
|
255
255
|
|
|
256
256
|
return result_csv_entries
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def get_number_of_cells_in_string_line(line: str) -> int:
|
|
260
|
+
"""
|
|
261
|
+
Function to get number of cells in CSV line.
|
|
262
|
+
|
|
263
|
+
:param line: String, line of CSV file.
|
|
264
|
+
:return: int, number of cells in the line.
|
|
265
|
+
"""
|
|
266
|
+
|
|
267
|
+
# Create CSV reader from 'input_file'. By default, the first row will be the header if 'fieldnames' is None.
|
|
268
|
+
csv_reader = csv.reader([line])
|
|
269
|
+
|
|
270
|
+
# Get the first row of the CSV file.
|
|
271
|
+
csv_list = list(csv_reader)
|
|
272
|
+
|
|
273
|
+
# Get the number of cells in the first row.
|
|
274
|
+
number_of_cells = len(csv_list[0])
|
|
275
|
+
|
|
276
|
+
return number_of_cells
|
atomicshop/mitm/config_static.py
CHANGED
|
@@ -63,15 +63,15 @@ class ModuleCategory:
|
|
|
63
63
|
|
|
64
64
|
def initialize_engine(self, logs_path: str, logger=None, reference_general: bool = False, **kwargs):
|
|
65
65
|
# Initiating logger for each engine by its name
|
|
66
|
-
loggingw.create_logger(
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
)
|
|
66
|
+
# loggingw.create_logger(
|
|
67
|
+
# logger_name=self.engine_name,
|
|
68
|
+
# directory_path=logs_path,
|
|
69
|
+
# add_stream=True,
|
|
70
|
+
# add_timedfile=True,
|
|
71
|
+
# formatter_streamhandler='DEFAULT',
|
|
72
|
+
# formatter_filehandler='DEFAULT',
|
|
73
|
+
# backupCount=config_static.LogRec.store_logs_for_x_days
|
|
74
|
+
# )
|
|
75
75
|
|
|
76
76
|
if not reference_general:
|
|
77
77
|
self.parser_class_object = import_first_class_name_from_file_path(
|
atomicshop/mitm/mitm_main.py
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import
|
|
1
|
+
import logging
|
|
2
2
|
import threading
|
|
3
|
+
import multiprocessing
|
|
3
4
|
import time
|
|
4
5
|
import datetime
|
|
5
6
|
|
|
@@ -19,6 +20,14 @@ from . import config_static, recs_files
|
|
|
19
20
|
|
|
20
21
|
NETWORK_INTERFACE_IS_DYNAMIC: bool = bool()
|
|
21
22
|
NETWORK_INTERFACE_IPV4_ADDRESS_LIST: list[str] = list()
|
|
23
|
+
# noinspection PyTypeChecker
|
|
24
|
+
RECS_PROCESS_INSTANCE: multiprocessing.Process = None
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
EXCEPTIONS_CSV_LOGGER_NAME: str = 'exceptions'
|
|
28
|
+
EXCEPTIONS_CSV_LOGGER_HEADER: str = 'time,exception'
|
|
29
|
+
# noinspection PyTypeChecker
|
|
30
|
+
MITM_ERROR_LOGGER: loggingw.ExceptionCsvLogger = None
|
|
22
31
|
|
|
23
32
|
|
|
24
33
|
def exit_cleanup():
|
|
@@ -31,9 +40,13 @@ def exit_cleanup():
|
|
|
31
40
|
dns.set_connection_dns_gateway_dynamic(use_default_connection=True)
|
|
32
41
|
print_api("Returned default DNS gateway...", color='blue')
|
|
33
42
|
|
|
43
|
+
print_api(RECS_PROCESS_INSTANCE.is_alive())
|
|
44
|
+
RECS_PROCESS_INSTANCE.terminate()
|
|
45
|
+
RECS_PROCESS_INSTANCE.join()
|
|
34
46
|
|
|
35
|
-
|
|
36
|
-
|
|
47
|
+
|
|
48
|
+
def mitm_server(config_file_path: str):
|
|
49
|
+
on_exit.register_exit_handler(exit_cleanup, at_exit=False)
|
|
37
50
|
|
|
38
51
|
# Main function should return integer with error code, 0 is successful.
|
|
39
52
|
# Since listening server is infinite, this will not be reached.
|
|
@@ -45,23 +58,26 @@ def mitm_server_main(config_file_path: str):
|
|
|
45
58
|
if result != 0:
|
|
46
59
|
return result
|
|
47
60
|
|
|
61
|
+
global MITM_ERROR_LOGGER
|
|
62
|
+
MITM_ERROR_LOGGER = loggingw.ExceptionCsvLogger(EXCEPTIONS_CSV_LOGGER_NAME, config_static.LogRec.logs_path)
|
|
63
|
+
|
|
48
64
|
# Create folders.
|
|
49
65
|
filesystem.create_directory(config_static.LogRec.logs_path)
|
|
50
66
|
|
|
51
67
|
if config_static.LogRec.enable_request_response_recordings_in_logs:
|
|
52
68
|
filesystem.create_directory(config_static.LogRec.recordings_path)
|
|
53
69
|
# Compress recordings of the previous days if there are any.
|
|
54
|
-
|
|
70
|
+
global RECS_PROCESS_INSTANCE
|
|
71
|
+
RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(config_static.LogRec.recordings_path)
|
|
55
72
|
|
|
56
73
|
if config_static.Certificates.sni_get_server_certificate_from_server_socket:
|
|
57
74
|
filesystem.create_directory(
|
|
58
75
|
config_static.Certificates.sni_server_certificate_from_server_socket_download_directory)
|
|
59
76
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
file_path=f"{config_static.LogRec.logs_path}{os.sep}{logger_name}.txt",
|
|
77
|
+
network_logger_name = config_static.MainConfig.LOGGER_NAME
|
|
78
|
+
network_logger = loggingw.create_logger(
|
|
79
|
+
logger_name=network_logger_name,
|
|
80
|
+
directory_path=config_static.LogRec.logs_path,
|
|
65
81
|
add_stream=True,
|
|
66
82
|
add_timedfile=True,
|
|
67
83
|
formatter_streamhandler='DEFAULT',
|
|
@@ -69,6 +85,10 @@ def mitm_server_main(config_file_path: str):
|
|
|
69
85
|
backupCount=config_static.LogRec.store_logs_for_x_days
|
|
70
86
|
)
|
|
71
87
|
|
|
88
|
+
# Initiate Listener logger, which is a child of network logger, so he uses the same settings and handlers
|
|
89
|
+
listener_logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
|
|
90
|
+
system_logger = loggingw.get_logger_with_level(f'{network_logger_name}.system')
|
|
91
|
+
|
|
72
92
|
# Writing first log.
|
|
73
93
|
system_logger.info("======================================")
|
|
74
94
|
system_logger.info("Server Started.")
|
|
@@ -202,22 +222,6 @@ def mitm_server_main(config_file_path: str):
|
|
|
202
222
|
# Assigning all the engines domains to all time domains, that will be responsible for adding new domains.
|
|
203
223
|
config_static.Certificates.domains_all_times = list(domains_engine_list_full)
|
|
204
224
|
|
|
205
|
-
network_logger_name = "network"
|
|
206
|
-
network_logger = loggingw.create_logger(
|
|
207
|
-
logger_name=network_logger_name,
|
|
208
|
-
directory_path=config_static.LogRec.logs_path,
|
|
209
|
-
add_stream=True,
|
|
210
|
-
add_timedfile=True,
|
|
211
|
-
formatter_streamhandler='DEFAULT',
|
|
212
|
-
formatter_filehandler='DEFAULT',
|
|
213
|
-
backupCount=config_static.LogRec.store_logs_for_x_days
|
|
214
|
-
)
|
|
215
|
-
system_logger.info(f"Loaded network logger: {network_logger}")
|
|
216
|
-
|
|
217
|
-
# Initiate Listener logger, which is a child of network logger, so he uses the same settings and handlers
|
|
218
|
-
listener_logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
|
|
219
|
-
system_logger.info(f"Loaded listener logger: {listener_logger}")
|
|
220
|
-
|
|
221
225
|
print_api("Press [Ctrl]+[C] to stop.", color='blue')
|
|
222
226
|
|
|
223
227
|
# Create request domain queue.
|
|
@@ -330,7 +334,7 @@ def mitm_server_main(config_file_path: str):
|
|
|
330
334
|
global NETWORK_INTERFACE_IS_DYNAMIC, NETWORK_INTERFACE_IPV4_ADDRESS_LIST
|
|
331
335
|
NETWORK_INTERFACE_IS_DYNAMIC, NETWORK_INTERFACE_IPV4_ADDRESS_LIST = dns.get_default_dns_gateway()
|
|
332
336
|
if set_dns_gateway:
|
|
333
|
-
# Set the DNS gateway to the specified one only if the DNS gateway is dynamic or it is static but different
|
|
337
|
+
# Set the DNS gateway to the specified one only if the DNS gateway is dynamic, or it is static but different
|
|
334
338
|
# from the one specified in the configuration file.
|
|
335
339
|
if (NETWORK_INTERFACE_IS_DYNAMIC or (not NETWORK_INTERFACE_IS_DYNAMIC and
|
|
336
340
|
NETWORK_INTERFACE_IPV4_ADDRESS_LIST != dns_gateway_server_list)):
|
|
@@ -339,22 +343,16 @@ def mitm_server_main(config_file_path: str):
|
|
|
339
343
|
use_default_connection=True
|
|
340
344
|
)
|
|
341
345
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
'reference_function_args': (network_logger, statistics_writer, engines_list, reference_module,)
|
|
350
|
-
}
|
|
351
|
-
)
|
|
346
|
+
socket_thread = threading.Thread(
|
|
347
|
+
target=socket_wrapper_instance.loop_for_incoming_sockets,
|
|
348
|
+
kwargs={
|
|
349
|
+
'reference_function_name': thread_worker_main,
|
|
350
|
+
'reference_function_args': (network_logger, statistics_writer, engines_list, reference_module,)
|
|
351
|
+
}
|
|
352
|
+
)
|
|
352
353
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
except Exception:
|
|
356
|
-
message = f"Unhandled Exception occurred in 'loop_for_incoming_sockets' function"
|
|
357
|
-
print_api(message, error_type=True, color="red", logger=network_logger, traceback_string=True)
|
|
354
|
+
socket_thread.daemon = True
|
|
355
|
+
socket_thread.start()
|
|
358
356
|
|
|
359
357
|
# Compress recordings each day in a separate process.
|
|
360
358
|
recs_archiver_thread = threading.Thread(target=_loop_at_midnight_recs_archive)
|
|
@@ -375,8 +373,25 @@ def _loop_at_midnight_recs_archive():
|
|
|
375
373
|
# If it's midnight, start the archiving process.
|
|
376
374
|
if current_date != previous_date:
|
|
377
375
|
if config_static.LogRec.enable_request_response_recordings_in_logs:
|
|
378
|
-
|
|
376
|
+
global RECS_PROCESS_INSTANCE
|
|
377
|
+
RECS_PROCESS_INSTANCE = recs_files.recs_archiver_in_process(config_static.LogRec.recordings_path)
|
|
379
378
|
# Update the previous date.
|
|
380
379
|
previous_date = current_date
|
|
381
380
|
# Sleep for 1 minute.
|
|
382
381
|
time.sleep(60)
|
|
382
|
+
|
|
383
|
+
|
|
384
|
+
def mitm_server_main(config_file_path: str):
|
|
385
|
+
try:
|
|
386
|
+
# Main function should return integer with error code, 0 is successful.
|
|
387
|
+
return mitm_server(config_file_path)
|
|
388
|
+
except KeyboardInterrupt:
|
|
389
|
+
print_api("Server Stopped by [KeyboardInterrupt].", color='blue')
|
|
390
|
+
exit_cleanup()
|
|
391
|
+
return 0
|
|
392
|
+
except Exception as e:
|
|
393
|
+
RECS_PROCESS_INSTANCE.terminate()
|
|
394
|
+
|
|
395
|
+
MITM_ERROR_LOGGER.write(e)
|
|
396
|
+
exit_cleanup()
|
|
397
|
+
return 1
|
atomicshop/mitm/recs_files.py
CHANGED
|
@@ -64,10 +64,11 @@ def recs_archiver(recs_directory: str) -> list:
|
|
|
64
64
|
return archived_files
|
|
65
65
|
|
|
66
66
|
|
|
67
|
-
def recs_archiver_in_process(recs_directory: str):
|
|
67
|
+
def recs_archiver_in_process(recs_directory: str) -> multiprocessing.Process:
|
|
68
68
|
"""
|
|
69
69
|
Archive recs files in a directory for each day in a separate process.
|
|
70
70
|
"""
|
|
71
71
|
|
|
72
72
|
process = multiprocessing.Process(target=recs_archiver, args=(recs_directory,))
|
|
73
73
|
process.start()
|
|
74
|
+
return process
|
|
@@ -4,6 +4,8 @@ import inspect
|
|
|
4
4
|
from ..wrappers.loggingw import loggingw
|
|
5
5
|
from ..basics import dicts
|
|
6
6
|
|
|
7
|
+
from . import config_static
|
|
8
|
+
|
|
7
9
|
|
|
8
10
|
# If the string has several dot characters (".") - return the most right string after the last dot.
|
|
9
11
|
# Meaning if the class name contains several child classes like: classes.parsers.parser_something,
|
|
@@ -32,7 +34,8 @@ def create_custom_logger():
|
|
|
32
34
|
# 'f_globals' is a dictionary of all the global variables of the calling initiated class.
|
|
33
35
|
class_name = calling_frame.f_globals['__name__']
|
|
34
36
|
# Get the logger name only.
|
|
35
|
-
|
|
37
|
+
engine_logger_part = build_module_names(class_name)[0]
|
|
38
|
+
logger_name = f'{config_static.MainConfig.LOGGER_NAME}.{engine_logger_part}'
|
|
36
39
|
|
|
37
40
|
return loggingw.get_logger_with_level(logger_name)
|
|
38
41
|
|
atomicshop/on_exit.py
CHANGED
|
@@ -91,25 +91,54 @@ class ExitHandler:
|
|
|
91
91
|
# Exit the process gracefully
|
|
92
92
|
raise SystemExit(0)
|
|
93
93
|
|
|
94
|
-
def register_handlers(
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
94
|
+
def register_handlers(
|
|
95
|
+
self,
|
|
96
|
+
at_exit: bool = True,
|
|
97
|
+
console_close: bool = True,
|
|
98
|
+
kill_signal: bool = True
|
|
99
|
+
):
|
|
100
|
+
"""
|
|
101
|
+
Register the exit handlers.
|
|
102
|
+
|
|
103
|
+
:param at_exit: Register the atexit handler.
|
|
104
|
+
Just remember that the atexit handler will be called right away on [Ctrl+C], meaning if you want to do
|
|
105
|
+
something specifically on KeyboardInterrupt, you should handle it separately and set this parameter to False.
|
|
106
|
+
Same goes for all the exceptions.
|
|
107
|
+
:param console_close: Register the console close handler.
|
|
108
|
+
:param kill_signal: Register the kill signal handler.
|
|
109
|
+
"""
|
|
110
|
+
if at_exit:
|
|
111
|
+
atexit.register(self.atexit_handler)
|
|
112
|
+
if console_close:
|
|
113
|
+
win32api.SetConsoleCtrlHandler(self.console_handler, True)
|
|
114
|
+
if kill_signal:
|
|
115
|
+
signal.signal(signal.SIGINT, self.signal_handler)
|
|
116
|
+
signal.signal(signal.SIGTERM, self.signal_handler)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def register_exit_handler(
|
|
120
|
+
clean_up_function,
|
|
121
|
+
at_exit: bool = True,
|
|
122
|
+
console_close: bool = True,
|
|
123
|
+
kill_signal: bool = True,
|
|
124
|
+
*args, **kwargs):
|
|
102
125
|
"""
|
|
103
126
|
This function will register the exit handler to handle exit events: Closing the console, pressing 'CTRL+C',
|
|
104
127
|
Killing the process.
|
|
105
128
|
|
|
106
129
|
:param clean_up_function: The action to run when one of exit types is triggered.
|
|
130
|
+
:param at_exit: Register the atexit handler.
|
|
131
|
+
Just remember that the atexit handler will be called right away on [Ctrl+C], meaning if you want to do something
|
|
132
|
+
specifically on KeyboardInterrupt, you should handle it separately and set this parameter to False.
|
|
133
|
+
Same goes for all the exceptions.
|
|
134
|
+
:param console_close: Register the console close handler.
|
|
135
|
+
:param kill_signal: Register the kill signal handler.
|
|
107
136
|
:param args: The arguments to pass to the cleanup action.
|
|
108
137
|
:param kwargs: The keyword arguments to pass to the cleanup action.
|
|
109
138
|
"""
|
|
110
139
|
global EXIT_HANDLER_INSTANCE
|
|
111
140
|
EXIT_HANDLER_INSTANCE = ExitHandler(clean_up_function, args, kwargs)
|
|
112
|
-
EXIT_HANDLER_INSTANCE.register_handlers()
|
|
141
|
+
EXIT_HANDLER_INSTANCE.register_handlers(at_exit=at_exit, console_close=console_close, kill_signal=kill_signal)
|
|
113
142
|
|
|
114
143
|
|
|
115
144
|
def restart_function(callable_function, *args, **kwargs):
|
atomicshop/print_api.py
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import sys
|
|
2
|
+
import logging
|
|
2
3
|
|
|
3
4
|
from .basics.ansi_escape_codes import ColorsBasic, get_colors_basic_dict
|
|
4
5
|
from .basics import tracebacks
|
|
@@ -94,6 +95,20 @@ def print_api(
|
|
|
94
95
|
from bidi.algorithm import get_display
|
|
95
96
|
message = get_display(message)
|
|
96
97
|
|
|
98
|
+
if logger_method == 'error' or logger_method == 'critical':
|
|
99
|
+
error_type = True
|
|
100
|
+
|
|
101
|
+
# If exception was raised and 'stderr=True'.
|
|
102
|
+
if sys.exc_info()[0] is not None and stderr and traceback_string:
|
|
103
|
+
# If 'traceback' is set to 'True', we'll output traceback of exception.
|
|
104
|
+
if traceback_string:
|
|
105
|
+
if message:
|
|
106
|
+
message = f'{message}\n{tracebacks.get_as_string()}{message}'
|
|
107
|
+
else:
|
|
108
|
+
message = tracebacks.get_as_string()
|
|
109
|
+
|
|
110
|
+
color = 'red'
|
|
111
|
+
|
|
97
112
|
# If 'stdcolor' is 'True', the console output will be colored.
|
|
98
113
|
if stdcolor:
|
|
99
114
|
# If 'logger.error' should be outputted to console, and 'color' wasn't selected, then set color to 'yellow'.
|
|
@@ -103,19 +118,22 @@ def print_api(
|
|
|
103
118
|
elif logger_method == 'critical' and not color:
|
|
104
119
|
color = 'red'
|
|
105
120
|
|
|
106
|
-
if color:
|
|
121
|
+
if color and not logger:
|
|
107
122
|
message = get_colors_basic_dict(color) + message + ColorsBasic.END
|
|
123
|
+
elif color and logger:
|
|
124
|
+
# Save the original formatter
|
|
125
|
+
original_formatter = None
|
|
108
126
|
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
# If 'traceback' is set to 'True', we'll output traceback of exception.
|
|
115
|
-
if traceback_string:
|
|
116
|
-
message = f'{message} | Exception: {tracebacks.get_as_string()}'
|
|
127
|
+
# Find the stream handler and change its formatter
|
|
128
|
+
for handler in logger.handlers:
|
|
129
|
+
if isinstance(handler, logging.StreamHandler):
|
|
130
|
+
# Save the original formatter
|
|
131
|
+
original_formatter = handler.formatter
|
|
117
132
|
|
|
118
|
-
|
|
133
|
+
# Create a colored formatter for errors
|
|
134
|
+
color_formatter = logging.Formatter(
|
|
135
|
+
get_colors_basic_dict(color) + original_formatter + ColorsBasic.END)
|
|
136
|
+
handler.setFormatter(color_formatter)
|
|
119
137
|
|
|
120
138
|
# If 'online' is set to 'True', we'll output message as oneline.
|
|
121
139
|
if oneline:
|
|
@@ -130,6 +148,12 @@ def print_api(
|
|
|
130
148
|
if print_end == '\n':
|
|
131
149
|
# Use logger to output message.
|
|
132
150
|
getattr(logger, logger_method)(message)
|
|
151
|
+
|
|
152
|
+
if stdcolor:
|
|
153
|
+
# Restore the original formatter after logging
|
|
154
|
+
for handler in logger.handlers:
|
|
155
|
+
if isinstance(handler, logging.StreamHandler):
|
|
156
|
+
handler.setFormatter(original_formatter)
|
|
133
157
|
# If logger wasn't passed.
|
|
134
158
|
else:
|
|
135
159
|
# Use print to output the message.
|
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import logging
|
|
2
2
|
import os
|
|
3
3
|
from typing import Literal, Union
|
|
4
|
+
import datetime
|
|
4
5
|
|
|
5
6
|
from . import loggers, handlers
|
|
7
|
+
from ...file_io import csvs
|
|
8
|
+
from ...basics import tracebacks
|
|
9
|
+
from ...print_api import print_api
|
|
6
10
|
|
|
7
11
|
|
|
8
12
|
class LoggingwLoggerAlreadyExistsError(Exception):
|
|
@@ -258,3 +262,93 @@ def get_datetime_format_string_from_logger_file_handlers(logger: logging.Logger)
|
|
|
258
262
|
datetime_format_strings.append(date_time_format_string)
|
|
259
263
|
|
|
260
264
|
return datetime_format_strings
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def is_logger_exists(logger_name: str) -> bool:
|
|
268
|
+
"""
|
|
269
|
+
Function to check if the logger exists.
|
|
270
|
+
:param logger_name: Name of the logger.
|
|
271
|
+
:return: True if the logger exists, False if it doesn't.
|
|
272
|
+
"""
|
|
273
|
+
|
|
274
|
+
return loggers.is_logger_exists(logger_name)
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
class ExceptionCsvLogger:
|
|
278
|
+
def __init__(
|
|
279
|
+
self,
|
|
280
|
+
logger_name: str,
|
|
281
|
+
custom_header: str = None,
|
|
282
|
+
directory_path: str = None
|
|
283
|
+
):
|
|
284
|
+
"""
|
|
285
|
+
Initialize the ExceptionCsvLogger object.
|
|
286
|
+
|
|
287
|
+
:param logger_name: Name of the logger.
|
|
288
|
+
:param custom_header: Custom header to write to the log file.
|
|
289
|
+
If None, the default header will be used: "timestamp,exception", since that what is written to the log file.
|
|
290
|
+
If you want to add more columns to the csv file, you can provide a custom header:
|
|
291
|
+
"custom1,custom2,custom3".
|
|
292
|
+
These will be added to the default header as:
|
|
293
|
+
"timestamp,custom1,custom2,custom3,exception".
|
|
294
|
+
:param directory_path: Directory path where the log file will be saved.
|
|
295
|
+
You can leave it as None, but if the logger doesn't exist, you will get an exception.
|
|
296
|
+
"""
|
|
297
|
+
|
|
298
|
+
if custom_header:
|
|
299
|
+
self.header = f"timestamp,{custom_header},exception"
|
|
300
|
+
else:
|
|
301
|
+
self.header = "timestamp,exception"
|
|
302
|
+
|
|
303
|
+
if is_logger_exists(logger_name):
|
|
304
|
+
self.logger = get_logger_with_level(logger_name)
|
|
305
|
+
else:
|
|
306
|
+
if directory_path is None:
|
|
307
|
+
raise ValueError("You need to provide 'directory_path' if the logger doesn't exist.")
|
|
308
|
+
|
|
309
|
+
self.logger = create_logger(
|
|
310
|
+
logger_name=logger_name,
|
|
311
|
+
directory_path=directory_path,
|
|
312
|
+
file_type="csv",
|
|
313
|
+
add_timedfile=True,
|
|
314
|
+
formatter_filehandler='MESSAGE',
|
|
315
|
+
header=self.header)
|
|
316
|
+
|
|
317
|
+
def write(
|
|
318
|
+
self,
|
|
319
|
+
message: Union[str, Exception] = None,
|
|
320
|
+
custom_csv_string: str = None
|
|
321
|
+
):
|
|
322
|
+
"""
|
|
323
|
+
Write the message to the log file.
|
|
324
|
+
|
|
325
|
+
:param message: The message to write to the log file.
|
|
326
|
+
If None, the message will be retrieved from current traceback frame.
|
|
327
|
+
:param custom_csv_string: Custom CSV string to add between the timestamp and the exception.
|
|
328
|
+
Currently, without the 'custom_csv_string', the csv line written as "timestamp,exception" as the header.
|
|
329
|
+
If you add a 'custom_csv_string', the csv line will be written as "timestamp,custom_csv_string,exception".
|
|
330
|
+
Meaning, that you need to provide the 'custom_header' during the initialization of the object.
|
|
331
|
+
Off course, you can use as many commas as you need in the 'custom_csv_string': "custom1,custom2,custom3".
|
|
332
|
+
This need to be mirrored in the 'custom_header' as well: "custom1,custom2,custom3".
|
|
333
|
+
"""
|
|
334
|
+
|
|
335
|
+
if message is None or isinstance(message, Exception):
|
|
336
|
+
message = tracebacks.get_as_string()
|
|
337
|
+
|
|
338
|
+
if custom_csv_string:
|
|
339
|
+
output_csv_line: str = csvs.escape_csv_line_to_string([datetime.datetime.now(), custom_csv_string, message])
|
|
340
|
+
else:
|
|
341
|
+
output_csv_line: str = csvs.escape_csv_line_to_string([datetime.datetime.now(), message])
|
|
342
|
+
|
|
343
|
+
# If the number of cells in the 'output_csv_line' doesn't match the number of cells in the 'header',
|
|
344
|
+
# raise an exception.
|
|
345
|
+
if (csvs.get_number_of_cells_in_string_line(output_csv_line) !=
|
|
346
|
+
csvs.get_number_of_cells_in_string_line(self.header)):
|
|
347
|
+
raise ValueError(
|
|
348
|
+
"Number of cells in the 'output_csv_line' doesn't match the number of cells in the 'header'.")
|
|
349
|
+
|
|
350
|
+
self.logger.info(output_csv_line)
|
|
351
|
+
print_api('', error_type=True, color="red", traceback_string=True)
|
|
352
|
+
|
|
353
|
+
def get_logger(self):
|
|
354
|
+
return self.logger
|
|
@@ -5,12 +5,14 @@ import threading
|
|
|
5
5
|
import socket
|
|
6
6
|
import logging
|
|
7
7
|
from pathlib import Path
|
|
8
|
+
from typing import Literal
|
|
8
9
|
|
|
9
10
|
from ...print_api import print_api
|
|
10
11
|
from ..loggingw import loggingw
|
|
11
12
|
from ..psutilw import networks
|
|
12
13
|
from ... import queues
|
|
13
|
-
from ...basics import booleans
|
|
14
|
+
from ...basics import booleans, tracebacks
|
|
15
|
+
from ...file_io import csvs
|
|
14
16
|
|
|
15
17
|
# noinspection PyPackageRequirements
|
|
16
18
|
import dnslib
|
|
@@ -26,6 +28,115 @@ class DnsConfigurationValuesError(Exception):
|
|
|
26
28
|
pass
|
|
27
29
|
|
|
28
30
|
|
|
31
|
+
LOGGER_NAME: str = 'dns_traffic'
|
|
32
|
+
DNS_STATISTICS_HEADER: str = (
|
|
33
|
+
'timestamp,dns_type,client_ipv4,client_port,qname,qtype,qclass,header,error')
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class DnsStatisticsCSVWriter:
|
|
37
|
+
"""
|
|
38
|
+
Class to write statistics to CSV file.
|
|
39
|
+
This can be initiated at the main, and then passed to the thread worker function.
|
|
40
|
+
"""
|
|
41
|
+
def __init__(
|
|
42
|
+
self,
|
|
43
|
+
statistics_directory_path: str
|
|
44
|
+
):
|
|
45
|
+
self.csv_logger = loggingw.create_logger(
|
|
46
|
+
logger_name=LOGGER_NAME,
|
|
47
|
+
directory_path=statistics_directory_path,
|
|
48
|
+
add_timedfile=True,
|
|
49
|
+
formatter_filehandler='MESSAGE',
|
|
50
|
+
file_type='csv',
|
|
51
|
+
header=DNS_STATISTICS_HEADER
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
def write_row(
|
|
55
|
+
self,
|
|
56
|
+
client_address: tuple,
|
|
57
|
+
timestamp=None,
|
|
58
|
+
dns_type: Literal['request', 'response'] = None,
|
|
59
|
+
dns_request = None,
|
|
60
|
+
dns_response = None,
|
|
61
|
+
error: str = None,
|
|
62
|
+
):
|
|
63
|
+
if not timestamp:
|
|
64
|
+
timestamp = datetime.datetime.now()
|
|
65
|
+
|
|
66
|
+
if not dns_type:
|
|
67
|
+
if not dns_request and not dns_response:
|
|
68
|
+
raise ValueError("Either DNS Request or DNS Response must be provided.")
|
|
69
|
+
elif dns_request and dns_response:
|
|
70
|
+
raise ValueError("Either DNS Request or DNS Response must be provided. Not both.")
|
|
71
|
+
|
|
72
|
+
if dns_request:
|
|
73
|
+
dns_type = 'request'
|
|
74
|
+
elif dns_response:
|
|
75
|
+
dns_type = 'response'
|
|
76
|
+
|
|
77
|
+
if dns_type not in ['request', 'response']:
|
|
78
|
+
raise ValueError(f"DNS Type can be only 'request' or 'response'. Provided: {dns_type}")
|
|
79
|
+
|
|
80
|
+
client_ipv4, client_port = client_address
|
|
81
|
+
client_ipv4: str
|
|
82
|
+
client_port: str = str(client_port)
|
|
83
|
+
|
|
84
|
+
qname: str = str()
|
|
85
|
+
qtype: str = str()
|
|
86
|
+
qclass: str = str()
|
|
87
|
+
rr: str = str()
|
|
88
|
+
header: str = str()
|
|
89
|
+
|
|
90
|
+
if dns_request:
|
|
91
|
+
qname = str(dns_request.q.qname)[:-1]
|
|
92
|
+
qtype = dnslib.QTYPE[dns_request.q.qtype]
|
|
93
|
+
qclass = dnslib.CLASS[dns_request.q.qclass]
|
|
94
|
+
|
|
95
|
+
if dns_response:
|
|
96
|
+
qname = str(dns_response.q.qname)[:-1]
|
|
97
|
+
qtype = dnslib.QTYPE[dns_response.q.qtype]
|
|
98
|
+
qclass = dnslib.CLASS[dns_response.q.qclass]
|
|
99
|
+
rr: str = str(dns_response.rr)
|
|
100
|
+
header = str(dns_response.header)
|
|
101
|
+
|
|
102
|
+
escaped_line_string: str = csvs.escape_csv_line_to_string([
|
|
103
|
+
timestamp,
|
|
104
|
+
dns_type,
|
|
105
|
+
client_ipv4,
|
|
106
|
+
client_port,
|
|
107
|
+
qname,
|
|
108
|
+
qtype,
|
|
109
|
+
qclass,
|
|
110
|
+
rr,
|
|
111
|
+
header,
|
|
112
|
+
error
|
|
113
|
+
])
|
|
114
|
+
|
|
115
|
+
self.csv_logger.info(escaped_line_string)
|
|
116
|
+
|
|
117
|
+
def write_error(
|
|
118
|
+
self,
|
|
119
|
+
dns_type: Literal['request', 'response'],
|
|
120
|
+
error_message: str,
|
|
121
|
+
client_address: tuple
|
|
122
|
+
):
|
|
123
|
+
"""
|
|
124
|
+
Write the error message to the statistics CSV file.
|
|
125
|
+
This is used for easier execution, since most of the parameters will be empty on accept.
|
|
126
|
+
|
|
127
|
+
:param dns_type: Literal['request', 'response'], DNS request or response.
|
|
128
|
+
:param error_message: string, error message.
|
|
129
|
+
:param client_address: tuple, client address (IPv4, Port).
|
|
130
|
+
:return:
|
|
131
|
+
"""
|
|
132
|
+
|
|
133
|
+
self.write_row(
|
|
134
|
+
dns_type=dns_type,
|
|
135
|
+
client_address=client_address,
|
|
136
|
+
error=error_message
|
|
137
|
+
)
|
|
138
|
+
|
|
139
|
+
|
|
29
140
|
class DnsServer:
|
|
30
141
|
"""
|
|
31
142
|
DnsServer class is responsible to handle DNS Requests on port 53 based on configuration and send DNS Response back.
|
|
@@ -129,13 +240,7 @@ class DnsServer:
|
|
|
129
240
|
|
|
130
241
|
# Logger that logs all the DNS Requests and responses in DNS format. These entries will not present in
|
|
131
242
|
# network log of TCP Server module.
|
|
132
|
-
self.
|
|
133
|
-
logger_name="dns_full",
|
|
134
|
-
directory_path=self.log_directory_path,
|
|
135
|
-
add_timedfile=True,
|
|
136
|
-
formatter_filehandler='DEFAULT',
|
|
137
|
-
backupCount=backupCount_log_files_x_days
|
|
138
|
-
)
|
|
243
|
+
self.dns_statistics_csv_writer = DnsStatisticsCSVWriter(statistics_directory_path=log_directory_path)
|
|
139
244
|
|
|
140
245
|
# Check if the logger was provided, if not, create a new logger.
|
|
141
246
|
if not logger:
|
|
@@ -250,20 +355,19 @@ class DnsServer:
|
|
|
250
355
|
client_data: bytes
|
|
251
356
|
client_address: tuple
|
|
252
357
|
except ConnectionResetError:
|
|
358
|
+
traceback_string = tracebacks.get_as_string(one_line=True)
|
|
253
359
|
# This error happens when the client closes the connection before the server.
|
|
254
360
|
# This is not an error for a DNS Server, but we'll log it anyway only with the full DNS logger.
|
|
255
|
-
message = "Error: to receive DNS request, An existing connection was forcibly closed"
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
self.dns_full_logger.info("==========")
|
|
361
|
+
message = (f"Error: to receive DNS request, An existing connection was forcibly closed | "
|
|
362
|
+
f"{traceback_string}")
|
|
363
|
+
self.dns_statistics_csv_writer.write_error(
|
|
364
|
+
dns_type='request', client_address=client_address, error_message=message)
|
|
260
365
|
pass
|
|
261
366
|
continue
|
|
262
367
|
except Exception:
|
|
263
368
|
message = "Unknown Exception: to receive DNS request"
|
|
264
369
|
print_api(
|
|
265
370
|
message, logger=self.logger, logger_method='critical', traceback_string=True)
|
|
266
|
-
self.logger.info("==========")
|
|
267
371
|
pass
|
|
268
372
|
continue
|
|
269
373
|
|
|
@@ -297,26 +401,18 @@ class DnsServer:
|
|
|
297
401
|
# "dns_object.q.qname" returns only the questioned domain with "." (dot) in the end,
|
|
298
402
|
# which needs to be removed.
|
|
299
403
|
question_domain: str = str(dns_object.q.qname)[:-1]
|
|
300
|
-
self.
|
|
301
|
-
self.dns_full_logger.info(f"QTYPE: {qtype_string}")
|
|
302
|
-
self.dns_full_logger.info(f"Question Domain: {question_domain}")
|
|
404
|
+
self.dns_statistics_csv_writer.write_row(client_address=client_address, dns_request=dns_object)
|
|
303
405
|
|
|
304
406
|
message = (f"Received DNS request: {question_domain} | {qclass_string} | {qtype_string} | "
|
|
305
407
|
f"From: {client_address}.")
|
|
306
408
|
self.logger.info(message)
|
|
307
|
-
self.dns_full_logger.info(message)
|
|
308
|
-
|
|
309
|
-
self.dns_full_logger.info("--")
|
|
310
409
|
|
|
311
410
|
# Nullifying the DNS cache for current request before check.
|
|
312
411
|
dns_cached_request = False
|
|
313
412
|
# Check if the received data request from client is already in the cache
|
|
314
413
|
if client_data in self.dns_questions_to_answers_cache:
|
|
315
|
-
message = "!!! Question / Answer is already in the dictionary..."
|
|
414
|
+
# message = "!!! Question / Answer is already in the dictionary..."
|
|
316
415
|
# self.logger.info(message)
|
|
317
|
-
self.dns_full_logger.info(message)
|
|
318
|
-
|
|
319
|
-
self.dns_full_logger.info("--")
|
|
320
416
|
|
|
321
417
|
# Get the response from the cached answers list
|
|
322
418
|
dns_response = self.dns_questions_to_answers_cache[client_data]
|
|
@@ -395,7 +491,6 @@ class DnsServer:
|
|
|
395
491
|
message = f"!!! Question / Answer is in offline mode returning " \
|
|
396
492
|
f"{self.offline_route_ipv6}."
|
|
397
493
|
self.logger.info(message)
|
|
398
|
-
self.dns_full_logger.info(message)
|
|
399
494
|
|
|
400
495
|
# SRV Record type explanation:
|
|
401
496
|
# https://www.cloudflare.com/learning/dns/dns-records/dns-srv-record/
|
|
@@ -412,7 +507,6 @@ class DnsServer:
|
|
|
412
507
|
message = f"!!! Question / Answer is in offline mode returning: " \
|
|
413
508
|
f"{self.offline_srv_answer}."
|
|
414
509
|
self.logger.info(message)
|
|
415
|
-
self.dns_full_logger.info(message)
|
|
416
510
|
elif qtype_string == "ANY":
|
|
417
511
|
dns_built_response.add_answer(
|
|
418
512
|
*RR.fromZone(question_domain + " " + str(self.response_ttl) + " CNAME " +
|
|
@@ -422,7 +516,6 @@ class DnsServer:
|
|
|
422
516
|
message = f"!!! Question / Answer is in offline mode returning " \
|
|
423
517
|
f"{self.offline_route_domain}."
|
|
424
518
|
self.logger.info(message)
|
|
425
|
-
self.dns_full_logger.info(message)
|
|
426
519
|
else:
|
|
427
520
|
dns_built_response.add_answer(
|
|
428
521
|
*RR.fromZone(
|
|
@@ -433,7 +526,6 @@ class DnsServer:
|
|
|
433
526
|
message = f"!!! Question / Answer is in offline mode returning " \
|
|
434
527
|
f"{self.offline_route_ipv4}."
|
|
435
528
|
self.logger.info(message)
|
|
436
|
-
self.dns_full_logger.info(message)
|
|
437
529
|
# Values error means in most cases that you create wrong response
|
|
438
530
|
# for specific type of request.
|
|
439
531
|
except ValueError:
|
|
@@ -450,18 +542,14 @@ class DnsServer:
|
|
|
450
542
|
except Exception:
|
|
451
543
|
message = \
|
|
452
544
|
(f"Unknown exception while creating response for QTYPE: {qtype_string}. "
|
|
453
|
-
f"Response: ")
|
|
545
|
+
f"Response: \n{dns_built_response}")
|
|
454
546
|
print_api(message, logger=self.logger, logger_method='critical',
|
|
455
547
|
traceback_string=True)
|
|
456
|
-
print_api(f"{dns_built_response}", logger=self.logger, logger_method='critical',
|
|
457
|
-
traceback_string=True)
|
|
458
548
|
# Pass the exception.
|
|
459
549
|
pass
|
|
460
550
|
# Continue to the next DNS request, since there's nothing to do here right now.
|
|
461
551
|
continue
|
|
462
552
|
|
|
463
|
-
self.dns_full_logger.info("--")
|
|
464
|
-
|
|
465
553
|
# Encode the response that was built above to legit DNS Response
|
|
466
554
|
dns_response = dns_built_response.pack()
|
|
467
555
|
# If we're in online mode
|
|
@@ -476,7 +564,7 @@ class DnsServer:
|
|
|
476
564
|
# Since, it's probably going to succeed.
|
|
477
565
|
if counter > 0:
|
|
478
566
|
self.logger.info(f"Retry #: {counter}/{self.dns_service_retries}")
|
|
479
|
-
self.
|
|
567
|
+
self.logger.info(
|
|
480
568
|
f"Forwarding request. Creating UDP socket to: "
|
|
481
569
|
f"{self.forwarding_dns_service_ipv4}:"
|
|
482
570
|
f"{self.forwarding_dns_service_port}")
|
|
@@ -486,7 +574,7 @@ class DnsServer:
|
|
|
486
574
|
|
|
487
575
|
message = "Socket created, Forwarding..."
|
|
488
576
|
# self.logger.info(message)
|
|
489
|
-
self.
|
|
577
|
+
self.logger.info(message)
|
|
490
578
|
|
|
491
579
|
google_dns_ipv4_socket.sendto(client_data, (
|
|
492
580
|
self.forwarding_dns_service_ipv4,
|
|
@@ -495,7 +583,7 @@ class DnsServer:
|
|
|
495
583
|
# The script needs to wait a second or receive can hang
|
|
496
584
|
message = "Request sent to the forwarding DNS, Receiving the answer..."
|
|
497
585
|
# self.logger.info(message)
|
|
498
|
-
self.
|
|
586
|
+
self.logger.info(message)
|
|
499
587
|
|
|
500
588
|
dns_response, google_address = \
|
|
501
589
|
google_dns_ipv4_socket.recvfrom(self.buffer_size_receive)
|
|
@@ -516,7 +604,6 @@ class DnsServer:
|
|
|
516
604
|
f"Couldn't forward DNS request to: "
|
|
517
605
|
f"[{self.forwarding_dns_service_ipv4}]. "
|
|
518
606
|
f"Continuing to next request.")
|
|
519
|
-
self.dns_full_logger.info("==========")
|
|
520
607
|
|
|
521
608
|
# From here continue to the next iteration of While loop.
|
|
522
609
|
continue
|
|
@@ -529,12 +616,12 @@ class DnsServer:
|
|
|
529
616
|
if retried:
|
|
530
617
|
continue
|
|
531
618
|
|
|
532
|
-
self.
|
|
619
|
+
self.logger.info(
|
|
533
620
|
f"Answer received from: {self.forwarding_dns_service_ipv4}")
|
|
534
621
|
|
|
535
622
|
# Closing the socket to forwarding service
|
|
536
623
|
google_dns_ipv4_socket.close()
|
|
537
|
-
self.
|
|
624
|
+
self.logger.info("Closed socket to forwarding service")
|
|
538
625
|
|
|
539
626
|
# Appending current DNS Request and DNS Answer to the Cache
|
|
540
627
|
self.dns_questions_to_answers_cache.update({client_data: dns_response})
|
|
@@ -542,14 +629,15 @@ class DnsServer:
|
|
|
542
629
|
# If 'forward_to_tcp_server' it means that we built the response, and we don't need to reparse it,
|
|
543
630
|
# since we already have all the data.
|
|
544
631
|
if forward_to_tcp_server:
|
|
545
|
-
self.
|
|
546
|
-
self.
|
|
632
|
+
# self.logger.info(f"Response {dns_built_response.short()}")
|
|
633
|
+
self.dns_statistics_csv_writer.write_row(
|
|
634
|
+
client_address=client_address, dns_response=dns_built_response)
|
|
547
635
|
|
|
548
636
|
message = f"Response Details: {dns_built_response.rr}"
|
|
549
|
-
print_api(message, logger=self.
|
|
637
|
+
print_api(message, logger=self.logger, logger_method='info', oneline=True)
|
|
550
638
|
|
|
551
|
-
message = f"Response Full Details: {dns_built_response.format(prefix='', sort=True)}"
|
|
552
|
-
print_api(message, logger=self.
|
|
639
|
+
# message = f"Response Full Details: {dns_built_response.format(prefix='', sort=True)}"
|
|
640
|
+
# print_api(message, logger=self.logger, logger_method='info', oneline=True)
|
|
553
641
|
|
|
554
642
|
# Now we can turn it to false, so it won't trigger this
|
|
555
643
|
# condition next time if the response was not built
|
|
@@ -569,20 +657,23 @@ class DnsServer:
|
|
|
569
657
|
if dns_response_parsed.rr:
|
|
570
658
|
for rr in dns_response_parsed.rr:
|
|
571
659
|
if isinstance(rr.rdata, A):
|
|
572
|
-
self.
|
|
660
|
+
self.dns_statistics_csv_writer.write_row(
|
|
661
|
+
client_address=client_address, dns_response=dns_response_parsed)
|
|
662
|
+
|
|
663
|
+
self.logger.info(f"Response IP: {rr.rdata}")
|
|
573
664
|
|
|
574
665
|
# Adding the address to the list as 'str' object and not 'dnslib.dns.A'.
|
|
575
666
|
ipv4_addresses.append(str(rr.rdata))
|
|
576
667
|
|
|
577
|
-
message = f"Response Details: {dns_response_parsed.rr}"
|
|
578
|
-
print_api(message, logger=self.
|
|
579
|
-
|
|
580
|
-
message = f"Response Full Details: {dns_response_parsed}"
|
|
581
|
-
print_api(message, logger=self.
|
|
668
|
+
# message = f"Response Details: {dns_response_parsed.rr}"
|
|
669
|
+
# print_api(message, logger=self.dns_statistics_csv_writer, logger_method='info', oneline=True)
|
|
670
|
+
#
|
|
671
|
+
# message = f"Response Full Details: {dns_response_parsed}"
|
|
672
|
+
# print_api(message, logger=self.dns_statistics_csv_writer, logger_method='info', oneline=True)
|
|
582
673
|
|
|
583
|
-
self.
|
|
674
|
+
self.logger.info("Sending DNS response back to client...")
|
|
584
675
|
main_socket_object.sendto(dns_response, client_address)
|
|
585
|
-
self.
|
|
676
|
+
self.logger.info("DNS Response sent...")
|
|
586
677
|
|
|
587
678
|
# 'ipv4_addresses' list contains entries of type 'dnslib.dns.A' and not string.
|
|
588
679
|
# We'll convert each entry to string, so strings can be searched in this list.
|
|
@@ -661,9 +752,9 @@ class DnsServer:
|
|
|
661
752
|
) as output_file:
|
|
662
753
|
output_file.write(record_string_line)
|
|
663
754
|
|
|
664
|
-
self.
|
|
665
|
-
|
|
666
|
-
|
|
755
|
+
# self.logger.info(
|
|
756
|
+
# f"Saved new known domains file: "
|
|
757
|
+
# f"{self.log_directory_path}{os.sep}{self.known_domains_filename}")
|
|
667
758
|
|
|
668
759
|
# Known domain list managements EOF
|
|
669
760
|
# ==================================================================================================
|
|
@@ -720,9 +811,9 @@ class DnsServer:
|
|
|
720
811
|
) as output_file:
|
|
721
812
|
output_file.write(record_string_line)
|
|
722
813
|
|
|
723
|
-
self.
|
|
724
|
-
|
|
725
|
-
|
|
814
|
+
# self.logger.info(
|
|
815
|
+
# f"Saved new known IPv4 addresses file: "
|
|
816
|
+
# f"{self.log_directory_path}{os.sep}{self.known_ipv4_filename}")
|
|
726
817
|
|
|
727
818
|
# Known IPv4 address to domains list management EOF
|
|
728
819
|
# ==================================================================================================
|
|
@@ -743,7 +834,7 @@ class DnsServer:
|
|
|
743
834
|
# EOF Writing IPs by time.
|
|
744
835
|
# ==================================================================================================
|
|
745
836
|
|
|
746
|
-
self.
|
|
837
|
+
# self.logger.info("==========")
|
|
747
838
|
except Exception:
|
|
748
839
|
message = "Unknown Exception: to parse DNS request"
|
|
749
840
|
print_api(
|
|
@@ -5,6 +5,7 @@ from pathlib import Path
|
|
|
5
5
|
|
|
6
6
|
from ..psutilw import networks
|
|
7
7
|
from ..certauthw import certauthw
|
|
8
|
+
from ..loggingw import loggingw
|
|
8
9
|
from ...script_as_string_processor import ScriptAsStringProcessor
|
|
9
10
|
from ...permissions import permissions
|
|
10
11
|
from ... import queues, filesystem, certificates
|
|
@@ -143,6 +144,7 @@ class SocketWrapper:
|
|
|
143
144
|
:param ssh_pass: string, SSH password that will be used to connect to remote host.
|
|
144
145
|
:param ssh_script_to_execute: string, script that will be executed to get the process name on ssh remote host.
|
|
145
146
|
:param logger: logging.Logger object, logger object that will be used to log messages.
|
|
147
|
+
If not provided, logger will be created with default settings.
|
|
146
148
|
:param statistics_logs_directory: string, path to directory where daily statistics.csv files will be stored.
|
|
147
149
|
After you initialize the SocketWrapper object, you can get the statistics_writer object from it and use it
|
|
148
150
|
to write statistics to the file in a worker thread.
|
|
@@ -219,6 +221,16 @@ class SocketWrapper:
|
|
|
219
221
|
self.statistics_writer = statistics_csv.StatisticsCSVWriter(
|
|
220
222
|
statistics_directory_path=self.statistics_logs_directory)
|
|
221
223
|
|
|
224
|
+
if not self.logger:
|
|
225
|
+
self.logger = loggingw.create_logger(
|
|
226
|
+
logger_name='SocketWrapper',
|
|
227
|
+
directory_path=self.statistics_logs_directory,
|
|
228
|
+
add_stream=True,
|
|
229
|
+
add_timedfile=True,
|
|
230
|
+
formatter_streamhandler='DEFAULT',
|
|
231
|
+
formatter_filehandler='DEFAULT'
|
|
232
|
+
)
|
|
233
|
+
|
|
222
234
|
self.test_config()
|
|
223
235
|
|
|
224
236
|
def test_config(self):
|
|
@@ -5,9 +5,9 @@ from ..loggingw import loggingw
|
|
|
5
5
|
|
|
6
6
|
|
|
7
7
|
LOGGER_NAME: str = 'statistics'
|
|
8
|
-
STATISTICS_HEADER: str =
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
STATISTICS_HEADER: str = (
|
|
9
|
+
'request_time_sent,tls,protocol,host,path,command,status_code,request_size_bytes,response_size_bytes,file_path,'
|
|
10
|
+
'process_cmd,error')
|
|
11
11
|
|
|
12
12
|
|
|
13
13
|
class StatisticsCSVWriter:
|
|
@@ -19,8 +19,6 @@ class StatisticsCSVWriter:
|
|
|
19
19
|
self,
|
|
20
20
|
statistics_directory_path: str
|
|
21
21
|
):
|
|
22
|
-
self.statistics_directory_path = statistics_directory_path
|
|
23
|
-
|
|
24
22
|
self.csv_logger = loggingw.create_logger(
|
|
25
23
|
logger_name=LOGGER_NAME,
|
|
26
24
|
directory_path=statistics_directory_path,
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=uvGTfzSHIYxELvSkSy25hxKozcYi2WpTk6bDs1_wXVs,124
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
|
|
4
4
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
@@ -23,9 +23,9 @@ atomicshop/http_parse.py,sha256=nrf2rZcprLqtW8HVrV7TCZ1iTBcWRRy-mXIlAOzcaJs,9703
|
|
|
23
23
|
atomicshop/inspect_wrapper.py,sha256=sGRVQhrJovNygHTydqJj0hxES-aB2Eg9KbIk3G31apw,11429
|
|
24
24
|
atomicshop/ip_addresses.py,sha256=Hvi4TumEFoTEpKWaq5WNF-YzcRzt24IxmNgv-Mgax1s,1190
|
|
25
25
|
atomicshop/keyboard_press.py,sha256=1W5kRtOB75fulVx-uF2yarBhW0_IzdI1k73AnvXstk0,452
|
|
26
|
-
atomicshop/on_exit.py,sha256=
|
|
26
|
+
atomicshop/on_exit.py,sha256=Rpg2SaF0aginuO7JYwA49YJYnS8F6K2jUqhjH65WzuU,6889
|
|
27
27
|
atomicshop/pbtkmultifile_argparse.py,sha256=aEk8nhvoQVu-xyfZosK3ma17CwIgOjzO1erXXdjwtS4,4574
|
|
28
|
-
atomicshop/print_api.py,sha256=
|
|
28
|
+
atomicshop/print_api.py,sha256=37zugPqm04XFLmMnXxNo0bJtv1ImNSDHosPTlOuYyhw,12760
|
|
29
29
|
atomicshop/process.py,sha256=PeLvyixXaCfftdUF3oMbohI1L4MdLtvQVDx2V1Tz_Rk,16662
|
|
30
30
|
atomicshop/python_file_patcher.py,sha256=-uhbUX-um5k-If_XXuOfCr8wMzZ3QE6h9N8xGWw6W_o,5486
|
|
31
31
|
atomicshop/python_functions.py,sha256=zJg4ogUwECxrDD7xdDN5JikIUctITM5lsyabr_ZNsRw,4435
|
|
@@ -114,7 +114,7 @@ atomicshop/etws/traces/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3
|
|
|
114
114
|
atomicshop/etws/traces/trace_dns.py,sha256=WvOZm7KNdP4r6ofkZhUGi9WjtYlkV3mUp_yxita3Qg4,6399
|
|
115
115
|
atomicshop/etws/traces/trace_sysmon_process_creation.py,sha256=OM-bkK38uYMwWLZKNOTDa0Xdk3sO6sqsxoMUIiPvm5g,4656
|
|
116
116
|
atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
117
|
-
atomicshop/file_io/csvs.py,sha256=
|
|
117
|
+
atomicshop/file_io/csvs.py,sha256=XFZ-kvg2nWLXUpFh_FyPwOXB_TYLFRrHpS5hWMblTi8,9353
|
|
118
118
|
atomicshop/file_io/docxs.py,sha256=ffJhnmM_WyD8mCoq2dGdpfahdIrGTPy96QVlH5EWjeI,5754
|
|
119
119
|
atomicshop/file_io/file_io.py,sha256=IVqs0Kd7PV8O-NH9321hK_IvPqWx2z_45zdDtqgw4II,7113
|
|
120
120
|
atomicshop/file_io/jsons.py,sha256=q9ZU8slBKnHLrtn3TnbK1qxrRpj5ZvCm6AlsFzoANjo,5303
|
|
@@ -122,15 +122,15 @@ atomicshop/file_io/tomls.py,sha256=ol8EvQPf9sryTmZUf1v55BYSUQ6ml7HVVBHpNKbsIlA,9
|
|
|
122
122
|
atomicshop/file_io/xlsxs.py,sha256=v_dyg9GD4LqgWi6wA1QuWRZ8zG4ZwB6Dz52ytdcmmmI,2184
|
|
123
123
|
atomicshop/file_io/xmls.py,sha256=zh3SuK-dNaFq2NDNhx6ivcf4GYCfGM8M10PcEwDSpxk,2104
|
|
124
124
|
atomicshop/mitm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
125
|
-
atomicshop/mitm/config_static.py,sha256=
|
|
125
|
+
atomicshop/mitm/config_static.py,sha256=ROAtbibSWSsF3BraUbhu-QO3MPIFqYY5KUKgsQbiSkk,7813
|
|
126
126
|
atomicshop/mitm/config_toml_editor.py,sha256=2p1CMcktWRR_NW-SmyDwylu63ad5e0-w1QPMa8ZLDBw,1635
|
|
127
127
|
atomicshop/mitm/connection_thread_worker.py,sha256=rfullUfMRR5YUEI45KJdGXlbOGYF89GfPkhUBoHizhM,20297
|
|
128
128
|
atomicshop/mitm/import_config.py,sha256=_nu8mgA-M4s6dZ8_QWx3x0aVb75upvsCuX_PIUg4X2w,8345
|
|
129
|
-
atomicshop/mitm/initialize_engines.py,sha256=
|
|
129
|
+
atomicshop/mitm/initialize_engines.py,sha256=VyJE8QnzlgD3QbX5inz5o6rC3zQ3is9CeTL7-B10g1w,8292
|
|
130
130
|
atomicshop/mitm/message.py,sha256=d_sm3O_aoZf87dDQP44xOMNEG-uZBN1ZecQgMCacbZs,1814
|
|
131
|
-
atomicshop/mitm/mitm_main.py,sha256=
|
|
132
|
-
atomicshop/mitm/recs_files.py,sha256=
|
|
133
|
-
atomicshop/mitm/shared_functions.py,sha256=
|
|
131
|
+
atomicshop/mitm/mitm_main.py,sha256=L8fcFNU_azIJepymb3JZRnfvvRIsUf1gWNcFQKeQTcs,21876
|
|
132
|
+
atomicshop/mitm/recs_files.py,sha256=VjgtJF7vy365mBjctwB2bpDYoLMkEeogzF8kbb01dAk,2977
|
|
133
|
+
atomicshop/mitm/shared_functions.py,sha256=jjCDZVQCwQ8hf9QNMe3T8W3ISkfZo4Mm2HtXOJLZYgI,1999
|
|
134
134
|
atomicshop/mitm/statistic_analyzer.py,sha256=mBmwEe68WAjnIskGndQTRldnsAsDHuaOW0O7UXlM3Pc,26702
|
|
135
135
|
atomicshop/mitm/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
136
136
|
atomicshop/mitm/engines/create_module_template.py,sha256=tRjVSm1sD6FzML71Qbuwvita0qsusdFGm8NZLsZ-XMs,4853
|
|
@@ -249,7 +249,7 @@ atomicshop/wrappers/loggingw/filters.py,sha256=CMs5PAMb68zxJgBcQobaOFDG5kLJBOVYn
|
|
|
249
249
|
atomicshop/wrappers/loggingw/formatters.py,sha256=7XUJvlB0CK4DCkEp8NTL0S0dkyrZD0UTADgEwkStKOY,5483
|
|
250
250
|
atomicshop/wrappers/loggingw/handlers.py,sha256=hAPFJQ-wFoNO8QzGrJRSvyuP09Q1F0Dl9_w7zzlgcW0,18155
|
|
251
251
|
atomicshop/wrappers/loggingw/loggers.py,sha256=mmM__XR3W4QC82wbsDRG_M4_0JYGGEP0Qn0WCOSp-go,2910
|
|
252
|
-
atomicshop/wrappers/loggingw/loggingw.py,sha256=
|
|
252
|
+
atomicshop/wrappers/loggingw/loggingw.py,sha256=oqH_u1AFyu4YG468ZDnCalRNNu2438AEYZa2MFB9iqg,16143
|
|
253
253
|
atomicshop/wrappers/loggingw/reading.py,sha256=ERBSiQbEksySKpXpu2E_6k9dZ6MPH95ZIsmdjWW9MUE,16436
|
|
254
254
|
atomicshop/wrappers/mongodbw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
255
255
|
atomicshop/wrappers/mongodbw/install_mongodb.py,sha256=3ZPqrXxj3lC-PnAKGXclylLuOqsbyXYeUpb5iGjdeUU,6626
|
|
@@ -296,7 +296,7 @@ atomicshop/wrappers/socketw/accepter.py,sha256=hZZKVYlF3LOHQJsSIEKXZUf6QXXWm-Atq
|
|
|
296
296
|
atomicshop/wrappers/socketw/base.py,sha256=evoOIxg5Xff3THJnrVX00D5HobaOpDp6_e_gso7TJmA,2191
|
|
297
297
|
atomicshop/wrappers/socketw/certificator.py,sha256=3CpQKtcW68FSbH6LVSEZTqWBS6Yg_-3K0x4nFkId4UY,12236
|
|
298
298
|
atomicshop/wrappers/socketw/creator.py,sha256=3_OraDkw2DAWZfoSdY3svCGMOIxpjLEEY7NxWd7M5P4,9873
|
|
299
|
-
atomicshop/wrappers/socketw/dns_server.py,sha256=
|
|
299
|
+
atomicshop/wrappers/socketw/dns_server.py,sha256=r1kWeKVoiqqNb_HpZdk0Pm3Xo-NGDkCs0_0DzMmtE8o,48954
|
|
300
300
|
atomicshop/wrappers/socketw/exception_wrapper.py,sha256=B-X5SHLSUIWToihH2MKnOB1F4A81_X0DpLLfnYKYbEc,7067
|
|
301
301
|
atomicshop/wrappers/socketw/get_process.py,sha256=zKEqh98cB9UDLFhtxVpperfXsCjyIMNANHilDD06p0U,6094
|
|
302
302
|
atomicshop/wrappers/socketw/receiver.py,sha256=XVvWOoeCo3vA0O5p19ryi-hcDIyx382WNG7WzMNVeYk,9322
|
|
@@ -304,13 +304,13 @@ atomicshop/wrappers/socketw/sender.py,sha256=OcX1aeI2OBDPPfvQkiEcV4Ak22DSJVvjMGY
|
|
|
304
304
|
atomicshop/wrappers/socketw/sni.py,sha256=fVwyh3h9IqfLMnf4__bMIzcF4c-Kk9mlbDWMRXKN-ow,17155
|
|
305
305
|
atomicshop/wrappers/socketw/socket_client.py,sha256=hcHtOh43UhJAxYe7DSg4YVSJnCEKLRWKPiZ8DD8ite8,20237
|
|
306
306
|
atomicshop/wrappers/socketw/socket_server_tester.py,sha256=Qobmh4XV8ZxLUaw-eW4ESKAbeSLecCKn2OWFzMhadk0,6420
|
|
307
|
-
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=
|
|
307
|
+
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=VQNvR68xj0RD_v1OL19tDZf4xRskZQjrVE6piHvRXR8,33896
|
|
308
308
|
atomicshop/wrappers/socketw/ssl_base.py,sha256=kmiif84kMhBr5yjQW17p935sfjR5JKG0LxIwBA4iVvU,2275
|
|
309
|
-
atomicshop/wrappers/socketw/statistics_csv.py,sha256=
|
|
309
|
+
atomicshop/wrappers/socketw/statistics_csv.py,sha256=w1AH-zf4mBuT4euf28UKij9ihM-b1BRU9Qfby0QDdqI,2957
|
|
310
310
|
atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
311
311
|
atomicshop/wrappers/winregw/winreg_network.py,sha256=bQ8Jql8bVGBJ0dt3VQ56lga_1LBOMLI3Km_otvvbU6c,7138
|
|
312
|
-
atomicshop-2.16.
|
|
313
|
-
atomicshop-2.16.
|
|
314
|
-
atomicshop-2.16.
|
|
315
|
-
atomicshop-2.16.
|
|
316
|
-
atomicshop-2.16.
|
|
312
|
+
atomicshop-2.16.27.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
313
|
+
atomicshop-2.16.27.dist-info/METADATA,sha256=JD7sHdbqBZwiuJmXdvhYXs4qgSy5S5riQpv0m6lAMKU,10473
|
|
314
|
+
atomicshop-2.16.27.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
315
|
+
atomicshop-2.16.27.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
316
|
+
atomicshop-2.16.27.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|