atomicshop 2.16.11__py3-none-any.whl → 2.16.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/basics/enums.py +2 -2
- atomicshop/basics/list_of_classes.py +29 -0
- atomicshop/file_io/docxs.py +4 -4
- atomicshop/filesystem.py +265 -198
- atomicshop/mitm/config_static.py +1 -1
- atomicshop/mitm/initialize_engines.py +6 -6
- atomicshop/mitm/{initialize_mitm_server.py → mitm_main.py} +4 -3
- atomicshop/mitm/recs_files.py +17 -16
- atomicshop/mitm/statistic_analyzer.py +2 -2
- atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +1 -1
- atomicshop/wrappers/loggingw/reading.py +63 -100
- atomicshop/wrappers/socketw/socket_server_tester.py +5 -5
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.12.dist-info}/METADATA +1 -1
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.12.dist-info}/RECORD +18 -17
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.12.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.12.dist-info}/WHEEL +0 -0
- {atomicshop-2.16.11.dist-info → atomicshop-2.16.12.dist-info}/top_level.txt +0 -0
atomicshop/mitm/config_static.py
CHANGED
|
@@ -52,12 +52,12 @@ class ModuleCategory:
|
|
|
52
52
|
raise ValueError(f"Engine Configuration file doesn't contain any domains: {engine_config_file_path}")
|
|
53
53
|
|
|
54
54
|
# Full path to file
|
|
55
|
-
self.parser_file_path = filesystem.
|
|
56
|
-
engine_directory_path, file_name_check_pattern=configuration_data['parser_file'])[0]
|
|
57
|
-
self.responder_file_path = filesystem.
|
|
58
|
-
engine_directory_path, file_name_check_pattern=configuration_data['responder_file'])[0]
|
|
59
|
-
self.recorder_file_path = filesystem.
|
|
60
|
-
engine_directory_path, file_name_check_pattern=configuration_data['recorder_file'])[0]
|
|
55
|
+
self.parser_file_path = filesystem.get_paths_from_directory(
|
|
56
|
+
engine_directory_path, get_file=True, file_name_check_pattern=configuration_data['parser_file'])[0].path
|
|
57
|
+
self.responder_file_path = filesystem.get_paths_from_directory(
|
|
58
|
+
engine_directory_path, get_file=True, file_name_check_pattern=configuration_data['responder_file'])[0].path
|
|
59
|
+
self.recorder_file_path = filesystem.get_paths_from_directory(
|
|
60
|
+
engine_directory_path, get_file=True, file_name_check_pattern=configuration_data['recorder_file'])[0].path
|
|
61
61
|
|
|
62
62
|
def initialize_engine(self, logs_path: str, logger=None, reference_general: bool = False, **kwargs):
|
|
63
63
|
if not reference_general:
|
|
@@ -32,7 +32,7 @@ def exit_cleanup():
|
|
|
32
32
|
print_api("Returned default DNS gateway...", color='blue')
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def
|
|
35
|
+
def mitm_server_main(config_file_path: str):
|
|
36
36
|
on_exit.register_exit_handler(exit_cleanup)
|
|
37
37
|
|
|
38
38
|
# Main function should return integer with error code, 0 is successful.
|
|
@@ -110,8 +110,9 @@ def initialize_mitm_server(config_file_path: str):
|
|
|
110
110
|
system_logger.info("Importing engine modules.")
|
|
111
111
|
|
|
112
112
|
# Get full paths of all the 'engine_config.ini' files.
|
|
113
|
-
engine_config_path_list = filesystem.
|
|
113
|
+
engine_config_path_list = filesystem.get_paths_from_directory(
|
|
114
114
|
directory_path=config_static.MainConfig.ENGINES_DIRECTORY_PATH,
|
|
115
|
+
get_file=True,
|
|
115
116
|
file_name_check_pattern=config_static.MainConfig.ENGINE_CONFIG_FILE_NAME)
|
|
116
117
|
|
|
117
118
|
# Iterate through all the 'engine_config.ini' file paths.
|
|
@@ -120,7 +121,7 @@ def initialize_mitm_server(config_file_path: str):
|
|
|
120
121
|
for engine_config_path in engine_config_path_list:
|
|
121
122
|
# Initialize engine.
|
|
122
123
|
current_module = ModuleCategory(config_static.MainConfig.SCRIPT_DIRECTORY)
|
|
123
|
-
current_module.fill_engine_fields_from_config(engine_config_path)
|
|
124
|
+
current_module.fill_engine_fields_from_config(engine_config_path.path)
|
|
124
125
|
current_module.initialize_engine(logs_path=config_static.LogRec.logs_path,
|
|
125
126
|
logger=system_logger)
|
|
126
127
|
|
atomicshop/mitm/recs_files.py
CHANGED
|
@@ -22,43 +22,44 @@ def recs_archiver(recs_directory: str) -> list:
|
|
|
22
22
|
today_date_string = datetime.datetime.now().strftime(REC_FILE_DATE_FORMAT)
|
|
23
23
|
|
|
24
24
|
# There should not be recording json files in recs root.
|
|
25
|
-
files_in_recs_root: list = filesystem.
|
|
26
|
-
recs_directory, file_name_check_pattern='
|
|
25
|
+
files_in_recs_root: list = filesystem.get_paths_from_directory(
|
|
26
|
+
recs_directory, get_file=True, file_name_check_pattern='*\\.json', recursive=False)
|
|
27
27
|
if files_in_recs_root:
|
|
28
28
|
raise NotImplementedError("The files in recs root directory are not implemented yet.")
|
|
29
29
|
|
|
30
30
|
# Each engine should have its own directory inside recordings. We will find all the directories inside recs folder.
|
|
31
|
-
directory_paths_in_recs: list = filesystem.
|
|
31
|
+
directory_paths_in_recs: list = filesystem.get_paths_from_directory(
|
|
32
|
+
recs_directory, get_directory=True, recursive=False)
|
|
32
33
|
|
|
33
34
|
file_list_per_directory: list = list()
|
|
34
35
|
for directory_path in directory_paths_in_recs:
|
|
35
|
-
all_recs_files =
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
36
|
+
all_recs_files = filesystem.get_paths_from_directory(
|
|
37
|
+
directory_path=directory_path.path,
|
|
38
|
+
get_file=True,
|
|
39
|
+
file_name_check_pattern='*.json',
|
|
40
|
+
datetime_format=REC_FILE_DATE_FORMAT,
|
|
41
|
+
recursive=False
|
|
39
42
|
)
|
|
40
43
|
file_list_per_directory.append((directory_path, all_recs_files))
|
|
41
44
|
|
|
42
45
|
archived_files: list = list()
|
|
43
46
|
for directory_path, all_recs_files in file_list_per_directory:
|
|
44
|
-
|
|
45
|
-
for recs_file_dict in all_recs_files:
|
|
47
|
+
for recs_atomic_path in all_recs_files:
|
|
46
48
|
# We don't need to archive today's files.
|
|
47
|
-
if today_date_string ==
|
|
49
|
+
if today_date_string == recs_atomic_path.datetime_string:
|
|
48
50
|
continue
|
|
49
51
|
|
|
50
|
-
target_directory_path: str = f"{directory_path}{os.sep}{
|
|
51
|
-
if target_directory_path not in archive_directories:
|
|
52
|
-
archive_directories.append(target_directory_path)
|
|
53
|
-
|
|
52
|
+
target_directory_path: str = f"{directory_path.path}{os.sep}{recs_atomic_path.datetime_string}"
|
|
54
53
|
filesystem.create_directory(target_directory_path)
|
|
55
54
|
filesystem.move_file(
|
|
56
|
-
|
|
55
|
+
recs_atomic_path.path, f'{target_directory_path}{os.sep}{recs_atomic_path.name}')
|
|
57
56
|
|
|
58
57
|
# Archive directories.
|
|
58
|
+
archive_directories: list = filesystem.get_paths_from_directory(
|
|
59
|
+
directory_path.path, get_directory=True, recursive=False)
|
|
59
60
|
for archive_directory in archive_directories:
|
|
60
61
|
archived_file: str = zips.archive_directory(
|
|
61
|
-
archive_directory, remove_original=True, include_root_directory=True)
|
|
62
|
+
archive_directory.path, remove_original=True, include_root_directory=True)
|
|
62
63
|
archived_files.append(archived_file)
|
|
63
64
|
|
|
64
65
|
return archived_files
|
|
@@ -28,9 +28,9 @@ def analyze(main_file_path: str):
|
|
|
28
28
|
summary_path: str = filesystem.check_absolute_path___add_full(config['report_file_path'], script_directory)
|
|
29
29
|
|
|
30
30
|
# Get the content from statistics files.
|
|
31
|
+
log_file_path_pattern: str = f"{config['statistic_files_path']}{os.sep}statistics.csv"
|
|
31
32
|
statistics_content: list = reading.get_all_log_files_into_list(
|
|
32
|
-
|
|
33
|
-
file_name_pattern='statistics*.csv',
|
|
33
|
+
log_file_path=log_file_path_pattern,
|
|
34
34
|
log_type='csv'
|
|
35
35
|
)
|
|
36
36
|
|
|
@@ -66,7 +66,7 @@ def install_before_restart(
|
|
|
66
66
|
|
|
67
67
|
if not fact_source_archive_path:
|
|
68
68
|
# Download the FACT_core repo.
|
|
69
|
-
if not filesystem.
|
|
69
|
+
if not filesystem.get_paths_from_directory(installation_directory, get_file=True):
|
|
70
70
|
git_wrapper = githubw.GitHubWrapper(repo_url=config_install.FACT_CORE_GITHUB_URL)
|
|
71
71
|
git_wrapper.build_links_from_repo_url()
|
|
72
72
|
git_wrapper.download_and_extract_branch(
|
|
@@ -4,18 +4,12 @@ from pathlib import Path
|
|
|
4
4
|
import datetime
|
|
5
5
|
|
|
6
6
|
from ... import filesystem, datetimes
|
|
7
|
-
from ...basics import booleans
|
|
7
|
+
from ...basics import booleans, list_of_classes
|
|
8
8
|
from ...file_io import csvs
|
|
9
9
|
|
|
10
10
|
|
|
11
|
-
class LogReaderTimeCouldntBeFoundInFileNameError(Exception):
|
|
12
|
-
pass
|
|
13
|
-
|
|
14
|
-
|
|
15
11
|
def get_logs_paths(
|
|
16
|
-
|
|
17
|
-
log_file_path: str = None,
|
|
18
|
-
file_name_pattern: str = '*.*',
|
|
12
|
+
log_file_path: str,
|
|
19
13
|
date_format: str = None,
|
|
20
14
|
latest_only: bool = False,
|
|
21
15
|
previous_day_only: bool = False,
|
|
@@ -24,8 +18,6 @@ def get_logs_paths(
|
|
|
24
18
|
"""
|
|
25
19
|
This function gets the logs file paths from the directory. Supports rotating files to get the logs by time.
|
|
26
20
|
|
|
27
|
-
:param log_files_directory_path: Path to the log files. If specified, the function will get all the files from the
|
|
28
|
-
directory by the 'file_name_pattern'.
|
|
29
21
|
:param log_file_path: Path to the log file. If specified, the function will get the file and all the rotated logs
|
|
30
22
|
associated with this file. The 'file_name_pattern' will become the file name using the file name and extension.
|
|
31
23
|
|
|
@@ -37,8 +29,6 @@ def get_logs_paths(
|
|
|
37
29
|
|
|
38
30
|
# The 'log_files_directory_path' will also be taken from the 'log_file_path':
|
|
39
31
|
log_files_directory_path = 'C:/logs'
|
|
40
|
-
:param file_name_pattern: Pattern to match the log files names.
|
|
41
|
-
Default file_name_pattern will match all the files.
|
|
42
32
|
:param date_format: date format string pattern to match the date in the log file name.
|
|
43
33
|
If specified, the function will get the log file by the date pattern.
|
|
44
34
|
If not specified, the function will get the file date by file last modified time.
|
|
@@ -53,11 +43,6 @@ def get_logs_paths(
|
|
|
53
43
|
|
|
54
44
|
"""
|
|
55
45
|
|
|
56
|
-
if not log_files_directory_path and not log_file_path:
|
|
57
|
-
raise ValueError('Either "log_files_directory_path" or "log_file_path" must be specified.')
|
|
58
|
-
elif log_files_directory_path and log_file_path:
|
|
59
|
-
raise ValueError('Both "log_files_directory_path" and "log_file_path" cannot be specified at the same time.')
|
|
60
|
-
|
|
61
46
|
if latest_only or previous_day_only or specific_date:
|
|
62
47
|
booleans.check_3_booleans_when_only_1_can_be_true(
|
|
63
48
|
(latest_only, 'latest_only'),
|
|
@@ -67,73 +52,63 @@ def get_logs_paths(
|
|
|
67
52
|
if not date_format and specific_date:
|
|
68
53
|
raise ValueError('If "specific_date" is specified, "date_format" must be specified.')
|
|
69
54
|
|
|
70
|
-
#
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
log_file_extension: str = Path(log_file_path).suffix
|
|
75
|
-
file_name_pattern = f'{log_file_name}*{log_file_extension}'
|
|
55
|
+
# Get the file_name_pattern from the file name. Build the file_name_pattern.
|
|
56
|
+
log_file_name: str = Path(log_file_path).stem
|
|
57
|
+
log_file_extension: str = Path(log_file_path).suffix
|
|
58
|
+
file_name_pattern: str = f'{log_file_name}*{log_file_extension}'
|
|
76
59
|
|
|
77
|
-
|
|
78
|
-
|
|
60
|
+
# Get the directory path from the file path.
|
|
61
|
+
log_files_directory_path: str = str(Path(log_file_path).parent)
|
|
79
62
|
|
|
80
|
-
# Get all the log file paths by the file_name_pattern.
|
|
81
|
-
logs_files: list = filesystem.
|
|
63
|
+
# Get all the log file paths by the file_name_pattern and the date_format string.
|
|
64
|
+
logs_files: list = filesystem.get_paths_from_directory(
|
|
82
65
|
log_files_directory_path,
|
|
66
|
+
get_file=True,
|
|
83
67
|
file_name_check_pattern=file_name_pattern,
|
|
84
68
|
add_last_modified_time=True,
|
|
85
|
-
sort_by_last_modified_time=True
|
|
69
|
+
sort_by_last_modified_time=True,
|
|
70
|
+
datetime_format=date_format
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
# The above will not include the latest log file if it is not rotated yet.
|
|
74
|
+
# noinspection PyTypeChecker
|
|
75
|
+
last_log_file_atomic_path: filesystem.AtomicPath = None
|
|
76
|
+
if os.path.isfile(log_file_path):
|
|
77
|
+
last_log_file_atomic_path = filesystem.AtomicPath(log_file_path)
|
|
78
|
+
last_log_file_atomic_path.update(update_last_modified=True)
|
|
79
|
+
|
|
80
|
+
if logs_files and last_log_file_atomic_path and date_format:
|
|
81
|
+
# The problem here is the file name that doesn't contain the date string in the name.
|
|
82
|
+
# If it is regular log rotation, then there will be one file that doesn't have the date string in the name.
|
|
83
|
+
# If the function used to get the previous day log, then there will be no file that doesn't have the date
|
|
84
|
+
# string.
|
|
85
|
+
|
|
86
|
+
# Get the latest timestamp from the files with dates.
|
|
87
|
+
latest_datetime_float: float = 0
|
|
88
|
+
for file_index, single_file in enumerate(logs_files):
|
|
89
|
+
if single_file.datetime_float > latest_datetime_float:
|
|
90
|
+
latest_datetime_float = single_file.datetime_float
|
|
91
|
+
|
|
92
|
+
# We will add one day to the latest date that we found and assign to the latest file in rotation
|
|
93
|
+
# which is without the datetime string.
|
|
94
|
+
latest_datetime_float += 86400
|
|
95
|
+
last_log_file_atomic_path.datetime_float = latest_datetime_float
|
|
96
|
+
last_log_file_atomic_path.datetime_datetime = datetime.datetime.fromtimestamp(latest_datetime_float)
|
|
97
|
+
last_log_file_atomic_path.datetime_string = (
|
|
98
|
+
last_log_file_atomic_path.datetime_datetime.strftime(date_format))
|
|
99
|
+
last_log_file_atomic_path.datetime_format = date_format
|
|
100
|
+
|
|
101
|
+
# Add the last log file to the list.
|
|
102
|
+
logs_files.append(last_log_file_atomic_path)
|
|
103
|
+
|
|
104
|
+
# Sort the files by the last modified time.
|
|
105
|
+
logs_files = list_of_classes.sort_by_attributes(logs_files, ['datetime_float'])
|
|
106
|
+
elif last_log_file_atomic_path and logs_files and not date_format:
|
|
107
|
+
logs_files.append(last_log_file_atomic_path)
|
|
108
|
+
elif last_log_file_atomic_path and not logs_files:
|
|
109
|
+
logs_files = [last_log_file_atomic_path]
|
|
86
110
|
|
|
87
|
-
# Get the datetime object from the first file name by the date pattern.
|
|
88
|
-
first_date_string = None
|
|
89
111
|
if logs_files:
|
|
90
|
-
first_file_name: str = Path(logs_files[0]['file_path']).name
|
|
91
|
-
first_datetime_object, first_date_string, first_timestamp_float = (
|
|
92
|
-
datetimes.get_datetime_from_complex_string_by_pattern(first_file_name, date_format))
|
|
93
|
-
|
|
94
|
-
# The problem here is the file name that doesn't contain the date string in the name.
|
|
95
|
-
# If it is regular log rotation, then there will be one file that doesn't have the date string in the name.
|
|
96
|
-
# If the function used to get the previous day log, then there will be no file that doesn't have the date string.
|
|
97
|
-
if len(logs_files) > 1 or (len(logs_files) == 1 and first_date_string):
|
|
98
|
-
if date_format:
|
|
99
|
-
latest_timestamp: float = 0
|
|
100
|
-
for file_index, single_file in enumerate(logs_files):
|
|
101
|
-
# Get file name from current loop file path.
|
|
102
|
-
current_file_name: str = Path(single_file['file_path']).name
|
|
103
|
-
logs_files[file_index]['file_name'] = current_file_name
|
|
104
|
-
|
|
105
|
-
# Get the datetime object from the file name by the date format pattern.
|
|
106
|
-
datetime_object, date_string, timestamp_float = (
|
|
107
|
-
datetimes.get_datetime_from_complex_string_by_pattern(current_file_name, date_format))
|
|
108
|
-
|
|
109
|
-
# Update the last modified time to the dictionary.
|
|
110
|
-
logs_files[file_index]['last_modified'] = timestamp_float
|
|
111
|
-
logs_files[file_index]['datetime'] = datetime_object
|
|
112
|
-
logs_files[file_index]['date_string'] = date_string
|
|
113
|
-
|
|
114
|
-
if timestamp_float and timestamp_float > latest_timestamp:
|
|
115
|
-
latest_timestamp = timestamp_float
|
|
116
|
-
|
|
117
|
-
# Check timestamps, if more than 1 file is None, then the function that gets the date from the file name
|
|
118
|
-
# didn't work properly, probably because of the string datetime format or the filenames.
|
|
119
|
-
none_timestamps = [single_file['last_modified'] for single_file in logs_files].count(None)
|
|
120
|
-
if none_timestamps > 1:
|
|
121
|
-
raise LogReaderTimeCouldntBeFoundInFileNameError(
|
|
122
|
-
'The date pattern could not be found in the file name. Check the date pattern and the file names.')
|
|
123
|
-
|
|
124
|
-
# Now, there should be a file that doesn't have the string date pattern in the file name.
|
|
125
|
-
# We will add one day to the latest date that we found and assign to that file path.
|
|
126
|
-
for file_index, single_file in enumerate(logs_files):
|
|
127
|
-
if single_file['last_modified'] is None:
|
|
128
|
-
latest_timestamp += 86400
|
|
129
|
-
logs_files[file_index]['last_modified'] = latest_timestamp
|
|
130
|
-
logs_files[file_index]['datetime'] = datetime.datetime.fromtimestamp(latest_timestamp)
|
|
131
|
-
logs_files[file_index]['date_string'] = logs_files[file_index]['datetime'].strftime(date_format)
|
|
132
|
-
break
|
|
133
|
-
|
|
134
|
-
# Sort the files by the last modified time.
|
|
135
|
-
logs_files = sorted(logs_files, key=lambda x: x['last_modified'], reverse=False)
|
|
136
|
-
|
|
137
112
|
if latest_only:
|
|
138
113
|
logs_files = [logs_files[-1]]
|
|
139
114
|
|
|
@@ -146,19 +121,13 @@ def get_logs_paths(
|
|
|
146
121
|
|
|
147
122
|
if specific_date:
|
|
148
123
|
# Check if there is a specific date log file.
|
|
149
|
-
logs_files = [single_file for single_file in logs_files if single_file
|
|
150
|
-
# If there is only one file, meaning it is the current day log.
|
|
151
|
-
# If the 'previous_day_only' is True, then there are no previous day logs to output.
|
|
152
|
-
elif len(logs_files) == 1 and previous_day_only:
|
|
153
|
-
logs_files = []
|
|
124
|
+
logs_files = [single_file for single_file in logs_files if single_file.datetime_string == specific_date]
|
|
154
125
|
|
|
155
126
|
return logs_files
|
|
156
127
|
|
|
157
128
|
|
|
158
129
|
def get_all_log_files_into_list(
|
|
159
|
-
log_files_directory_path: str = None,
|
|
160
130
|
log_file_path: str = None,
|
|
161
|
-
file_name_pattern: str = '*.*',
|
|
162
131
|
date_format: str = None,
|
|
163
132
|
log_type: Literal['csv'] = 'csv',
|
|
164
133
|
header_type_of_files: Literal['first', 'all'] = 'first',
|
|
@@ -170,10 +139,7 @@ def get_all_log_files_into_list(
|
|
|
170
139
|
This function gets the logs contents from the log files. Supports rotating files to get the logs by time.
|
|
171
140
|
All the contents will be merged into one list.
|
|
172
141
|
|
|
173
|
-
:param log_files_directory_path: Path to the log files. Check the 'get_logs_paths' function for more details.
|
|
174
142
|
:param log_file_path: Path to the log file. Check the 'get_logs_paths' function for more details.
|
|
175
|
-
:param file_name_pattern: Pattern to match the log files names.
|
|
176
|
-
Default file_name_pattern will match all the files.
|
|
177
143
|
:param date_format: date format string pattern to match the date in the log file name.
|
|
178
144
|
If specified, the function will get the log file by the date pattern.
|
|
179
145
|
If not specified, the function will get the file date by file last modified time.
|
|
@@ -202,9 +168,7 @@ def get_all_log_files_into_list(
|
|
|
202
168
|
|
|
203
169
|
# Get all the log file paths by the file_name_pattern.
|
|
204
170
|
logs_files: list = get_logs_paths(
|
|
205
|
-
log_files_directory_path=log_files_directory_path,
|
|
206
171
|
log_file_path=log_file_path,
|
|
207
|
-
file_name_pattern=file_name_pattern,
|
|
208
172
|
date_format=date_format)
|
|
209
173
|
|
|
210
174
|
# Read all the logs.
|
|
@@ -213,13 +177,13 @@ def get_all_log_files_into_list(
|
|
|
213
177
|
for single_file in logs_files:
|
|
214
178
|
if log_type == 'csv':
|
|
215
179
|
if header_type_of_files == 'all':
|
|
216
|
-
csv_content, _ = csvs.read_csv_to_list_of_dicts_by_header(single_file
|
|
180
|
+
csv_content, _ = csvs.read_csv_to_list_of_dicts_by_header(single_file.path, **print_kwargs)
|
|
217
181
|
logs_content.extend(csv_content)
|
|
218
182
|
elif header_type_of_files == 'first':
|
|
219
183
|
# The function gets empty header to read it from the CSV file, the returns the header that it read.
|
|
220
184
|
# Then each time the header is fed once again to the function.
|
|
221
185
|
csv_content, header = csvs.read_csv_to_list_of_dicts_by_header(
|
|
222
|
-
single_file
|
|
186
|
+
single_file.path, header=header, **print_kwargs)
|
|
223
187
|
# Any way the first file will be read with header.
|
|
224
188
|
logs_content.extend(csv_content)
|
|
225
189
|
|
|
@@ -230,7 +194,7 @@ def get_all_log_files_into_list(
|
|
|
230
194
|
if remove_logs:
|
|
231
195
|
# Remove the statistics files.
|
|
232
196
|
for single_file in logs_files:
|
|
233
|
-
filesystem.remove_file(single_file
|
|
197
|
+
filesystem.remove_file(single_file.path)
|
|
234
198
|
|
|
235
199
|
if move_to_path:
|
|
236
200
|
# Get formatted time stamp for file name.
|
|
@@ -243,9 +207,8 @@ def get_all_log_files_into_list(
|
|
|
243
207
|
filesystem.create_directory(move_to_path_with_timestamp)
|
|
244
208
|
# Move the statistics files.
|
|
245
209
|
for single_file in logs_files:
|
|
246
|
-
|
|
247
|
-
move_to_path_with_file
|
|
248
|
-
filesystem.move_file(single_file['file_path'], move_to_path_with_file)
|
|
210
|
+
move_to_path_with_file = f'{move_to_path_with_timestamp}{os.sep}{single_file.name}'
|
|
211
|
+
filesystem.move_file(single_file.path, move_to_path_with_file)
|
|
249
212
|
|
|
250
213
|
return logs_content
|
|
251
214
|
|
|
@@ -301,9 +264,9 @@ class LogReader:
|
|
|
301
264
|
:param log_type: Type of log to get.
|
|
302
265
|
:param get_previous_file: Boolean, if True, the function will get the previous log file.
|
|
303
266
|
For example, your log is set to rotate every Midnight.
|
|
304
|
-
Meaning, once the day will change, the function will get the log file from the previous day in the
|
|
305
|
-
of the return tuple. This happens only once each 24 hours. Not from the time
|
|
306
|
-
the time the day changed.
|
|
267
|
+
Meaning, once the day will change, the function will get the log file from the previous day in the
|
|
268
|
+
third entry of the return tuple. This happens only once each 24 hours. Not from the time
|
|
269
|
+
the function was called, but from the time the day changed.
|
|
307
270
|
:param header: List of strings that will be the header of the CSV file. Default is 'None'.
|
|
308
271
|
None: the header from the CSV file will be used. The first row of the CSV file will be the header.
|
|
309
272
|
Meaning, that the first line will be skipped and the second line will be the first row of the content.
|
|
@@ -362,7 +325,7 @@ class LogReader:
|
|
|
362
325
|
# if not latest_statistics_file_path_object:
|
|
363
326
|
# return [], [], self.header
|
|
364
327
|
|
|
365
|
-
latest_statistics_file_path: str = latest_statistics_file_path_object[0]
|
|
328
|
+
latest_statistics_file_path: str = latest_statistics_file_path_object[0].path
|
|
366
329
|
|
|
367
330
|
# Get the previous day statistics file path.
|
|
368
331
|
previous_day_statistics_file_path: Union[str, None] = None
|
|
@@ -371,7 +334,7 @@ class LogReader:
|
|
|
371
334
|
log_file_path=self.log_file_path,
|
|
372
335
|
date_format=self.date_format,
|
|
373
336
|
previous_day_only=True
|
|
374
|
-
)[0]
|
|
337
|
+
)[0].path
|
|
375
338
|
# If you get IndexError, it means that there are no previous day logs to read.
|
|
376
339
|
except IndexError:
|
|
377
340
|
pass
|
|
@@ -3,7 +3,7 @@ import threading
|
|
|
3
3
|
from .socket_client import SocketClient
|
|
4
4
|
from ..configparserw import ConfigParserWrapper
|
|
5
5
|
from ..loggingw import loggingw
|
|
6
|
-
from ...
|
|
6
|
+
from ... import filesystem
|
|
7
7
|
from ...file_io import jsons, file_io
|
|
8
8
|
|
|
9
9
|
|
|
@@ -57,14 +57,14 @@ def execute_test(config_static):
|
|
|
57
57
|
loggingw.get_logger_with_stream_handler("network")
|
|
58
58
|
|
|
59
59
|
# Get all the files in requests folder recursively.
|
|
60
|
-
request_file_list =
|
|
60
|
+
request_file_list = filesystem.get_paths_from_directory(config['requests_directory'], get_file=True)
|
|
61
61
|
print(f"Found request files: {len(request_file_list)}")
|
|
62
62
|
|
|
63
63
|
# Get contents of all request files to list of contents.
|
|
64
64
|
requests_bytes_list: list = list()
|
|
65
65
|
for request_file_path in request_file_list:
|
|
66
66
|
if config['request_type'] == 'json':
|
|
67
|
-
request_file_content = jsons.read_json_file(request_file_path)
|
|
67
|
+
request_file_content = jsons.read_json_file(request_file_path.path)
|
|
68
68
|
|
|
69
69
|
# If imported json is regular and not combined json.
|
|
70
70
|
if isinstance(request_file_content, dict):
|
|
@@ -79,13 +79,13 @@ def execute_test(config_static):
|
|
|
79
79
|
requests_bytes_list.extend(
|
|
80
80
|
get_key_values_from_json(json_dict, config['request_json_hex_key_list']))
|
|
81
81
|
elif config['request_type'] == 'string':
|
|
82
|
-
request_file_content = file_io.read_file(request_file_path)
|
|
82
|
+
request_file_content = file_io.read_file(request_file_path.path)
|
|
83
83
|
# Convert string content to bytes and append to list.
|
|
84
84
|
requests_bytes_list.append(request_file_content.encode())
|
|
85
85
|
print(f"Extracted 1 request.")
|
|
86
86
|
elif config['request_type'] == 'binary':
|
|
87
87
|
# The content is already in bytes, so just appending.
|
|
88
|
-
requests_bytes_list.append(file_io.read_file(request_file_path, 'rb'))
|
|
88
|
+
requests_bytes_list.append(file_io.read_file(request_file_path.path, 'rb'))
|
|
89
89
|
print(f"Extracted 1 request.")
|
|
90
90
|
|
|
91
91
|
print(f"Finished parsing. Parsed requests: {len(requests_bytes_list)}")
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
atomicshop/__init__.py,sha256=
|
|
1
|
+
atomicshop/__init__.py,sha256=62uoYqbNQHiHzG_4RP9rz3BnzTcxCNm9aF_cLTSSAto,124
|
|
2
2
|
atomicshop/_basics_temp.py,sha256=6cu2dd6r2dLrd1BRNcVDKTHlsHs_26Gpw8QS6v32lQ0,3699
|
|
3
3
|
atomicshop/_create_pdf_demo.py,sha256=Yi-PGZuMg0RKvQmLqVeLIZYadqEZwUm-4A9JxBl_vYA,3713
|
|
4
4
|
atomicshop/_patch_import.py,sha256=ENp55sKVJ0e6-4lBvZnpz9PQCt3Otbur7F6aXDlyje4,6334
|
|
@@ -14,7 +14,7 @@ atomicshop/dns.py,sha256=J4yX6vCaRdL0McYWnlJ9arCDKW-yRui7Y5WyL5BoD6M,6391
|
|
|
14
14
|
atomicshop/domains.py,sha256=Rxu6JhhMqFZRcoFs69IoEd1PtYca0lMCG6F1AomP7z4,3197
|
|
15
15
|
atomicshop/emails.py,sha256=I0KyODQpIMEsNRi9YWSOL8EUPBiWyon3HRdIuSj3AEU,1410
|
|
16
16
|
atomicshop/file_types.py,sha256=-0jzQMRlmU1AP9DARjk-HJm1tVE22E6ngP2mRblyEjY,763
|
|
17
|
-
atomicshop/filesystem.py,sha256=
|
|
17
|
+
atomicshop/filesystem.py,sha256=5xkZFIKYoTTmRBkJo5p4R1G48Xbdjlyc7R514QqNXQs,58711
|
|
18
18
|
atomicshop/functions.py,sha256=pK8hoCE9z61PtWCxQJsda7YAphrLH1wxU5x-1QJP-sY,499
|
|
19
19
|
atomicshop/get_process_list.py,sha256=8cxb7gKe9sl4R6H2yMi8J6oe-RkonTvCdKjRFqi-Fs4,6075
|
|
20
20
|
atomicshop/get_process_name_cmd_dll.py,sha256=CtaSp3mgxxJKCCVW8BLx6BJNx4giCklU_T7USiCEwfc,5162
|
|
@@ -88,12 +88,13 @@ atomicshop/basics/classes.py,sha256=EijW_g4EhdNBnKPMG3nT3HjFspTchtM7to6zm9Ad_Mk,
|
|
|
88
88
|
atomicshop/basics/dicts.py,sha256=DeYHIh940pMMBrFhpXt4dsigFVYzTrlqWymNo4Pq_Js,14049
|
|
89
89
|
atomicshop/basics/dicts_nested.py,sha256=StYxYnYPa0SEJr1lmEwAv5zfERWWqoULeyG8e0zRAwE,4107
|
|
90
90
|
atomicshop/basics/enumerations.py,sha256=41VVQYh_vnVapggxKg2IRU5e_EiMpZzX1n1mtxvoSzM,1364
|
|
91
|
-
atomicshop/basics/enums.py,sha256=
|
|
91
|
+
atomicshop/basics/enums.py,sha256=aAk1jFeQLvrC4NOpk9kgyX1-DCBr2ArPhZ8Ad7cMAVA,3537
|
|
92
92
|
atomicshop/basics/exceptions.py,sha256=-1Gu8bHA3hrf11mmeaPuVE73jWV3EOyRgh2vSpWfveg,504
|
|
93
93
|
atomicshop/basics/guids.py,sha256=iRx5n18ATZWhpo748BwEjuLWLsu9y3OwF5-Adp-Dtik,403
|
|
94
94
|
atomicshop/basics/hexs.py,sha256=i8CTG-J0TGGa25yFSbWEvpVyHFnof_qSWUrmXY-ylKM,1054
|
|
95
95
|
atomicshop/basics/if_else.py,sha256=MakivJChofZCpr0mOVjwCthzpiaBxXVB-zv7GwMOqVo,202
|
|
96
96
|
atomicshop/basics/isinstancing.py,sha256=fQ35xfqbguQz2BUn-3a4KVGskhTcIn8JjRtxV2rFcRQ,876
|
|
97
|
+
atomicshop/basics/list_of_classes.py,sha256=PJoE1VJdhhQ4gSFr88zW7IApXd4Ez7xLz-7vAM-7gug,978
|
|
97
98
|
atomicshop/basics/list_of_dicts.py,sha256=tj0LNPf1ljNI_qpoO-PiOT4Ulmk1M-UpTGyn9twVcw8,8039
|
|
98
99
|
atomicshop/basics/lists.py,sha256=I0C62vrDrNwCTNl0EjUZNa1Jsd8l0rTkp28GEx9QoEI,4258
|
|
99
100
|
atomicshop/basics/multiprocesses.py,sha256=nSskxJSlEdalPM_Uf8cc9kAYYlVwYM1GonBLAhCL2mM,18831
|
|
@@ -114,23 +115,23 @@ atomicshop/etws/traces/trace_dns.py,sha256=WvOZm7KNdP4r6ofkZhUGi9WjtYlkV3mUp_yxi
|
|
|
114
115
|
atomicshop/etws/traces/trace_sysmon_process_creation.py,sha256=OM-bkK38uYMwWLZKNOTDa0Xdk3sO6sqsxoMUIiPvm5g,4656
|
|
115
116
|
atomicshop/file_io/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
116
117
|
atomicshop/file_io/csvs.py,sha256=oZiaIEd1q50ypNdd9mlHWb-f7HAdGa_D6jLd3T_4sWU,8777
|
|
117
|
-
atomicshop/file_io/docxs.py,sha256=
|
|
118
|
+
atomicshop/file_io/docxs.py,sha256=ffJhnmM_WyD8mCoq2dGdpfahdIrGTPy96QVlH5EWjeI,5754
|
|
118
119
|
atomicshop/file_io/file_io.py,sha256=fkOgoeS8Ow69rj6KfEDmnMeFT3FhJmOJ0X-C_7Udiik,7047
|
|
119
120
|
atomicshop/file_io/jsons.py,sha256=q9ZU8slBKnHLrtn3TnbK1qxrRpj5ZvCm6AlsFzoANjo,5303
|
|
120
121
|
atomicshop/file_io/tomls.py,sha256=ol8EvQPf9sryTmZUf1v55BYSUQ6ml7HVVBHpNKbsIlA,9768
|
|
121
122
|
atomicshop/file_io/xlsxs.py,sha256=v_dyg9GD4LqgWi6wA1QuWRZ8zG4ZwB6Dz52ytdcmmmI,2184
|
|
122
123
|
atomicshop/file_io/xmls.py,sha256=zh3SuK-dNaFq2NDNhx6ivcf4GYCfGM8M10PcEwDSpxk,2104
|
|
123
124
|
atomicshop/mitm/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
124
|
-
atomicshop/mitm/config_static.py,sha256=
|
|
125
|
+
atomicshop/mitm/config_static.py,sha256=bcZPp9g12cMNcmHm6chpI6ZrbW6b1Xrz4IVOzJahao4,7105
|
|
125
126
|
atomicshop/mitm/config_toml_editor.py,sha256=2p1CMcktWRR_NW-SmyDwylu63ad5e0-w1QPMa8ZLDBw,1635
|
|
126
127
|
atomicshop/mitm/connection_thread_worker.py,sha256=1MBpRoLpzWJMvxqQKizo6IVQ4XYsbKGsjxianNQLUlE,20051
|
|
127
128
|
atomicshop/mitm/import_config.py,sha256=5peDr6cT0ZWK3J53yG-VEew77CKrvB88CphM10SQd3I,7868
|
|
128
|
-
atomicshop/mitm/initialize_engines.py,sha256=
|
|
129
|
-
atomicshop/mitm/initialize_mitm_server.py,sha256=oSmL-MspKZY_5s-RDlTZ7-v4gIK_L-6Jyus3R1YeTe8,20665
|
|
129
|
+
atomicshop/mitm/initialize_engines.py,sha256=F_FESrvTghEln-P9wesvXnr3_sxczYJBN6QtmQ3d2WE,8249
|
|
130
130
|
atomicshop/mitm/message.py,sha256=d_sm3O_aoZf87dDQP44xOMNEG-uZBN1ZecQgMCacbZs,1814
|
|
131
|
-
atomicshop/mitm/
|
|
131
|
+
atomicshop/mitm/mitm_main.py,sha256=UQ8vCHE83H5Q5Uffv3dIGCH8Buigq6FxSWgLm6_vRGc,20683
|
|
132
|
+
atomicshop/mitm/recs_files.py,sha256=btOuYQca4DuBOAKp9OY21HGjeEVOx9r_k-AnZOqs3Dk,3007
|
|
132
133
|
atomicshop/mitm/shared_functions.py,sha256=hplm98tz8pgJ4WHUVI9sf_oVqUM2KJ1Y2pD6EFSb8P0,1879
|
|
133
|
-
atomicshop/mitm/statistic_analyzer.py,sha256=
|
|
134
|
+
atomicshop/mitm/statistic_analyzer.py,sha256=AzL9rQhg0tLJj33gZfxdwWghmbXGLh_HyMBDpzuBmsQ,24709
|
|
134
135
|
atomicshop/mitm/engines/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
135
136
|
atomicshop/mitm/engines/create_module_template.py,sha256=tRjVSm1sD6FzML71Qbuwvita0qsusdFGm8NZLsZ-XMs,4853
|
|
136
137
|
atomicshop/mitm/engines/create_module_template_example.py,sha256=X5xhvbV6-g9jU_bQVhf_crZmaH50LRWz3bS-faQ18ds,489
|
|
@@ -226,7 +227,7 @@ atomicshop/wrappers/factw/fact_extractor/docker_image.py,sha256=2FyYjnw8gxFNwISQ
|
|
|
226
227
|
atomicshop/wrappers/factw/fact_extractor/get_extractor.py,sha256=2mfOAftHIlCcGt1s7MWdq7DsDCuI6wX3MtvcEZ4SK-0,756
|
|
227
228
|
atomicshop/wrappers/factw/install/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
228
229
|
atomicshop/wrappers/factw/install/install_after_restart.py,sha256=-VXC3KDX2BzF0oi0ELCmfch55vLk-3t16KxlmYyUGD8,1560
|
|
229
|
-
atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py,sha256=
|
|
230
|
+
atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py,sha256=GFsO9MTH0czKoxkiPJtjalilUwsmFLBCcx9Znv37S4M,5945
|
|
230
231
|
atomicshop/wrappers/factw/postgresql/__init__.py,sha256=xMBn2d3Exo23IPP2F_9-SXmOlhFbwWDgS9KwozSTjA0,162
|
|
231
232
|
atomicshop/wrappers/factw/postgresql/analysis.py,sha256=2Rxzy2jyq3zEKIo53z8VkjuslKE_i5mq2ZpmJAvyd6U,716
|
|
232
233
|
atomicshop/wrappers/factw/postgresql/file_object.py,sha256=VRiCXnsd6yDbnsE-TEKYPC-gkAgFVkE6rygRrJLQShI,713
|
|
@@ -249,7 +250,7 @@ atomicshop/wrappers/loggingw/formatters.py,sha256=7XUJvlB0CK4DCkEp8NTL0S0dkyrZD0
|
|
|
249
250
|
atomicshop/wrappers/loggingw/handlers.py,sha256=yFYBeTkxnpmtlauoH3ZEFEHUYQYu9YL-ycd9sYTvOl4,16928
|
|
250
251
|
atomicshop/wrappers/loggingw/loggers.py,sha256=DHOOTAtqkwn1xgvLHSkOiBm6yFGNuQy1kvbhG-TDog8,2374
|
|
251
252
|
atomicshop/wrappers/loggingw/loggingw.py,sha256=lo4OZPXCbYZi3GqpaaJSs9SOGFfqD2EgHzzTK7f5IR4,11275
|
|
252
|
-
atomicshop/wrappers/loggingw/reading.py,sha256=
|
|
253
|
+
atomicshop/wrappers/loggingw/reading.py,sha256=tplnJlQ7RMxWv2s782tWGOo1C7WNemk2SRTZCCgD9J0,16474
|
|
253
254
|
atomicshop/wrappers/mongodbw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
254
255
|
atomicshop/wrappers/mongodbw/infrastructure.py,sha256=tHqtt__yKGtj24CT5AIk0V0k9t1p_PjezFExXRmmmcA,1517
|
|
255
256
|
atomicshop/wrappers/mongodbw/install_mongodb.py,sha256=21Re0NpikgYOKelyPTkosoQQxzu67dy80Kj204TL5eo,6644
|
|
@@ -301,14 +302,14 @@ atomicshop/wrappers/socketw/receiver.py,sha256=G3hDTacm7nwwUNHEbKZpxO0c8rHcl0NeK
|
|
|
301
302
|
atomicshop/wrappers/socketw/sender.py,sha256=d7YQFlCBMFTYtkGxbS-8cm5rh5WWFeBVvrEivWHYstI,3666
|
|
302
303
|
atomicshop/wrappers/socketw/sni.py,sha256=fVwyh3h9IqfLMnf4__bMIzcF4c-Kk9mlbDWMRXKN-ow,17155
|
|
303
304
|
atomicshop/wrappers/socketw/socket_client.py,sha256=FNmTt94YvjZP0X4RPb7icO3xD_nBHQ_XynnObdWFiAU,19682
|
|
304
|
-
atomicshop/wrappers/socketw/socket_server_tester.py,sha256=
|
|
305
|
+
atomicshop/wrappers/socketw/socket_server_tester.py,sha256=wAwyst8YdVyVfZfERav1A9_OnMJAiVBy-4uY0RpNqkU,6339
|
|
305
306
|
atomicshop/wrappers/socketw/socket_wrapper.py,sha256=g7f_8RkW80EZeQWNTqGYnfrQkgAI56T3SwWybq7ZsXg,28521
|
|
306
307
|
atomicshop/wrappers/socketw/ssl_base.py,sha256=k4V3gwkbq10MvOH4btU4onLX2GNOsSfUAdcHmL1rpVE,2274
|
|
307
308
|
atomicshop/wrappers/socketw/statistics_csv.py,sha256=Jc0D12crkKRaqoCRQ-2Mz1zm6n4UUx9dXakf-N2TYWA,3065
|
|
308
309
|
atomicshop/wrappers/winregw/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
309
310
|
atomicshop/wrappers/winregw/winreg_network.py,sha256=bQ8Jql8bVGBJ0dt3VQ56lga_1LBOMLI3Km_otvvbU6c,7138
|
|
310
|
-
atomicshop-2.16.
|
|
311
|
-
atomicshop-2.16.
|
|
312
|
-
atomicshop-2.16.
|
|
313
|
-
atomicshop-2.16.
|
|
314
|
-
atomicshop-2.16.
|
|
311
|
+
atomicshop-2.16.12.dist-info/LICENSE.txt,sha256=lLU7EYycfYcK2NR_1gfnhnRC8b8ccOTElACYplgZN88,1094
|
|
312
|
+
atomicshop-2.16.12.dist-info/METADATA,sha256=POvocbg-JIQO-oVd8Z3RIvANnUqSiQmXx0Q4A9vTvI0,10503
|
|
313
|
+
atomicshop-2.16.12.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
|
|
314
|
+
atomicshop-2.16.12.dist-info/top_level.txt,sha256=EgKJB-7xcrAPeqTRF2laD_Np2gNGYkJkd4OyXqpJphA,11
|
|
315
|
+
atomicshop-2.16.12.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|