atomicshop 2.12.16__py3-none-any.whl → 2.12.17__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/diff_check.py +95 -49
- atomicshop/monitor/change_monitor.py +77 -134
- atomicshop/monitor/checks/dns.py +100 -82
- atomicshop/monitor/checks/hash.py +39 -28
- atomicshop/monitor/checks/hash_checks/file.py +34 -35
- atomicshop/monitor/checks/hash_checks/url.py +41 -39
- atomicshop/monitor/checks/network.py +30 -28
- atomicshop/monitor/checks/process_running.py +7 -7
- atomicshop/timer.py +30 -11
- {atomicshop-2.12.16.dist-info → atomicshop-2.12.17.dist-info}/METADATA +1 -1
- {atomicshop-2.12.16.dist-info → atomicshop-2.12.17.dist-info}/RECORD +15 -15
- {atomicshop-2.12.16.dist-info → atomicshop-2.12.17.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.12.16.dist-info → atomicshop-2.12.17.dist-info}/WHEEL +0 -0
- {atomicshop-2.12.16.dist-info → atomicshop-2.12.17.dist-info}/top_level.txt +0 -0
atomicshop/monitor/checks/dns.py
CHANGED
|
@@ -1,108 +1,126 @@
|
|
|
1
|
+
from pathlib import Path
|
|
2
|
+
|
|
1
3
|
from ...etw.dns_trace import DnsTrace
|
|
2
4
|
from ...print_api import print_api
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
5
|
+
from ...diff_check import DiffChecker
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
INPUT_FILE_DEFAULT_NAME: str = 'known_domains.json'
|
|
9
|
+
INPUT_STATISTICS_FILE_DEFAULT_NAME: str = 'dns_statistics.json'
|
|
10
|
+
FETCH_ENGINE: DnsTrace = DnsTrace(enable_process_poller=True, attrs=['name', 'cmdline', 'domain', 'query_type'])
|
|
11
|
+
SETTINGS = {}
|
|
12
|
+
DIFF_CHECKER_AGGREGATION = DiffChecker(
|
|
13
|
+
check_object=list(), # DNS events will be appended to this list.
|
|
14
|
+
return_first_cycle=True,
|
|
15
|
+
operation_type='new_objects'
|
|
16
|
+
)
|
|
17
|
+
DIFF_CHECKER_STATISTICS = DiffChecker(
|
|
18
|
+
check_object=list(), # DNS events will be appended to this list.
|
|
19
|
+
return_first_cycle=True,
|
|
20
|
+
operation_type='hit_statistics',
|
|
21
|
+
hit_statistics_enable_queue=True
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def setup_check(change_monitor_instance):
|
|
26
|
+
global SETTINGS
|
|
27
|
+
SETTINGS = change_monitor_instance.object_type_settings
|
|
28
|
+
|
|
29
|
+
if SETTINGS['alert_always'] and SETTINGS['alert_about_missing_entries_after_learning']:
|
|
30
|
+
raise ValueError(
|
|
31
|
+
"ERROR: [alert_always] and [alert_about_missing_entries_after_learning] cannot be True at the same time.")
|
|
32
|
+
|
|
33
|
+
if not change_monitor_instance.input_file_name:
|
|
34
|
+
change_monitor_instance.input_file_name = INPUT_FILE_DEFAULT_NAME
|
|
35
|
+
input_file_path = (
|
|
36
|
+
str(Path(change_monitor_instance.input_directory, change_monitor_instance.input_file_name)))
|
|
37
|
+
|
|
38
|
+
if not change_monitor_instance.input_statistics_file_name:
|
|
39
|
+
change_monitor_instance.input_statistics_file_name = INPUT_STATISTICS_FILE_DEFAULT_NAME
|
|
40
|
+
input_statistic_file_path = (
|
|
41
|
+
str(Path(change_monitor_instance.input_directory, change_monitor_instance.input_statistics_file_name)))
|
|
42
|
+
|
|
43
|
+
if SETTINGS['learning_mode_create_unique_entries_list']:
|
|
44
|
+
DIFF_CHECKER_AGGREGATION.input_file_write_only = change_monitor_instance.input_file_write_only
|
|
45
|
+
DIFF_CHECKER_AGGREGATION.check_object_display_name = \
|
|
46
|
+
f'{change_monitor_instance.input_file_name}|{change_monitor_instance.object_type}'
|
|
47
|
+
DIFF_CHECKER_AGGREGATION.input_file_path = input_file_path
|
|
48
|
+
DIFF_CHECKER_AGGREGATION.new_objects_hours_then_difference = SETTINGS['learning_hours']
|
|
49
|
+
|
|
50
|
+
if SETTINGS['create_alert_statistics']:
|
|
51
|
+
DIFF_CHECKER_STATISTICS.input_file_write_only = change_monitor_instance.input_file_write_only
|
|
52
|
+
DIFF_CHECKER_STATISTICS.check_object_display_name = \
|
|
53
|
+
f'{change_monitor_instance.input_statistics_file_name}|{change_monitor_instance.object_type}'
|
|
54
|
+
DIFF_CHECKER_STATISTICS.input_file_path = input_statistic_file_path
|
|
55
|
+
DIFF_CHECKER_STATISTICS.hit_statistics_input_file_rotation_cycle_hours = SETTINGS['statistics_rotation_hours']
|
|
56
|
+
|
|
57
|
+
# Start DNS monitoring.
|
|
58
|
+
FETCH_ENGINE.start()
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def execute_cycle(print_kwargs: dict = None) -> list:
|
|
6
62
|
"""
|
|
7
63
|
This function executes the cycle of the change monitor: dns.
|
|
64
|
+
The function is blocking so while using it, the script will wait for the next DNS event.
|
|
65
|
+
No need to use 'time.sleep()'.
|
|
8
66
|
|
|
9
|
-
:param
|
|
10
|
-
|
|
67
|
+
:param print_kwargs: print_api kwargs.
|
|
11
68
|
:return: List of dictionaries with the results of the cycle.
|
|
12
69
|
"""
|
|
13
70
|
|
|
14
|
-
|
|
15
|
-
|
|
71
|
+
# 'emit()' method is blocking (it uses 'get' of queue instance)
|
|
72
|
+
# will return a dict with current DNS trace event.
|
|
73
|
+
event_dict = FETCH_ENGINE.emit()
|
|
16
74
|
|
|
17
|
-
|
|
18
|
-
|
|
75
|
+
return_list = list()
|
|
76
|
+
if SETTINGS['learning_mode_create_unique_entries_list']:
|
|
77
|
+
_aggregation_process(event_dict, return_list, print_kwargs)
|
|
19
78
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
enable_process_poller=True, attrs=['name', 'cmdline', 'domain', 'query_type'])
|
|
79
|
+
if SETTINGS['create_alert_statistics'] and SETTINGS['alert_always']:
|
|
80
|
+
_statistics_process(event_dict, return_list, print_kwargs)
|
|
23
81
|
|
|
24
|
-
|
|
25
|
-
change_monitor_instance.fetch_engine.start()
|
|
82
|
+
return return_list
|
|
26
83
|
|
|
27
|
-
# Change settings for the DiffChecker object.
|
|
28
|
-
change_monitor_instance.diff_check_list[0].return_first_cycle = True
|
|
29
|
-
# 'operation_type' is None.
|
|
30
|
-
if not change_monitor_instance.operation_type:
|
|
31
|
-
change_monitor_instance.diff_check_list[0].operation_type = 'new_objects'
|
|
32
84
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
# if change_monitor_instance.diff_check_list[0].operation_type == 'hit_statistics':
|
|
36
|
-
# change_monitor_instance.statistics_queue = change_monitor_instance.diff_check_list[0].statistics_queue
|
|
85
|
+
def _aggregation_process(event_dict: dict, return_list: list, print_kwargs: dict = None):
|
|
86
|
+
DIFF_CHECKER_AGGREGATION.check_object = [event_dict]
|
|
37
87
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
change_monitor_instance.input_file_name = f'{original_name}.json'
|
|
88
|
+
# Check if 'known_domains' list was updated from previous cycle.
|
|
89
|
+
result, message = DIFF_CHECKER_AGGREGATION.check_list_of_dicts(
|
|
90
|
+
sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)
|
|
42
91
|
|
|
43
|
-
|
|
44
|
-
|
|
92
|
+
if result:
|
|
93
|
+
# Check if 'updated' key is in the result. This means that this is a regular cycle.
|
|
94
|
+
if 'updated' in result:
|
|
95
|
+
if not result['time_passed']:
|
|
96
|
+
# Get list of new connections only.
|
|
97
|
+
# new_connections_only: list = list_of_dicts.get_difference(result['old'], result['updated'])
|
|
45
98
|
|
|
46
|
-
|
|
47
|
-
|
|
99
|
+
for connection in result['updated']:
|
|
100
|
+
message = \
|
|
101
|
+
f"[Learning] New Domain Added: {connection['name']} | " \
|
|
102
|
+
f"{connection['domain']} | {connection['query_type']} | " \
|
|
103
|
+
f"{connection['cmdline']}"
|
|
104
|
+
print_api(message, color='yellow', **(print_kwargs or {}))
|
|
48
105
|
|
|
49
|
-
|
|
50
|
-
change_monitor_instance._set_input_file_path()
|
|
106
|
+
return_list.append(message)
|
|
51
107
|
|
|
52
|
-
|
|
108
|
+
# Check if learning time passed, so we can alert the new entries.
|
|
109
|
+
if 'time_passed' in result:
|
|
110
|
+
if result['time_passed']:
|
|
111
|
+
_statistics_process(result['updated'], return_list, print_kwargs)
|
|
53
112
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
# If 'disable_diff_check' is True, we'll return the event_dict as is.
|
|
59
|
-
# if change_monitor_instance.disable_diff_check:
|
|
60
|
-
# return_list.append(event_dict)
|
|
61
|
-
#
|
|
62
|
-
# message = \
|
|
63
|
-
# (f"Current domain: {event_dict['name']} | {event_dict['domain']} | {event_dict['query_type']} | "
|
|
64
|
-
# f"{event_dict['cmdline']}")
|
|
65
|
-
# print_api(message, color='yellow', **print_kwargs)
|
|
66
|
-
#
|
|
67
|
-
# return return_list
|
|
68
|
-
# else:
|
|
69
|
-
change_monitor_instance.diff_check_list[0].check_object = [event_dict]
|
|
70
|
-
|
|
71
|
-
# if event_dict not in change_monitor_instance.diff_check_list[0].check_object:
|
|
72
|
-
# change_monitor_instance.diff_check_list[0].check_object.append(event_dict)
|
|
73
|
-
#
|
|
74
|
-
# # Sort list of dicts by process name and then by process cmdline.
|
|
75
|
-
# change_monitor_instance.diff_check_list[0].check_object = list_of_dicts.sort_by_keys(
|
|
76
|
-
# change_monitor_instance.diff_check_list[0].check_object, ['cmdline', 'name'], case_insensitive=True)
|
|
113
|
+
|
|
114
|
+
def _statistics_process(event_dict: dict, return_list: list, print_kwargs: dict = None):
|
|
115
|
+
DIFF_CHECKER_STATISTICS.check_object = [event_dict]
|
|
77
116
|
|
|
78
117
|
# Check if 'known_domains' list was updated from previous cycle.
|
|
79
|
-
result, message =
|
|
118
|
+
result, message = DIFF_CHECKER_STATISTICS.check_list_of_dicts(
|
|
80
119
|
sort_by_keys=['cmdline', 'name'], print_kwargs=print_kwargs)
|
|
81
120
|
|
|
82
121
|
if result:
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
# new_connections_only: list = list_of_dicts.get_difference(result['old'], result['updated'])
|
|
87
|
-
|
|
88
|
-
for connection in result['updated']:
|
|
89
|
-
message = \
|
|
90
|
-
f"New domain: {connection['name']} | " \
|
|
91
|
-
f"{connection['domain']} | {connection['query_type']} | " \
|
|
92
|
-
f"{connection['cmdline']}"
|
|
93
|
-
print_api(message, color='yellow', **print_kwargs)
|
|
94
|
-
|
|
95
|
-
return_list.append(message)
|
|
96
|
-
# Check if 'count' key is in the result. This means that this a statistics cycle.
|
|
97
|
-
elif 'count' in result:
|
|
98
|
-
message = f"Current domain: {result['entry']} | Times hit: {result['count']}"
|
|
99
|
-
print_api(message, color='yellow', **print_kwargs)
|
|
100
|
-
|
|
101
|
-
return_list.append(message)
|
|
102
|
-
elif 'count' not in result and 'entry' in result:
|
|
103
|
-
message = f"Current domain: {result['entry']}"
|
|
104
|
-
print_api(message, color='yellow', **print_kwargs)
|
|
122
|
+
if 'count' in result:
|
|
123
|
+
message = f"[Alert] DNS Request: {result['entry']} | Times hit: {result['count']}"
|
|
124
|
+
print_api(message, color='yellow', **(print_kwargs or {}))
|
|
105
125
|
|
|
106
126
|
return_list.append(message)
|
|
107
|
-
|
|
108
|
-
return return_list
|
|
@@ -2,11 +2,29 @@ from ...print_api import print_api
|
|
|
2
2
|
from .hash_checks import file, url
|
|
3
3
|
|
|
4
4
|
|
|
5
|
-
def
|
|
5
|
+
def setup_check(change_monitor_instance):
|
|
6
|
+
# if store_original_object and not (input_directory or input_file_path):
|
|
7
|
+
# raise ValueError(
|
|
8
|
+
# 'ERROR: if [store_original_object] is True, either '
|
|
9
|
+
# '[input_directory] or [input_file_path] must be specified .')
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
change_monitor_instance.set_input_file_path()
|
|
13
|
+
|
|
14
|
+
change_monitor_instance.diff_checker.operation_type = 'single_object'
|
|
15
|
+
|
|
16
|
+
if change_monitor_instance.object_type == 'file':
|
|
17
|
+
file.setup_check(change_monitor_instance, change_monitor_instance.check_object)
|
|
18
|
+
elif 'url_' in change_monitor_instance.object_type:
|
|
19
|
+
url.setup_check(change_monitor_instance, change_monitor_instance.check_object)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
6
23
|
"""
|
|
7
24
|
This function executes the cycle of the change monitor: hash.
|
|
8
25
|
|
|
9
26
|
:param change_monitor_instance: Instance of the ChangeMonitor class.
|
|
27
|
+
:param print_kwargs: print_api kwargs.
|
|
10
28
|
|
|
11
29
|
:return: List of dictionaries with the results of the cycle.
|
|
12
30
|
"""
|
|
@@ -15,32 +33,25 @@ def _execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
|
15
33
|
print_kwargs = dict()
|
|
16
34
|
|
|
17
35
|
return_list = list()
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
#
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
if result:
|
|
39
|
-
print_api(message, color='yellow', **print_kwargs)
|
|
40
|
-
# create_message_file(message, self.__class__.__name__, logger=self.logger)
|
|
41
|
-
|
|
42
|
-
return_list.append(message)
|
|
43
|
-
else:
|
|
44
|
-
print_api(message, color='green', **print_kwargs)
|
|
36
|
+
|
|
37
|
+
if change_monitor_instance.object_type == 'file':
|
|
38
|
+
# Get the hash of the object.
|
|
39
|
+
file.get_hash(change_monitor_instance, change_monitor_instance.check_object, print_kwargs=print_kwargs)
|
|
40
|
+
elif 'url_' in change_monitor_instance.object_type:
|
|
41
|
+
# Get the hash of the object.
|
|
42
|
+
url.get_hash(change_monitor_instance, change_monitor_instance.check_object, print_kwargs=print_kwargs)
|
|
43
|
+
|
|
44
|
+
# Check if the object was updated.
|
|
45
|
+
result, message = change_monitor_instance.diff_checker.check_string(
|
|
46
|
+
print_kwargs=print_kwargs)
|
|
47
|
+
|
|
48
|
+
# If the object was updated, print the message in yellow color, otherwise print in green color.
|
|
49
|
+
if result:
|
|
50
|
+
print_api(message, color='yellow', **print_kwargs)
|
|
51
|
+
# create_message_file(message, self.__class__.__name__, logger=self.logger)
|
|
52
|
+
|
|
53
|
+
return_list.append(message)
|
|
54
|
+
else:
|
|
55
|
+
print_api(message, color='green', **print_kwargs)
|
|
45
56
|
|
|
46
57
|
return return_list
|
|
@@ -1,48 +1,47 @@
|
|
|
1
1
|
from .... import filesystem, hashing
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
def
|
|
5
|
-
|
|
6
|
-
The function will get the hash of the URL content.
|
|
4
|
+
def setup_check(change_monitor_instance, check_object: str):
|
|
5
|
+
original_name: str = str()
|
|
7
6
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
7
|
+
# If 'generate_input_file_name' is True, or 'store_original_object' is True, we need to create a
|
|
8
|
+
# filename without extension.
|
|
9
|
+
if change_monitor_instance.store_original_object or change_monitor_instance.generate_input_file_name:
|
|
10
|
+
# Get the last directory from the url.
|
|
11
|
+
original_name = filesystem.get_file_name(check_object)
|
|
12
|
+
# Make characters lower case.
|
|
13
|
+
original_name = original_name.lower()
|
|
13
14
|
|
|
14
|
-
# If
|
|
15
|
-
if change_monitor_instance.
|
|
16
|
-
|
|
15
|
+
# If 'store_original_object' is True, then we need to create a filepath to store.
|
|
16
|
+
if change_monitor_instance.original_object_directory:
|
|
17
|
+
# Add extension to the file name.
|
|
18
|
+
original_file_name = original_name
|
|
17
19
|
|
|
18
|
-
#
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
# Get the last directory from the url.
|
|
22
|
-
original_name = filesystem.get_file_name(check_object)
|
|
23
|
-
# Make characters lower case.
|
|
24
|
-
original_name = original_name.lower()
|
|
20
|
+
# Make path for original object.
|
|
21
|
+
change_monitor_instance.original_object_file_path = filesystem.add_object_to_path(
|
|
22
|
+
change_monitor_instance.original_object_directory, original_file_name)
|
|
25
23
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
24
|
+
if change_monitor_instance.generate_input_file_name:
|
|
25
|
+
# Remove dots from the file name.
|
|
26
|
+
original_name_no_dots = original_name.replace('.', '-')
|
|
27
|
+
# Make path for 'input_file_name'.
|
|
28
|
+
change_monitor_instance.input_file_name = f'{original_name_no_dots}.txt'
|
|
30
29
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
change_monitor_instance.original_object_directory, original_file_name)
|
|
30
|
+
# Change settings for the DiffChecker object.
|
|
31
|
+
change_monitor_instance.diff_checker.return_first_cycle = False
|
|
34
32
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
original_name_no_dots = original_name.replace('.', '-')
|
|
38
|
-
# Make path for 'input_file_name'.
|
|
39
|
-
change_monitor_instance.input_file_name = f'{original_name_no_dots}.txt'
|
|
33
|
+
change_monitor_instance.diff_checker.check_object_display_name = \
|
|
34
|
+
f'{original_name}|{change_monitor_instance.object_type}'
|
|
40
35
|
|
|
41
|
-
# Change settings for the DiffChecker object.
|
|
42
|
-
change_monitor_instance.diff_check_list[check_object_index].return_first_cycle = False
|
|
43
36
|
|
|
44
|
-
|
|
45
|
-
|
|
37
|
+
def get_hash(change_monitor_instance, check_object: str, print_kwargs: dict = None):
|
|
38
|
+
"""
|
|
39
|
+
The function will get the hash of the URL content.
|
|
40
|
+
|
|
41
|
+
:param change_monitor_instance: Instance of the ChangeMonitor class.
|
|
42
|
+
:param check_object: string, full URL to a web page.
|
|
43
|
+
:param print_kwargs: dict, that contains all the arguments for 'print_api' function.
|
|
44
|
+
"""
|
|
46
45
|
|
|
47
46
|
# Copy the file to the original object directory.
|
|
48
47
|
if change_monitor_instance.original_object_file_path:
|
|
@@ -52,4 +51,4 @@ def get_hash(change_monitor_instance, check_object_index: int, check_object: str
|
|
|
52
51
|
hash_string = hashing.hash_file(check_object)
|
|
53
52
|
|
|
54
53
|
# Set the hash string to the 'check_object' variable.
|
|
55
|
-
change_monitor_instance.
|
|
54
|
+
change_monitor_instance.diff_checker.check_object = hash_string
|
|
@@ -1,54 +1,56 @@
|
|
|
1
1
|
from .... import filesystem, hashing, urls
|
|
2
2
|
|
|
3
3
|
|
|
4
|
-
def
|
|
4
|
+
def setup_check(change_monitor_instance, check_object: str):
|
|
5
|
+
# Extract the method name from the object type.
|
|
6
|
+
get_method = change_monitor_instance.object_type.split('_', 1)[1]
|
|
7
|
+
|
|
8
|
+
original_name: str = str()
|
|
9
|
+
|
|
10
|
+
# If 'generate_input_file_name' is True, or 'store_original_object' is True, we need to create a
|
|
11
|
+
# filename without extension.
|
|
12
|
+
if change_monitor_instance.store_original_object or change_monitor_instance.generate_input_file_name:
|
|
13
|
+
# Get the last directory from the url.
|
|
14
|
+
original_name = urls.url_parser(check_object)['directories'][-1]
|
|
15
|
+
# Make characters lower case.
|
|
16
|
+
original_name = original_name.lower()
|
|
17
|
+
|
|
18
|
+
# If 'store_original_object' is True, then we need to create a filepath to store.
|
|
19
|
+
extension = None
|
|
20
|
+
if change_monitor_instance.original_object_directory:
|
|
21
|
+
# Add extension to the file name.
|
|
22
|
+
if 'playwright' in get_method:
|
|
23
|
+
extension = get_method.split('_')[1]
|
|
24
|
+
elif get_method == 'urllib':
|
|
25
|
+
extension = 'html'
|
|
26
|
+
original_file_name = f'{original_name}.{extension}'
|
|
27
|
+
|
|
28
|
+
# Make path for original object.
|
|
29
|
+
change_monitor_instance.original_object_file_path = filesystem.add_object_to_path(
|
|
30
|
+
change_monitor_instance.original_object_directory, original_file_name)
|
|
31
|
+
|
|
32
|
+
if change_monitor_instance.generate_input_file_name:
|
|
33
|
+
# Make path for 'input_file_name'.
|
|
34
|
+
change_monitor_instance.input_file_name = f'{original_name}.txt'
|
|
35
|
+
|
|
36
|
+
# Change settings for the DiffChecker object.
|
|
37
|
+
change_monitor_instance.diff_checker.return_first_cycle = False
|
|
38
|
+
|
|
39
|
+
change_monitor_instance.diff_checker.check_object_display_name = \
|
|
40
|
+
f'{original_name}|{change_monitor_instance.object_type}'
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def get_hash(change_monitor_instance, check_object: str, print_kwargs: dict = None):
|
|
5
44
|
"""
|
|
6
45
|
The function will get the hash of the URL content.
|
|
7
46
|
|
|
8
47
|
:param change_monitor_instance: Instance of the ChangeMonitor class.
|
|
9
|
-
:param check_object_index: integer, index of the object in the 'check_object_list' list.
|
|
10
48
|
:param check_object: string, full URL to a web page.
|
|
11
49
|
:param print_kwargs: dict, that contains all the arguments for 'print_api' function.
|
|
12
50
|
"""
|
|
13
51
|
# Extract the method name from the object type.
|
|
14
52
|
get_method = change_monitor_instance.object_type.split('_', 1)[1]
|
|
15
53
|
|
|
16
|
-
# If this is the first cycle, we need to set several things.
|
|
17
|
-
if change_monitor_instance.first_cycle:
|
|
18
|
-
original_name: str = str()
|
|
19
|
-
|
|
20
|
-
# If 'generate_input_file_name' is True, or 'store_original_object' is True, we need to create a
|
|
21
|
-
# filename without extension.
|
|
22
|
-
if change_monitor_instance.store_original_object or change_monitor_instance.generate_input_file_name:
|
|
23
|
-
# Get the last directory from the url.
|
|
24
|
-
original_name = urls.url_parser(check_object)['directories'][-1]
|
|
25
|
-
# Make characters lower case.
|
|
26
|
-
original_name = original_name.lower()
|
|
27
|
-
|
|
28
|
-
# If 'store_original_object' is True, then we need to create a filepath to store.
|
|
29
|
-
extension = None
|
|
30
|
-
if change_monitor_instance.original_object_directory:
|
|
31
|
-
# Add extension to the file name.
|
|
32
|
-
if 'playwright' in get_method:
|
|
33
|
-
extension = get_method.split('_')[1]
|
|
34
|
-
elif get_method == 'urllib':
|
|
35
|
-
extension = 'html'
|
|
36
|
-
original_file_name = f'{original_name}.{extension}'
|
|
37
|
-
|
|
38
|
-
# Make path for original object.
|
|
39
|
-
change_monitor_instance.original_object_file_path = filesystem.add_object_to_path(
|
|
40
|
-
change_monitor_instance.original_object_directory, original_file_name)
|
|
41
|
-
|
|
42
|
-
if change_monitor_instance.generate_input_file_name:
|
|
43
|
-
# Make path for 'input_file_name'.
|
|
44
|
-
change_monitor_instance.input_file_name = f'{original_name}.txt'
|
|
45
|
-
|
|
46
|
-
# Change settings for the DiffChecker object.
|
|
47
|
-
change_monitor_instance.diff_check_list[check_object_index].return_first_cycle = False
|
|
48
|
-
|
|
49
|
-
change_monitor_instance.diff_check_list[check_object_index].check_object_display_name = \
|
|
50
|
-
f'{original_name}|{change_monitor_instance.object_type}'
|
|
51
|
-
|
|
52
54
|
# Get hash of the url. The hash will be different between direct hash of the URL content and the
|
|
53
55
|
# hash of the file that was downloaded from the URL. Since the file has headers and other information
|
|
54
56
|
# that is not part of the URL content. The Original downloaded file is for reference only to see
|
|
@@ -59,4 +61,4 @@ def get_hash(change_monitor_instance, check_object_index: int, check_object: str
|
|
|
59
61
|
)
|
|
60
62
|
|
|
61
63
|
# Set the hash string to the 'check_object' variable.
|
|
62
|
-
change_monitor_instance.
|
|
64
|
+
change_monitor_instance.diff_checker.check_object = hash_string
|
|
@@ -3,11 +3,35 @@ from ...basics import list_of_dicts
|
|
|
3
3
|
from ...print_api import print_api
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def
|
|
6
|
+
def setup_check(change_monitor_instance):
|
|
7
|
+
original_name: str = str()
|
|
8
|
+
|
|
9
|
+
# Initialize objects for network monitoring.
|
|
10
|
+
change_monitor_instance.fetch_engine = psutilw.PsutilConnections()
|
|
11
|
+
|
|
12
|
+
# Change settings for the DiffChecker object.
|
|
13
|
+
change_monitor_instance.diff_checker.return_first_cycle = True
|
|
14
|
+
|
|
15
|
+
if change_monitor_instance.generate_input_file_name:
|
|
16
|
+
original_name = 'known_connections'
|
|
17
|
+
# Make path for 'input_file_name'.
|
|
18
|
+
change_monitor_instance.input_file_name = f'{original_name}.txt'
|
|
19
|
+
|
|
20
|
+
change_monitor_instance.diff_checker.check_object_display_name = \
|
|
21
|
+
f'{original_name}|{change_monitor_instance.object_type}'
|
|
22
|
+
|
|
23
|
+
# Set the 'check_object' to empty list, since we will append the list of DNS events.
|
|
24
|
+
change_monitor_instance.diff_checker.check_object = list()
|
|
25
|
+
|
|
26
|
+
change_monitor_instance.diff_checker.operation_type = 'single_object'
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
7
30
|
"""
|
|
8
31
|
This function executes the cycle of the change monitor: network.
|
|
9
32
|
|
|
10
33
|
:param change_monitor_instance: Instance of the ChangeMonitor class.
|
|
34
|
+
:param print_kwargs: print_api kwargs.
|
|
11
35
|
|
|
12
36
|
:return: List of dictionaries with the results of the cycle.
|
|
13
37
|
"""
|
|
@@ -22,7 +46,7 @@ def _execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
|
22
46
|
change_monitor_instance._set_input_file_path()
|
|
23
47
|
|
|
24
48
|
# Check if 'known_domains' list was updated from previous cycle.
|
|
25
|
-
result, message = change_monitor_instance.
|
|
49
|
+
result, message = change_monitor_instance.diff_checker.check_list_of_dicts(print_kwargs=print_kwargs)
|
|
26
50
|
|
|
27
51
|
if result:
|
|
28
52
|
# Get list of new connections only.
|
|
@@ -50,28 +74,6 @@ def _get_list(change_monitor_instance):
|
|
|
50
74
|
:return: list of dicts, of new network sockets.
|
|
51
75
|
"""
|
|
52
76
|
|
|
53
|
-
if change_monitor_instance.first_cycle:
|
|
54
|
-
original_name: str = str()
|
|
55
|
-
|
|
56
|
-
# Initialize objects for network monitoring.
|
|
57
|
-
change_monitor_instance.fetch_engine = psutilw.PsutilConnections()
|
|
58
|
-
|
|
59
|
-
# Change settings for the DiffChecker object.
|
|
60
|
-
change_monitor_instance.diff_check_list[0].return_first_cycle = True
|
|
61
|
-
|
|
62
|
-
if change_monitor_instance.generate_input_file_name:
|
|
63
|
-
original_name = 'known_connections'
|
|
64
|
-
# Make path for 'input_file_name'.
|
|
65
|
-
change_monitor_instance.input_file_name = f'{original_name}.txt'
|
|
66
|
-
|
|
67
|
-
change_monitor_instance.diff_check_list[0].check_object_display_name = \
|
|
68
|
-
f'{original_name}|{change_monitor_instance.object_type}'
|
|
69
|
-
|
|
70
|
-
# Set the 'check_object' to empty list, since we will append the list of DNS events.
|
|
71
|
-
change_monitor_instance.diff_check_list[0].check_object = list()
|
|
72
|
-
|
|
73
|
-
change_monitor_instance.diff_check_list[0].operation_type = 'single_object'
|
|
74
|
-
|
|
75
77
|
# Get all connections (list of dicts), including process name and cmdline.
|
|
76
78
|
connections_list_of_dicts: list = \
|
|
77
79
|
change_monitor_instance.fetch_engine.get_connections_with_process_as_list_of_dicts(
|
|
@@ -80,10 +82,10 @@ def _get_list(change_monitor_instance):
|
|
|
80
82
|
|
|
81
83
|
# Get list of connections that are not in 'known_connections' list.
|
|
82
84
|
missing_connections_from_cycle: list = list_of_dicts.get_difference(
|
|
83
|
-
change_monitor_instance.
|
|
85
|
+
change_monitor_instance.diff_checker.check_object, connections_list_of_dicts)
|
|
84
86
|
# Add missing new connections to 'known_connections' list.
|
|
85
|
-
change_monitor_instance.
|
|
87
|
+
change_monitor_instance.diff_checker.check_object.extend(missing_connections_from_cycle)
|
|
86
88
|
|
|
87
89
|
# Sort list of dicts by process name and then by process cmdline.
|
|
88
|
-
change_monitor_instance.
|
|
89
|
-
change_monitor_instance.
|
|
90
|
+
change_monitor_instance.diff_checker.check_object = list_of_dicts.sort_by_keys(
|
|
91
|
+
change_monitor_instance.diff_checker.check_object, key_list=['cmdline', 'name'], case_insensitive=True)
|
|
@@ -3,12 +3,16 @@ from ...basics import list_of_dicts
|
|
|
3
3
|
from ...print_api import print_api
|
|
4
4
|
|
|
5
5
|
|
|
6
|
-
def
|
|
6
|
+
def setup_check(change_monitor_instance):
|
|
7
|
+
change_monitor_instance.fetch_engine = psutilw.PsutilProcesses()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
7
11
|
"""
|
|
8
12
|
This function executes the cycle of the change monitor: process_running.
|
|
9
13
|
|
|
10
14
|
:param change_monitor_instance: Instance of the ChangeMonitor class.
|
|
11
|
-
|
|
15
|
+
:param print_kwargs: Dictionary with the print arguments.
|
|
12
16
|
:return: List of dictionaries with the results of the cycle.
|
|
13
17
|
"""
|
|
14
18
|
|
|
@@ -19,7 +23,7 @@ def _execute_cycle(change_monitor_instance, print_kwargs: dict = None):
|
|
|
19
23
|
|
|
20
24
|
processes = _get_list(change_monitor_instance)
|
|
21
25
|
|
|
22
|
-
for process_name in change_monitor_instance.
|
|
26
|
+
for process_name in change_monitor_instance.check_object:
|
|
23
27
|
result = list_of_dicts.is_value_exist_in_key(processes, 'cmdline', process_name, value_case_insensitive=True)
|
|
24
28
|
|
|
25
29
|
# If the process name was found in the list of currently running processes.
|
|
@@ -45,9 +49,5 @@ def _get_list(change_monitor_instance):
|
|
|
45
49
|
:return: list of dicts, of new network sockets.
|
|
46
50
|
"""
|
|
47
51
|
|
|
48
|
-
if change_monitor_instance.first_cycle:
|
|
49
|
-
# Initialize objects for network monitoring.
|
|
50
|
-
change_monitor_instance.fetch_engine = psutilw.PsutilProcesses()
|
|
51
|
-
|
|
52
52
|
return change_monitor_instance.fetch_engine.get_processes_as_list_of_dicts(
|
|
53
53
|
default_keys=True, cmdline_to_string=True)
|