atomicshop 2.17.3__py3-none-any.whl → 2.18.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/basics/ansi_escape_codes.py +3 -1
- atomicshop/file_io/docxs.py +26 -17
- atomicshop/http_parse.py +118 -77
- atomicshop/mitm/config_static.py +1 -1
- atomicshop/mitm/connection_thread_worker.py +387 -257
- atomicshop/mitm/engines/__parent/parser___parent.py +2 -2
- atomicshop/mitm/engines/__parent/recorder___parent.py +83 -24
- atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +2 -2
- atomicshop/mitm/message.py +42 -15
- atomicshop/mitm/mitm_main.py +3 -2
- atomicshop/mitm/recs_files.py +2 -1
- atomicshop/system_resource_monitor.py +16 -3
- atomicshop/wrappers/socketw/base.py +17 -0
- atomicshop/wrappers/socketw/receiver.py +90 -100
- atomicshop/wrappers/socketw/sender.py +5 -8
- atomicshop/wrappers/socketw/sni.py +0 -1
- atomicshop/wrappers/socketw/socket_client.py +2 -2
- atomicshop/wrappers/socketw/statistics_csv.py +15 -6
- {atomicshop-2.17.3.dist-info → atomicshop-2.18.1.dist-info}/METADATA +1 -1
- {atomicshop-2.17.3.dist-info → atomicshop-2.18.1.dist-info}/RECORD +24 -24
- {atomicshop-2.17.3.dist-info → atomicshop-2.18.1.dist-info}/LICENSE.txt +0 -0
- {atomicshop-2.17.3.dist-info → atomicshop-2.18.1.dist-info}/WHEEL +0 -0
- {atomicshop-2.17.3.dist-info → atomicshop-2.18.1.dist-info}/top_level.txt +0 -0
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
from datetime import datetime
|
|
2
|
+
import threading
|
|
3
|
+
import queue
|
|
2
4
|
|
|
3
5
|
from ..wrappers.socketw import receiver, sender, socket_client, base
|
|
4
6
|
from .. import websocket_parse
|
|
@@ -23,47 +25,45 @@ def thread_worker_main(
|
|
|
23
25
|
engines_list,
|
|
24
26
|
reference_module
|
|
25
27
|
):
|
|
26
|
-
def output_statistics_csv_row():
|
|
28
|
+
def output_statistics_csv_row(client_message: ClientMessage):
|
|
27
29
|
# If there is no '.code' attribute in HTTPResponse, this means that this is not an HTTP message, so there is no
|
|
28
30
|
# status code.
|
|
29
31
|
try:
|
|
30
|
-
http_status_code: str =
|
|
32
|
+
http_status_code: str = str(client_message.response_auto_parsed.code)
|
|
31
33
|
except AttributeError:
|
|
32
34
|
http_status_code: str = str()
|
|
33
35
|
|
|
34
36
|
# Same goes for the '.path' attribute, if it is not HTTP message then there will be no path.
|
|
35
37
|
try:
|
|
36
|
-
http_path: str = client_message.
|
|
38
|
+
http_path: str = client_message.response_auto_parsed.path
|
|
37
39
|
except AttributeError:
|
|
38
40
|
http_path: str = str()
|
|
39
41
|
|
|
40
42
|
# Same goes for the '.command' attribute, if it is not HTTP message then there will be no command.
|
|
41
43
|
try:
|
|
42
|
-
http_command: str = client_message.
|
|
44
|
+
http_command: str = client_message.response_auto_parsed.command
|
|
43
45
|
except AttributeError:
|
|
44
46
|
http_command: str = str()
|
|
45
47
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
else:
|
|
51
|
-
response_size_bytes += str(len(response))
|
|
52
|
-
|
|
53
|
-
# If it is not the last entry, add the comma.
|
|
54
|
-
if response_index + 1 != len(client_message.response_list_of_raw_bytes):
|
|
55
|
-
response_size_bytes += ','
|
|
56
|
-
|
|
57
|
-
# response_size_bytes = ','.join([str(len(x)) for x in client_message.response_list_of_raw_bytes])
|
|
48
|
+
if client_message.request_raw_bytes is None:
|
|
49
|
+
request_size_bytes = ''
|
|
50
|
+
else:
|
|
51
|
+
request_size_bytes = str(len(client_message.request_raw_bytes))
|
|
58
52
|
|
|
59
|
-
if
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
53
|
+
if client_message.response_raw_bytes is None:
|
|
54
|
+
response_size_bytes = ''
|
|
55
|
+
else:
|
|
56
|
+
response_size_bytes = str(len(client_message.response_raw_bytes))
|
|
57
|
+
|
|
58
|
+
if client_message.errors and len(client_message.errors) > 1:
|
|
59
|
+
error_string = '||'.join(client_message.errors)
|
|
60
|
+
error_string = f'Error count: {len(client_message.errors)} | Errors: {error_string}'
|
|
61
|
+
elif client_message.errors and len(client_message.errors) == 1:
|
|
62
|
+
error_string = client_message.errors[0]
|
|
63
|
+
elif not client_message.errors:
|
|
64
64
|
error_string = str()
|
|
65
65
|
else:
|
|
66
|
-
raise ValueError(f"Error in statistics error list. Values: {
|
|
66
|
+
raise ValueError(f"Error in statistics error list. Values: {client_message.errors}")
|
|
67
67
|
|
|
68
68
|
statistics_writer.write_row(
|
|
69
69
|
thread_id=str(thread_id),
|
|
@@ -71,58 +71,78 @@ def thread_worker_main(
|
|
|
71
71
|
tls_type=tls_type,
|
|
72
72
|
tls_version=tls_version,
|
|
73
73
|
protocol=client_message.protocol,
|
|
74
|
+
protocol2=client_message.protocol2,
|
|
75
|
+
protocol3=client_message.protocol3,
|
|
74
76
|
path=http_path,
|
|
75
77
|
status_code=http_status_code,
|
|
76
78
|
command=http_command,
|
|
77
|
-
|
|
78
|
-
request_size_bytes=
|
|
79
|
+
timestamp=client_message.timestamp,
|
|
80
|
+
request_size_bytes=request_size_bytes,
|
|
79
81
|
response_size_bytes=response_size_bytes,
|
|
80
82
|
recorded_file_path=client_message.recorded_file_path,
|
|
81
83
|
process_cmd=process_commandline,
|
|
84
|
+
action=client_message.action,
|
|
82
85
|
error=error_string
|
|
83
86
|
)
|
|
84
87
|
|
|
85
|
-
def record_and_statistics_write():
|
|
88
|
+
def record_and_statistics_write(client_message: ClientMessage):
|
|
86
89
|
# If recorder wasn't executed before, then execute it now
|
|
87
90
|
if config_static.LogRec.enable_request_response_recordings_in_logs:
|
|
88
|
-
recorded_file = recorder(
|
|
89
|
-
class_client_message=client_message, record_path=config_static.LogRec.recordings_path).record()
|
|
91
|
+
recorded_file = recorder.record(class_client_message=client_message)
|
|
90
92
|
client_message.recorded_file_path = recorded_file
|
|
91
93
|
|
|
92
94
|
# Save statistics file.
|
|
93
|
-
output_statistics_csv_row()
|
|
95
|
+
output_statistics_csv_row(client_message)
|
|
94
96
|
|
|
95
|
-
def parse_http(
|
|
96
|
-
|
|
97
|
+
def parse_http(
|
|
98
|
+
raw_bytes: bytes,
|
|
99
|
+
client_message: ClientMessage):
|
|
97
100
|
nonlocal protocol
|
|
101
|
+
nonlocal protocol3
|
|
102
|
+
|
|
98
103
|
# Parsing the raw bytes as HTTP.
|
|
99
|
-
|
|
100
|
-
HTTPRequestParse(
|
|
104
|
+
request_http_parsed, is_http_request, request_parsing_error = (
|
|
105
|
+
HTTPRequestParse(raw_bytes).parse())
|
|
106
|
+
|
|
107
|
+
response_http_parsed, is_http_response, response_parsing_error = (
|
|
108
|
+
HTTPResponseParse(raw_bytes).parse())
|
|
101
109
|
|
|
102
110
|
if is_http_request:
|
|
103
111
|
if protocol == '':
|
|
104
112
|
protocol = 'HTTP'
|
|
105
113
|
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
114
|
+
auto_parsed = request_http_parsed
|
|
115
|
+
network_logger.info(
|
|
116
|
+
f"HTTP Request Parsed: Method: {request_http_parsed.command} | Path: {request_http_parsed.path}")
|
|
117
|
+
|
|
118
|
+
is_http_request_a_websocket(auto_parsed, client_message)
|
|
119
|
+
elif is_http_response:
|
|
120
|
+
auto_parsed = response_http_parsed
|
|
121
|
+
network_logger.info(
|
|
122
|
+
f"HTTP Response Parsed: Status: {response_http_parsed.code}")
|
|
123
|
+
protocol3 = auto_parsed.headers.get('Sec-WebSocket-Protocol', None)
|
|
124
|
+
if protocol3:
|
|
125
|
+
client_message.protocol3 = protocol3
|
|
126
|
+
elif protocol == 'Websocket':
|
|
127
|
+
client_message.protocol2 = 'Frame'
|
|
128
|
+
auto_parsed = parse_websocket(raw_bytes)
|
|
129
|
+
if protocol3:
|
|
130
|
+
client_message.protocol3 = protocol3
|
|
109
131
|
else:
|
|
110
|
-
|
|
111
|
-
# not to log it into statistics.
|
|
112
|
-
# statistics_error_list.append(error_message)
|
|
113
|
-
print_api(request_parsing_error, logger=network_logger, logger_method='error', color='yellow')
|
|
132
|
+
auto_parsed = None
|
|
114
133
|
|
|
115
|
-
|
|
134
|
+
return auto_parsed
|
|
116
135
|
|
|
117
|
-
def is_http_request_a_websocket(
|
|
136
|
+
def is_http_request_a_websocket(
|
|
137
|
+
auto_parsed,
|
|
138
|
+
client_message: ClientMessage):
|
|
118
139
|
nonlocal protocol
|
|
119
140
|
|
|
120
141
|
if protocol == 'HTTP':
|
|
121
|
-
if (
|
|
122
|
-
|
|
123
|
-
'Upgrade' in client_message.request_raw_decoded.headers):
|
|
124
|
-
if client_message.request_raw_decoded.headers['Upgrade'] == 'websocket':
|
|
142
|
+
if auto_parsed and hasattr(auto_parsed, 'headers') and 'Upgrade' in auto_parsed.headers:
|
|
143
|
+
if auto_parsed.headers['Upgrade'] == 'websocket':
|
|
125
144
|
protocol = 'Websocket'
|
|
145
|
+
client_message.protocol2 = 'Handshake'
|
|
126
146
|
|
|
127
147
|
network_logger.info(f'Protocol upgraded to Websocket')
|
|
128
148
|
|
|
@@ -132,72 +152,40 @@ def thread_worker_main(
|
|
|
132
152
|
def finish_thread():
|
|
133
153
|
# At this stage there could be several times that the same socket was used to the service server - we need to
|
|
134
154
|
# close this socket as well if it still opened.
|
|
135
|
-
if
|
|
155
|
+
# The first part of the condition is to check if the service socket was connected at all.
|
|
156
|
+
# If the service socket couldn't connect, then the instance will be None.
|
|
157
|
+
if service_socket_instance and service_socket_instance.fileno() != -1:
|
|
136
158
|
if service_client.socket_instance:
|
|
137
159
|
service_client.close_socket()
|
|
138
160
|
|
|
139
161
|
# If client socket is still opened - close
|
|
140
|
-
if client_socket:
|
|
162
|
+
if client_socket.fileno() != -1:
|
|
141
163
|
client_socket.close()
|
|
142
|
-
network_logger.info(f"Closed client socket [{
|
|
164
|
+
network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
|
|
143
165
|
|
|
144
166
|
network_logger.info("Thread Finished. Will continue listening on the Main thread")
|
|
145
167
|
|
|
146
|
-
def
|
|
147
|
-
"""
|
|
148
|
-
Process the client raw data request.
|
|
149
|
-
|
|
150
|
-
:return: True if the socket should be closed, False if not.
|
|
151
|
-
"""
|
|
152
|
-
|
|
153
|
-
# If the message is empty, then the connection was closed already by the other side,
|
|
154
|
-
# so we can close the socket as well.
|
|
155
|
-
# If the received message from the client is not empty, then continue.
|
|
156
|
-
if not client_received_raw_data:
|
|
157
|
-
return True
|
|
158
|
-
|
|
159
|
-
# Putting the received message to the aggregating message class.
|
|
160
|
-
client_message.request_raw_bytes = client_received_raw_data
|
|
161
|
-
|
|
162
|
-
parse_http()
|
|
163
|
-
if protocol != '':
|
|
164
|
-
client_message.protocol = protocol
|
|
165
|
-
|
|
166
|
-
# Parse websocket frames only if it is not the first protocol upgrade request.
|
|
167
|
-
if protocol == 'Websocket' and cycle_count != 0:
|
|
168
|
-
client_message.request_raw_decoded = parse_websocket(client_message.request_raw_bytes)
|
|
169
|
-
|
|
170
|
-
# Custom parser, should parse HTTP body or the whole message if not HTTP.
|
|
171
|
-
parser_instance = parser(client_message)
|
|
172
|
-
parser_instance.parse()
|
|
173
|
-
|
|
174
|
-
# Converting body parsed to string on logging, since there is no strict rule for the parameter
|
|
175
|
-
# to be string.
|
|
176
|
-
parser_instance.logger.info(f"{str(client_message.request_body_parsed)[0: 100]}...")
|
|
177
|
-
|
|
178
|
-
return False
|
|
179
|
-
|
|
180
|
-
def create_responder_response():
|
|
168
|
+
def create_responder_response(client_message: ClientMessage) -> list[bytes]:
|
|
181
169
|
# Since we're in response mode, we'll record the request anyway, after the responder did its job.
|
|
182
170
|
client_message.info = "In Server Response Mode"
|
|
183
171
|
|
|
184
|
-
# Re-initiate the 'client_message.response_list_of_raw_bytes' list, since we'll be appending
|
|
185
|
-
# new entries for empty list.
|
|
186
|
-
client_message.response_list_of_raw_bytes = list()
|
|
187
|
-
|
|
188
172
|
# If it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
|
|
189
173
|
# response automatically.
|
|
190
|
-
if protocol == 'Websocket' and
|
|
191
|
-
|
|
174
|
+
if protocol == 'Websocket' and client_receive_count == 0:
|
|
175
|
+
responses: list = list()
|
|
176
|
+
responses.append(
|
|
192
177
|
websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
|
|
193
|
-
|
|
194
|
-
|
|
178
|
+
else:
|
|
179
|
+
# Creating response for parsed message and printing
|
|
180
|
+
responses: list = responder.create_response(client_message)
|
|
195
181
|
|
|
196
182
|
# Output first 100 characters of all the responses in the list.
|
|
197
|
-
for response_raw_bytes_single in
|
|
183
|
+
for response_raw_bytes_single in responses:
|
|
198
184
|
responder.logger.info(f"{response_raw_bytes_single[0: 100]}...")
|
|
199
185
|
|
|
200
|
-
|
|
186
|
+
return responses
|
|
187
|
+
|
|
188
|
+
def create_client_socket(client_message: ClientMessage):
|
|
201
189
|
# If there is a custom certificate for the client for this domain, then we'll use it.
|
|
202
190
|
# noinspection PyTypeChecker
|
|
203
191
|
custom_client_pem_certificate_path: str = None
|
|
@@ -234,59 +222,258 @@ def thread_worker_main(
|
|
|
234
222
|
|
|
235
223
|
return service_client_instance
|
|
236
224
|
|
|
237
|
-
def
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
225
|
+
def process_client_raw_data(
|
|
226
|
+
client_received_raw_data: bytes,
|
|
227
|
+
error_string: str,
|
|
228
|
+
client_message: ClientMessage):
|
|
229
|
+
"""
|
|
230
|
+
Process the client raw data request.
|
|
231
|
+
"""
|
|
232
|
+
nonlocal protocol
|
|
233
|
+
|
|
234
|
+
client_message.request_raw_bytes = client_received_raw_data
|
|
235
|
+
|
|
236
|
+
if error_string:
|
|
237
|
+
client_message.errors.append(error_string)
|
|
238
|
+
|
|
239
|
+
if client_received_raw_data == b'':
|
|
240
|
+
return
|
|
241
|
+
|
|
242
|
+
client_message.response_auto_parsed = parse_http(client_message.request_raw_bytes, client_message)
|
|
243
|
+
if protocol != '':
|
|
244
|
+
client_message.protocol = protocol
|
|
245
|
+
|
|
246
|
+
# Parse websocket frames only if it is not the first protocol upgrade request.
|
|
247
|
+
if protocol == 'Websocket' and client_receive_count != 0:
|
|
248
|
+
client_message.request_auto_parsed = parse_websocket(client_message.request_raw_bytes)
|
|
249
|
+
|
|
250
|
+
# Custom parser, should parse HTTP body or the whole message if not HTTP.
|
|
251
|
+
parser_instance = parser(client_message)
|
|
252
|
+
parser_instance.parse()
|
|
253
|
+
|
|
254
|
+
# Converting body parsed to string on logging, there is no strict rule for the parameter to be string.
|
|
255
|
+
parser_instance.logger.info(f"{str(client_message.request_custom_parsed)[0: 100]}...")
|
|
256
|
+
|
|
257
|
+
def process_server_raw_data(
|
|
258
|
+
service_received_raw_data: bytes,
|
|
259
|
+
error_string: str,
|
|
260
|
+
client_message: ClientMessage
|
|
261
|
+
):
|
|
262
|
+
nonlocal protocol
|
|
263
|
+
|
|
264
|
+
client_message.response_raw_bytes = service_received_raw_data
|
|
265
|
+
|
|
266
|
+
if error_string:
|
|
267
|
+
client_message.errors.append(error_string)
|
|
268
|
+
|
|
269
|
+
if service_received_raw_data == b'':
|
|
270
|
+
return
|
|
271
|
+
|
|
272
|
+
client_message.response_auto_parsed = parse_http(client_message.response_raw_bytes, client_message)
|
|
273
|
+
if protocol != '':
|
|
274
|
+
client_message.protocol = protocol
|
|
275
|
+
|
|
276
|
+
def client_message_first_start() -> ClientMessage:
|
|
277
|
+
client_message: ClientMessage = ClientMessage()
|
|
278
|
+
client_message.client_ip = client_ip
|
|
279
|
+
client_message.source_port = source_port
|
|
280
|
+
client_message.destination_port = destination_port
|
|
281
|
+
client_message.server_name = server_name
|
|
282
|
+
client_message.thread_id = thread_id
|
|
283
|
+
client_message.process_name = process_commandline
|
|
284
|
+
|
|
285
|
+
return client_message
|
|
286
|
+
|
|
287
|
+
def responder_thread_worker():
|
|
288
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
289
|
+
|
|
290
|
+
client_message: ClientMessage = client_message_first_start()
|
|
291
|
+
try:
|
|
292
|
+
while True:
|
|
293
|
+
client_message = responder_queue.get()
|
|
294
|
+
raw_responses: list[bytes] = create_responder_response(client_message)
|
|
295
|
+
|
|
296
|
+
is_socket_closed: bool = False
|
|
297
|
+
for response_raw_bytes in raw_responses:
|
|
298
|
+
client_message.reinitialize_dynamic_vars()
|
|
299
|
+
client_message.timestamp = datetime.now()
|
|
300
|
+
client_message.response_raw_bytes = response_raw_bytes
|
|
301
|
+
error_on_send: str = sender.Sender(
|
|
302
|
+
ssl_socket=client_socket, class_message=client_message.response_raw_bytes,
|
|
303
|
+
logger=network_logger).send()
|
|
304
|
+
|
|
305
|
+
# If there was problem with sending data, we'll break the main while loop.
|
|
306
|
+
if error_on_send:
|
|
307
|
+
client_message.errors.append(error_on_send)
|
|
308
|
+
record_and_statistics_write(client_message)
|
|
309
|
+
is_socket_closed = True
|
|
310
|
+
|
|
311
|
+
if is_socket_closed:
|
|
312
|
+
exception_or_close_in_receiving_thread = True
|
|
313
|
+
return
|
|
314
|
+
except Exception as exc:
|
|
315
|
+
handle_exceptions_on_sub_connection_thread(client_message, client_exception_queue, exc)
|
|
316
|
+
|
|
317
|
+
def receive_send_start(
|
|
318
|
+
receiving_socket,
|
|
319
|
+
sending_socket = None,
|
|
320
|
+
exception_queue: queue.Queue = None
|
|
321
|
+
):
|
|
322
|
+
nonlocal client_receive_count
|
|
323
|
+
nonlocal server_receive_count
|
|
324
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
325
|
+
|
|
326
|
+
# Set the thread name to the custom name for logging
|
|
327
|
+
# threading.current_thread().name = thread_name
|
|
328
|
+
|
|
329
|
+
# Initialize the client message object with current thread's data.
|
|
330
|
+
client_message: ClientMessage = client_message_first_start()
|
|
331
|
+
|
|
332
|
+
try:
|
|
333
|
+
if receiving_socket is client_socket:
|
|
334
|
+
side: str = 'Client'
|
|
335
|
+
elif receiving_socket is service_socket_instance:
|
|
336
|
+
side: str = 'Service'
|
|
337
|
+
else:
|
|
338
|
+
raise ValueError(f"Unknown side of the socket: {receiving_socket}")
|
|
339
|
+
|
|
340
|
+
while True:
|
|
341
|
+
client_message.reinitialize_dynamic_vars()
|
|
342
|
+
|
|
343
|
+
if side == 'Client':
|
|
344
|
+
client_receive_count += 1
|
|
345
|
+
current_count = client_receive_count
|
|
346
|
+
else:
|
|
347
|
+
server_receive_count += 1
|
|
348
|
+
current_count = server_receive_count
|
|
349
|
+
|
|
350
|
+
network_logger.info(
|
|
351
|
+
f"Initializing Receiver for {side} cycle: {str(current_count)}")
|
|
352
|
+
|
|
353
|
+
# Getting message from the client over the socket using specific class.
|
|
354
|
+
received_raw_data, is_socket_closed, error_message = receiver.Receiver(
|
|
355
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
356
|
+
|
|
357
|
+
# Getting current time of message received, either from client or service.
|
|
358
|
+
client_message.timestamp = datetime.now()
|
|
359
|
+
|
|
360
|
+
# In case of client socket, we'll process the raw data specifically for the client.
|
|
361
|
+
if side == 'Client':
|
|
362
|
+
process_client_raw_data(received_raw_data, error_message, client_message)
|
|
363
|
+
client_message.action = 'client_receive'
|
|
364
|
+
# In case of service socket, we'll process the raw data specifically for the service.
|
|
365
|
+
else:
|
|
366
|
+
process_server_raw_data(received_raw_data, error_message, client_message)
|
|
367
|
+
client_message.action = 'service_receive'
|
|
368
|
+
|
|
369
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
370
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
371
|
+
if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
|
|
372
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
373
|
+
logger_method='info')
|
|
374
|
+
return
|
|
375
|
+
|
|
376
|
+
# We will record only if there was no closing signal, because if there was, it means that we initiated
|
|
377
|
+
# the close on the opposite socket.
|
|
378
|
+
record_and_statistics_write(client_message)
|
|
379
|
+
|
|
380
|
+
if is_socket_closed:
|
|
381
|
+
exception_or_close_in_receiving_thread = True
|
|
382
|
+
finish_thread()
|
|
383
|
+
return
|
|
384
|
+
|
|
385
|
+
# If we're in response mode, execute responder.
|
|
386
|
+
if config_static.TCPServer.server_response_mode:
|
|
387
|
+
responder_queue.put(client_message)
|
|
388
|
+
else:
|
|
389
|
+
# if side == 'Client':
|
|
390
|
+
# raise NotImplementedError
|
|
391
|
+
client_message.reinitialize_dynamic_vars()
|
|
392
|
+
error_on_send: str = sender.Sender(
|
|
393
|
+
ssl_socket=sending_socket, class_message=received_raw_data,
|
|
394
|
+
logger=network_logger).send()
|
|
395
|
+
|
|
396
|
+
if error_on_send:
|
|
397
|
+
client_message.reinitialize_dynamic_vars()
|
|
398
|
+
client_message.errors.append(error_on_send)
|
|
399
|
+
client_message.timestamp = datetime.now()
|
|
400
|
+
if side == 'Client':
|
|
401
|
+
client_message.action = 'service_send'
|
|
402
|
+
else:
|
|
403
|
+
client_message.action = 'client_send'
|
|
404
|
+
|
|
405
|
+
record_and_statistics_write(client_message)
|
|
406
|
+
|
|
407
|
+
# If the socket was closed, then we'll break the loop.
|
|
408
|
+
if is_socket_closed or error_on_send:
|
|
409
|
+
exception_or_close_in_receiving_thread = True
|
|
410
|
+
finish_thread()
|
|
411
|
+
return
|
|
412
|
+
except Exception as exc:
|
|
413
|
+
# If the sockets were already closed, then there is nothing to do here besides log.
|
|
414
|
+
# if (isinstance(exc, OSError) and exc.errno == 10038 and
|
|
415
|
+
# client_socket.fileno() == -1 and service_socket_instance.fileno() == -1):
|
|
416
|
+
if isinstance(exc, OSError) and exc.errno == 10038:
|
|
417
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger, logger_method='info')
|
|
266
418
|
else:
|
|
267
|
-
client_message
|
|
419
|
+
handle_exceptions_on_sub_connection_thread(client_message, exception_queue, exc)
|
|
420
|
+
|
|
421
|
+
def handle_exceptions_on_sub_connection_thread(
|
|
422
|
+
client_message: ClientMessage,
|
|
423
|
+
exception_queue: queue.Queue,
|
|
424
|
+
exc: Exception
|
|
425
|
+
):
|
|
426
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
268
427
|
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
428
|
+
exception_or_close_in_receiving_thread=True
|
|
429
|
+
# handle_exceptions(exc, client_message, recorded)
|
|
430
|
+
exception_message = tracebacks.get_as_string(one_line=True)
|
|
431
|
+
|
|
432
|
+
error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
|
|
433
|
+
print_api("Exception in a thread, forwarding to parent thread.", logger_method='info', logger=network_logger)
|
|
434
|
+
client_message.errors.append(error_message)
|
|
435
|
+
|
|
436
|
+
# if not recorded:
|
|
437
|
+
# record_and_statistics_write(client_message)
|
|
438
|
+
|
|
439
|
+
finish_thread()
|
|
440
|
+
exception_queue.put(exc)
|
|
441
|
+
|
|
442
|
+
def handle_exceptions_on_main_connection_thread(
|
|
443
|
+
exc: Exception,
|
|
444
|
+
client_message: ClientMessage
|
|
445
|
+
):
|
|
446
|
+
exception_message = tracebacks.get_as_string(one_line=True)
|
|
447
|
+
error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
|
|
448
|
+
print_api(error_message, logger_method='critical', logger=network_logger)
|
|
449
|
+
client_message.errors.append(error_message)
|
|
450
|
+
|
|
451
|
+
# === At this point while loop of 'client_connection_boolean' was broken =======================================
|
|
452
|
+
# If recorder wasn't executed before, then execute it now
|
|
453
|
+
record_and_statistics_write(client_message)
|
|
454
|
+
|
|
455
|
+
finish_thread()
|
|
456
|
+
|
|
457
|
+
# After the socket clean up, we will still raise the exception to the main thread.
|
|
458
|
+
raise exc
|
|
459
|
+
|
|
460
|
+
# ================================================================================================================
|
|
461
|
+
# This is the start of the thread_worker_main function
|
|
275
462
|
|
|
276
463
|
# Only protocols that are encrypted with TLS have the server name attribute.
|
|
277
464
|
if is_tls:
|
|
278
465
|
# Get current destination domain
|
|
279
466
|
server_name = client_socket.server_hostname
|
|
280
|
-
# client_message.server_name = domain_from_dns
|
|
281
467
|
# If the protocol is not TLS, then we'll use the domain from the DNS.
|
|
282
468
|
else:
|
|
283
469
|
server_name = domain_from_dns
|
|
284
|
-
client_message.server_name = server_name
|
|
285
470
|
|
|
286
471
|
thread_id = threads.current_thread_id()
|
|
287
|
-
client_message.thread_id = thread_id
|
|
288
472
|
|
|
473
|
+
# This is the main protocols.
|
|
289
474
|
protocol: str = str()
|
|
475
|
+
# This is the secondary protocol in the websocket.
|
|
476
|
+
protocol3: str = str()
|
|
290
477
|
# # This is Client Masked Frame Parser.
|
|
291
478
|
# websocket_masked_frame_parser = websocket_parse.WebsocketFrameParser()
|
|
292
479
|
# # This is Server UnMasked Frame Parser.
|
|
@@ -295,7 +482,7 @@ def thread_worker_main(
|
|
|
295
482
|
|
|
296
483
|
# Loading parser by domain, if there is no parser for current domain - general reference parser is loaded.
|
|
297
484
|
# These should be outside any loop and initialized only once entering the thread.
|
|
298
|
-
parser, responder,
|
|
485
|
+
parser, responder, recorder_no_init, mtls_dict = assign_class_by_domain(
|
|
299
486
|
engines_usage=config_static.TCPServer.engines_usage,
|
|
300
487
|
engines_list=engines_list,
|
|
301
488
|
message_domain_name=server_name,
|
|
@@ -303,140 +490,83 @@ def thread_worker_main(
|
|
|
303
490
|
logger=network_logger
|
|
304
491
|
)
|
|
305
492
|
|
|
493
|
+
recorder = recorder_no_init(record_path=config_static.LogRec.recordings_path)
|
|
494
|
+
|
|
495
|
+
# Initializing the client message object with current thread's data.
|
|
496
|
+
# This is needed only to skip error alerts after 'try'.
|
|
497
|
+
client_message_connection: ClientMessage = ClientMessage()
|
|
498
|
+
# This is needed to indicate if there was an exception or socket was closed in any of the receiving thread.
|
|
499
|
+
exception_or_close_in_receiving_thread: bool = False
|
|
500
|
+
# Responder queue for ClientMessage objects.
|
|
501
|
+
responder_queue: queue.Queue = queue.Queue()
|
|
502
|
+
|
|
306
503
|
try:
|
|
307
504
|
client_ip, source_port = client_socket.getpeername()
|
|
308
|
-
client_message.client_ip = client_ip
|
|
309
|
-
client_message.source_port = source_port
|
|
310
|
-
|
|
311
505
|
destination_port = client_socket.getsockname()[1]
|
|
312
|
-
client_message.destination_port = destination_port
|
|
313
506
|
|
|
314
507
|
network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
|
|
315
508
|
f"Destination service: [{server_name}:{destination_port}]")
|
|
316
509
|
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
while True:
|
|
323
|
-
# If cycle count is None, then it's the first cycle, else it's not.
|
|
324
|
-
# The cycle_count should be added 1 in the beginning of each cycle, and not in the end, since not always
|
|
325
|
-
# the cycle will be executed till the end.
|
|
326
|
-
if cycle_count is None:
|
|
327
|
-
cycle_count = 0
|
|
328
|
-
else:
|
|
329
|
-
cycle_count += 1
|
|
330
|
-
|
|
331
|
-
recorded: bool = False
|
|
332
|
-
statistics_error_list: list[str] = list()
|
|
333
|
-
|
|
334
|
-
client_message = ClientMessage()
|
|
335
|
-
client_message.thread_id = thread_id
|
|
336
|
-
client_message.client_ip = client_ip
|
|
337
|
-
client_message.source_port = source_port
|
|
338
|
-
client_message.destination_port = destination_port
|
|
339
|
-
client_message.process_name = process_commandline
|
|
340
|
-
client_message.server_name = server_name
|
|
341
|
-
# Getting current time of message received from client.
|
|
342
|
-
client_message.request_time_received = datetime.now()
|
|
343
|
-
|
|
344
|
-
# Peek if there is some data in the socket.
|
|
345
|
-
# This is needed to check if the client just connects without sending data, if so we need to try and
|
|
346
|
-
# receive data from the server and send it to the client.
|
|
347
|
-
# We will do it only on the first cycle, after that the connection should work as usual.
|
|
348
|
-
# Sometimes the client will execute connection without sending data, just for the server to send response.
|
|
349
|
-
is_socket_ready: bool = True
|
|
350
|
-
if cycle_count == 0:
|
|
351
|
-
is_socket_ready = receiver.is_socket_ready_for_read(client_socket)
|
|
352
|
-
|
|
353
|
-
if is_socket_ready:
|
|
354
|
-
network_logger.info(f"Initializing Receiver on cycle: {str(cycle_count+1)}")
|
|
355
|
-
# Getting message from the client over the socket using specific class.
|
|
356
|
-
client_received_raw_data = receiver.Receiver(
|
|
357
|
-
ssl_socket=client_socket, logger=network_logger).receive()
|
|
358
|
-
|
|
359
|
-
end_socket = process_client_raw_data_request()
|
|
360
|
-
|
|
361
|
-
if not end_socket:
|
|
362
|
-
# If we're in response mode, execute responder.
|
|
363
|
-
response_raw_bytes = None
|
|
364
|
-
if config_static.TCPServer.server_response_mode:
|
|
365
|
-
create_responder_response()
|
|
366
|
-
# Else, we're not in response mode, then execute client connect and record section.
|
|
367
|
-
else:
|
|
368
|
-
# If "service_client" object is not defined, we'll define it.
|
|
369
|
-
# If it's defined, then there's still active "ssl_socket" with connection to the service domain.
|
|
370
|
-
if not service_client:
|
|
371
|
-
service_client = create_client_socket()
|
|
372
|
-
|
|
373
|
-
# Sending current client message and receiving a response.
|
|
374
|
-
# If there was an error it will be passed to "client_message" object class and if not, "None" will
|
|
375
|
-
# be passed.
|
|
376
|
-
# If there was connection error or socket close, then "ssl_socket" of the "service_client"
|
|
377
|
-
# will be empty.
|
|
378
|
-
response_raw_bytes, client_message.error, client_message.server_ip, service_ssl_socket = (
|
|
379
|
-
service_client.send_receive_to_service(client_message.request_raw_bytes, (not is_socket_ready)))
|
|
380
|
-
|
|
381
|
-
process_received_response_from_service_client()
|
|
382
|
-
|
|
383
|
-
# So if the socket was closed and there was an error we can break the loop
|
|
384
|
-
if not service_ssl_socket:
|
|
385
|
-
record_and_statistics_write()
|
|
386
|
-
recorded = True
|
|
387
|
-
break
|
|
388
|
-
|
|
389
|
-
# If there is a response(s), then send it.
|
|
390
|
-
if client_message.response_list_of_raw_bytes:
|
|
391
|
-
# Sending response/s to client no matter if in record mode or not.
|
|
392
|
-
network_logger.info(
|
|
393
|
-
f"Sending messages to client: {len(client_message.response_list_of_raw_bytes)}")
|
|
394
|
-
|
|
395
|
-
# Iterate through the list of byte responses.
|
|
396
|
-
for response_raw_bytes in client_message.response_list_of_raw_bytes:
|
|
397
|
-
error_on_send: str = sender.Sender(
|
|
398
|
-
ssl_socket=client_socket, class_message=response_raw_bytes,
|
|
399
|
-
logger=network_logger).send()
|
|
400
|
-
|
|
401
|
-
# If there was problem with sending data, we'll break current loop.
|
|
402
|
-
if error_on_send:
|
|
403
|
-
statistics_error_list.append(error_on_send)
|
|
404
|
-
break
|
|
405
|
-
# If response from server came back empty, then the server has closed the connection,
|
|
406
|
-
# we will do the same.
|
|
407
|
-
else:
|
|
408
|
-
network_logger.info(f"Response empty, nothing to send to client.")
|
|
409
|
-
break
|
|
510
|
+
# If we're in response mode, we'll start the responder thread.
|
|
511
|
+
if config_static.TCPServer.server_response_mode:
|
|
512
|
+
responder_thread: threading.Thread = threading.Thread(
|
|
513
|
+
target=responder_thread_worker, name=f"Thread-{thread_id}-Responder", daemon=True)
|
|
514
|
+
responder_thread.start()
|
|
410
515
|
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
# be empty response from the server, so we can close the socket as well and exceptions will be raised.
|
|
416
|
-
if end_socket:
|
|
417
|
-
# If it's the first cycle we will record the message from the client if it came empty.
|
|
418
|
-
if cycle_count == 0:
|
|
419
|
-
record_and_statistics_write()
|
|
420
|
-
|
|
421
|
-
# In other cases, we'll just break the loop, since empty message means that the other side closed the
|
|
422
|
-
# connection.
|
|
423
|
-
recorded = True
|
|
516
|
+
service_client = None
|
|
517
|
+
client_receive_count: int = 0
|
|
518
|
+
server_receive_count: int = 0
|
|
519
|
+
client_message_connection = client_message_first_start()
|
|
424
520
|
|
|
425
|
-
|
|
521
|
+
# If we're not in response mode, then we'll create the client socket to the service.
|
|
522
|
+
# noinspection PyTypeChecker
|
|
523
|
+
connection_error: str = None
|
|
524
|
+
service_socket_instance = None
|
|
525
|
+
if not config_static.TCPServer.server_response_mode:
|
|
526
|
+
# If "service_client" object is not defined, we'll define it.
|
|
527
|
+
# If it's defined, then there's still active "ssl_socket" with connection to the service domain.
|
|
528
|
+
if not service_client:
|
|
529
|
+
service_client = create_client_socket(client_message_connection)
|
|
530
|
+
service_socket_instance, connection_error = service_client.service_connection()
|
|
531
|
+
else:
|
|
532
|
+
client_message_connection.timestamp = datetime.now()
|
|
533
|
+
client_message_connection.action = 'service_connect'
|
|
534
|
+
client_message_connection.info = 'Server Response Mode'
|
|
535
|
+
responder_queue.put(client_message_connection)
|
|
536
|
+
|
|
537
|
+
if connection_error:
|
|
538
|
+
client_message_connection.timestamp = datetime.now()
|
|
539
|
+
client_message_connection.errors.append(connection_error)
|
|
540
|
+
client_message_connection.action = 'service_connect'
|
|
541
|
+
record_and_statistics_write(client_message_connection)
|
|
542
|
+
else:
|
|
543
|
+
client_exception_queue: queue.Queue = queue.Queue()
|
|
544
|
+
service_exception_queue: queue.Queue = queue.Queue()
|
|
545
|
+
|
|
546
|
+
client_thread = threading.Thread(
|
|
547
|
+
target=receive_send_start, args=(client_socket, service_socket_instance, client_exception_queue),
|
|
548
|
+
name=f"Thread-{thread_id}-Client")
|
|
549
|
+
client_thread.daemon = True
|
|
550
|
+
client_thread.start()
|
|
551
|
+
|
|
552
|
+
service_thread = threading.Thread(
|
|
553
|
+
target=receive_send_start, args=(service_socket_instance, client_socket, service_exception_queue),
|
|
554
|
+
name=f"Thread-{thread_id}-Service")
|
|
555
|
+
service_thread.daemon = True
|
|
556
|
+
service_thread.start()
|
|
557
|
+
|
|
558
|
+
client_thread.join()
|
|
559
|
+
service_thread.join()
|
|
560
|
+
|
|
561
|
+
# If there was an exception in any of the threads, then we'll raise it here.
|
|
562
|
+
if not client_exception_queue.empty():
|
|
563
|
+
raise client_exception_queue.get()
|
|
564
|
+
if not service_exception_queue.empty():
|
|
565
|
+
raise service_exception_queue.get()
|
|
426
566
|
|
|
427
567
|
finish_thread()
|
|
428
568
|
except Exception as e:
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
print_api(error_message, logger_method='critical', logger=network_logger)
|
|
432
|
-
statistics_error_list.append(error_message)
|
|
433
|
-
|
|
434
|
-
# === At this point while loop of 'client_connection_boolean' was broken =======================================
|
|
435
|
-
# If recorder wasn't executed before, then execute it now
|
|
436
|
-
if not recorded:
|
|
437
|
-
record_and_statistics_write()
|
|
438
|
-
|
|
439
|
-
finish_thread()
|
|
569
|
+
if not client_message_connection.timestamp:
|
|
570
|
+
client_message_connection.timestamp = datetime.now()
|
|
440
571
|
|
|
441
|
-
|
|
442
|
-
raise e
|
|
572
|
+
handle_exceptions_on_main_connection_thread(e, client_message_connection)
|