atomicshop 3.3.28__py3-none-any.whl → 3.10.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of atomicshop might be problematic. Click here for more details.
- atomicshop/__init__.py +1 -1
- atomicshop/a_mains/get_local_tcp_ports.py +85 -0
- atomicshop/a_mains/install_ca_certificate.py +172 -0
- atomicshop/a_mains/process_from_port.py +119 -0
- atomicshop/a_mains/set_default_dns_gateway.py +90 -0
- atomicshop/basics/strings.py +1 -1
- atomicshop/certificates.py +2 -2
- atomicshop/dns.py +26 -28
- atomicshop/etws/traces/trace_tcp.py +1 -2
- atomicshop/mitm/centered_settings.py +133 -0
- atomicshop/mitm/config_static.py +18 -43
- atomicshop/mitm/connection_thread_worker.py +376 -162
- atomicshop/mitm/engines/__parent/recorder___parent.py +1 -1
- atomicshop/mitm/engines/__parent/requester___parent.py +1 -1
- atomicshop/mitm/engines/__parent/responder___parent.py +15 -2
- atomicshop/mitm/engines/create_module_template.py +1 -2
- atomicshop/mitm/import_config.py +79 -88
- atomicshop/mitm/initialize_engines.py +1 -2
- atomicshop/mitm/message.py +5 -4
- atomicshop/mitm/mitm_main.py +222 -121
- atomicshop/mitm/recs_files.py +61 -5
- atomicshop/mitm/ssh_tester.py +82 -0
- atomicshop/networks.py +108 -93
- atomicshop/package_mains_processor.py +84 -0
- atomicshop/permissions/ubuntu_permissions.py +47 -0
- atomicshop/print_api.py +3 -5
- atomicshop/python_functions.py +23 -108
- atomicshop/speech_recognize.py +8 -0
- atomicshop/ssh_remote.py +115 -51
- atomicshop/web.py +20 -7
- atomicshop/web_apis/google_llm.py +22 -14
- atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
- atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +2 -1
- atomicshop/wrappers/dockerw/dockerw.py +2 -2
- atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +5 -5
- atomicshop/wrappers/githubw.py +175 -63
- atomicshop/wrappers/loggingw/handlers.py +1 -1
- atomicshop/wrappers/loggingw/loggingw.py +17 -1
- atomicshop/wrappers/netshw.py +124 -3
- atomicshop/wrappers/playwrightw/scenarios.py +1 -1
- atomicshop/wrappers/powershell_networking.py +80 -0
- atomicshop/wrappers/psutilw/psutil_networks.py +9 -0
- atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +3 -105
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +3 -57
- atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +12 -27
- atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +15 -9
- atomicshop/wrappers/socketw/certificator.py +19 -9
- atomicshop/wrappers/socketw/creator.py +30 -7
- atomicshop/wrappers/socketw/dns_server.py +6 -6
- atomicshop/wrappers/socketw/exception_wrapper.py +3 -3
- atomicshop/wrappers/socketw/process_getter.py +86 -0
- atomicshop/wrappers/socketw/receiver.py +29 -9
- atomicshop/wrappers/socketw/sender.py +10 -9
- atomicshop/wrappers/socketw/sni.py +23 -6
- atomicshop/wrappers/socketw/{base.py → socket_base.py} +33 -1
- atomicshop/wrappers/socketw/socket_client.py +6 -8
- atomicshop/wrappers/socketw/socket_wrapper.py +82 -21
- atomicshop/wrappers/socketw/ssl_base.py +6 -2
- atomicshop/wrappers/win_auditw.py +189 -0
- {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/METADATA +25 -30
- {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/RECORD +74 -88
- atomicshop/_basics_temp.py +0 -101
- atomicshop/a_installs/ubuntu/docker_rootless.py +0 -11
- atomicshop/a_installs/ubuntu/docker_sudo.py +0 -11
- atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
- atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
- atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
- atomicshop/addons/package_setup/Setup.cmd +0 -7
- atomicshop/archiver/__init__.py +0 -0
- atomicshop/archiver/_search_in_zip.py +0 -189
- atomicshop/archiver/search_in_archive.py +0 -284
- atomicshop/archiver/sevenz_app_w.py +0 -86
- atomicshop/archiver/sevenzs.py +0 -73
- atomicshop/archiver/shutils.py +0 -34
- atomicshop/archiver/zips.py +0 -353
- atomicshop/file_types.py +0 -24
- atomicshop/pbtkmultifile_argparse.py +0 -88
- atomicshop/script_as_string_processor.py +0 -42
- atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
- atomicshop/ssh_scripts/process_from_port.py +0 -27
- atomicshop/wrappers/_process_wrapper_curl.py +0 -27
- atomicshop/wrappers/_process_wrapper_tar.py +0 -21
- atomicshop/wrappers/dockerw/install_docker.py +0 -449
- atomicshop/wrappers/ffmpegw.py +0 -125
- atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
- atomicshop/wrappers/socketw/get_process.py +0 -123
- /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compile.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.dll +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.exp +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.lib +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +0 -0
- {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/WHEEL +0 -0
- {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/licenses/LICENSE.txt +0 -0
- {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/top_level.txt +0 -0
|
@@ -1,9 +1,13 @@
|
|
|
1
|
+
import multiprocessing
|
|
1
2
|
from datetime import datetime
|
|
2
3
|
import threading
|
|
3
4
|
import queue
|
|
4
5
|
import socket
|
|
6
|
+
import ssl
|
|
7
|
+
from typing import Literal
|
|
8
|
+
import struct
|
|
5
9
|
|
|
6
|
-
from ..wrappers.socketw import receiver, sender, socket_client,
|
|
10
|
+
from ..wrappers.socketw import receiver, sender, socket_client, socket_base
|
|
7
11
|
from .. import websocket_parse, ip_addresses
|
|
8
12
|
from ..http_parse import HTTPRequestParse, HTTPResponseParse
|
|
9
13
|
from ..basics import threads, tracebacks
|
|
@@ -25,7 +29,7 @@ def thread_worker_main(
|
|
|
25
29
|
tls_version: str,
|
|
26
30
|
domain_from_dns,
|
|
27
31
|
statistics_writer,
|
|
28
|
-
engines_list,
|
|
32
|
+
engines_list: list[initialize_engines.ModuleCategory],
|
|
29
33
|
|
|
30
34
|
# These parameters come from the main mitm module.
|
|
31
35
|
config_static: cf
|
|
@@ -168,32 +172,55 @@ def thread_worker_main(
|
|
|
168
172
|
def parse_websocket(raw_bytes):
|
|
169
173
|
return websocket_frame_parser.parse_frame_bytes(raw_bytes)
|
|
170
174
|
|
|
171
|
-
def finish_thread():
|
|
175
|
+
def finish_thread(send_connection_reset: bool = False):
|
|
176
|
+
"""
|
|
177
|
+
Finishing the thread, closing sockets.
|
|
178
|
+
|
|
179
|
+
:param send_connection_reset: Whether to send TCP RST flag to the client when closing the socket.
|
|
180
|
+
Basically what happens is that the server socket is closed abruptly sending us any Connection*Error exception.
|
|
181
|
+
We can simulate the ConnectionResetError on the client side by sending TCP RST flag when closing the socket.
|
|
182
|
+
But not the ConnectionAbortedError, since it is caused by other reasons (local TCP stack and has nothing
|
|
183
|
+
to do with the remote server).
|
|
184
|
+
"""
|
|
172
185
|
# At this stage there could be several times that the same socket was used to the service server - we need to
|
|
173
186
|
# close this socket as well if it still opened.
|
|
174
187
|
# The first part of the condition is to check if the service socket was connected at all.
|
|
175
188
|
# If the service socket couldn't connect, then the instance will be None.
|
|
176
189
|
if service_socket_instance and service_socket_instance.fileno() != -1:
|
|
177
|
-
if
|
|
178
|
-
|
|
190
|
+
if origin_service_client_instance.socket_instance:
|
|
191
|
+
origin_service_client_instance.close_socket()
|
|
179
192
|
|
|
180
193
|
# If client socket is still opened - close
|
|
181
194
|
if client_socket.fileno() != -1:
|
|
195
|
+
if send_connection_reset:
|
|
196
|
+
# Abort the connection
|
|
197
|
+
linger = struct.pack('ii', 1, 0)
|
|
198
|
+
client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger)
|
|
182
199
|
client_socket.close()
|
|
183
|
-
|
|
200
|
+
|
|
201
|
+
if send_connection_reset:
|
|
202
|
+
network_logger.info(f"Closed client socket [{client_ip}:{source_port}] with TCP RST flag (Sent ConnectionResetError)...")
|
|
203
|
+
else:
|
|
204
|
+
network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
|
|
184
205
|
|
|
185
206
|
network_logger.info("Thread Finished. Will continue listening on the Main thread")
|
|
186
207
|
|
|
187
208
|
def create_requester_request(
|
|
188
209
|
client_message: ClientMessage,
|
|
189
210
|
sending_socket: socket.socket
|
|
190
|
-
) ->
|
|
211
|
+
) -> tuple[bytes, bool]:
|
|
212
|
+
request_received_raw: bytes = client_message.request_raw_bytes
|
|
191
213
|
request_custom_raw: bytes = requester.create_request(client_message, sending_socket=sending_socket)
|
|
192
214
|
|
|
193
|
-
|
|
194
|
-
|
|
215
|
+
if request_custom_raw is None or request_received_raw == request_custom_raw:
|
|
216
|
+
is_requester_worked: bool = False
|
|
217
|
+
else:
|
|
218
|
+
is_requester_worked: bool = True
|
|
219
|
+
|
|
220
|
+
# Output first 100 characters of the request.
|
|
221
|
+
requester.logger.info(f"{request_custom_raw[0: 100]}...")
|
|
195
222
|
|
|
196
|
-
return
|
|
223
|
+
return request_custom_raw, is_requester_worked
|
|
197
224
|
|
|
198
225
|
def create_responder_response(client_message: ClientMessage) -> list[bytes]:
|
|
199
226
|
if client_message.action == 'service_connect':
|
|
@@ -201,13 +228,19 @@ def thread_worker_main(
|
|
|
201
228
|
else:
|
|
202
229
|
# If we're in offline mode, and it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
|
|
203
230
|
# response automatically.
|
|
204
|
-
if config_static.MainConfig.
|
|
231
|
+
if config_static.MainConfig.is_offline and protocol == 'Websocket' and client_receive_count == 1:
|
|
205
232
|
responses: list = list()
|
|
206
233
|
responses.append(
|
|
207
234
|
websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
|
|
235
|
+
responder.logger.info(f"Generated automatic WebSocket response in Offline Mode.")
|
|
208
236
|
else:
|
|
209
237
|
# Creating response for parsed message and printing
|
|
210
|
-
|
|
238
|
+
responder_responses: list = responder.create_response(client_message)
|
|
239
|
+
if responder_responses is None:
|
|
240
|
+
responses: list = [client_message.response_raw_bytes]
|
|
241
|
+
else:
|
|
242
|
+
responses: list = responder_responses
|
|
243
|
+
responder.logger.info(f"Generated {len(responses)} responses from responder.")
|
|
211
244
|
|
|
212
245
|
# Output first 100 characters of all the responses in the list.
|
|
213
246
|
for response_raw_bytes_single in responses:
|
|
@@ -242,7 +275,7 @@ def thread_worker_main(
|
|
|
242
275
|
else:
|
|
243
276
|
# If we're on localhost, then use external services list in order to resolve the domain:
|
|
244
277
|
# config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
|
|
245
|
-
if client_message.client_ip in
|
|
278
|
+
if client_message.client_ip in socket_base.THIS_DEVICE_IP_LIST:
|
|
246
279
|
service_client_instance = socket_client.SocketClient(
|
|
247
280
|
service_name=client_message.server_name,
|
|
248
281
|
service_port=client_message.destination_port,
|
|
@@ -330,11 +363,291 @@ def thread_worker_main(
|
|
|
330
363
|
client_message.destination_port = destination_port
|
|
331
364
|
client_message.server_name = server_name
|
|
332
365
|
client_message.thread_id = thread_id
|
|
366
|
+
client_message.thread_process = thread_process_name
|
|
333
367
|
client_message.process_name = process_commandline
|
|
334
368
|
client_message.engine_name = engine_name
|
|
335
369
|
|
|
336
370
|
return client_message
|
|
337
371
|
|
|
372
|
+
def receive_send_service_connect(
|
|
373
|
+
client_connection_message: ClientMessage,
|
|
374
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
375
|
+
) -> Literal['continue', 'return'] | None:
|
|
376
|
+
|
|
377
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
378
|
+
|
|
379
|
+
client_message = client_connection_message
|
|
380
|
+
|
|
381
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
382
|
+
print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]",
|
|
383
|
+
logger=network_logger, logger_method='info')
|
|
384
|
+
|
|
385
|
+
# If the client message is the connection message, then we'll skip to the next iteration.
|
|
386
|
+
if not bytes_to_send_list:
|
|
387
|
+
return 'continue'
|
|
388
|
+
|
|
389
|
+
# is_socket_closed: bool = False
|
|
390
|
+
error_on_send: str = str()
|
|
391
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
392
|
+
client_message.reinitialize_dynamic_vars()
|
|
393
|
+
client_message.timestamp = datetime.now()
|
|
394
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
395
|
+
client_message.action = 'service_responder'
|
|
396
|
+
record_and_statistics_write(client_message)
|
|
397
|
+
|
|
398
|
+
# Send the bytes back to the client socket.
|
|
399
|
+
error_on_send: str = sender.Sender(
|
|
400
|
+
ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
|
|
401
|
+
logger=network_logger).send()
|
|
402
|
+
|
|
403
|
+
if error_on_send:
|
|
404
|
+
client_message.reinitialize_dynamic_vars()
|
|
405
|
+
client_message.errors.append(error_on_send)
|
|
406
|
+
client_message.timestamp = datetime.now()
|
|
407
|
+
client_message.action = 'client_send'
|
|
408
|
+
record_and_statistics_write(client_message)
|
|
409
|
+
|
|
410
|
+
if error_on_send:
|
|
411
|
+
exception_or_close_in_receiving_thread = True
|
|
412
|
+
finish_thread()
|
|
413
|
+
return 'return'
|
|
414
|
+
|
|
415
|
+
return None
|
|
416
|
+
|
|
417
|
+
def receive_send_client_offline(
|
|
418
|
+
client_message: ClientMessage,
|
|
419
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
420
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
421
|
+
) -> Literal['return'] | None:
|
|
422
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
423
|
+
nonlocal client_receive_count
|
|
424
|
+
|
|
425
|
+
client_receive_count += 1
|
|
426
|
+
|
|
427
|
+
network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
|
|
428
|
+
client_message.timestamp = datetime.now()
|
|
429
|
+
|
|
430
|
+
received_raw_data, is_socket_closed, error_message = receiver.Receiver(
|
|
431
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
432
|
+
|
|
433
|
+
process_client_raw_data(received_raw_data, error_message, client_message)
|
|
434
|
+
client_message.action = 'client_receive'
|
|
435
|
+
record_and_statistics_write(client_message)
|
|
436
|
+
if error_message:
|
|
437
|
+
print_api(error_message, logger=network_logger, logger_method='critical')
|
|
438
|
+
|
|
439
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
440
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
441
|
+
if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
|
|
442
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
443
|
+
logger_method='info')
|
|
444
|
+
return 'return'
|
|
445
|
+
|
|
446
|
+
# If the socket was closed on receive, and we're in offline mode, then we'll finish the thread right away.
|
|
447
|
+
# Since nothing more can be done, like responding to service or using requester.
|
|
448
|
+
if is_socket_closed:
|
|
449
|
+
exception_or_close_in_receiving_thread = True
|
|
450
|
+
finish_thread()
|
|
451
|
+
return 'return'
|
|
452
|
+
|
|
453
|
+
# Send to requester.
|
|
454
|
+
# THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
|
|
455
|
+
request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
|
|
456
|
+
# We will not process the raw data if requester didn't change anything.
|
|
457
|
+
if is_requester_worked:
|
|
458
|
+
client_message.reinitialize_dynamic_vars()
|
|
459
|
+
client_message.timestamp = datetime.now()
|
|
460
|
+
client_message.request_raw_bytes = request_custom_raw
|
|
461
|
+
client_message.action = 'client_requester'
|
|
462
|
+
process_client_raw_data(request_custom_raw, error_message, client_message)
|
|
463
|
+
record_and_statistics_write(client_message)
|
|
464
|
+
|
|
465
|
+
print_api("Offline Mode, sending to responder directly.", logger=network_logger,
|
|
466
|
+
logger_method='info')
|
|
467
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
468
|
+
|
|
469
|
+
error_on_send: str = str()
|
|
470
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
471
|
+
client_message.reinitialize_dynamic_vars()
|
|
472
|
+
client_message.timestamp = datetime.now()
|
|
473
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
474
|
+
client_message.action = 'client_responder_offline'
|
|
475
|
+
process_server_raw_data(bytes_to_send_single, '', client_message)
|
|
476
|
+
record_and_statistics_write(client_message)
|
|
477
|
+
|
|
478
|
+
error_on_send: str = sender.Sender(
|
|
479
|
+
ssl_socket=receiving_socket, bytes_to_send=bytes_to_send_single,
|
|
480
|
+
logger=network_logger).send()
|
|
481
|
+
|
|
482
|
+
if error_on_send:
|
|
483
|
+
client_message.reinitialize_dynamic_vars()
|
|
484
|
+
client_message.errors.append(error_on_send)
|
|
485
|
+
client_message.timestamp = datetime.now()
|
|
486
|
+
client_message.action = 'service_send'
|
|
487
|
+
|
|
488
|
+
record_and_statistics_write(client_message)
|
|
489
|
+
|
|
490
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
491
|
+
if is_socket_closed or error_on_send:
|
|
492
|
+
exception_or_close_in_receiving_thread = True
|
|
493
|
+
finish_thread()
|
|
494
|
+
return 'return'
|
|
495
|
+
|
|
496
|
+
return None
|
|
497
|
+
|
|
498
|
+
def receive_send_client(
|
|
499
|
+
client_message: ClientMessage,
|
|
500
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
501
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
502
|
+
) -> Literal['return'] | None:
|
|
503
|
+
|
|
504
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
505
|
+
nonlocal client_receive_count
|
|
506
|
+
|
|
507
|
+
client_receive_count += 1
|
|
508
|
+
|
|
509
|
+
network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
|
|
510
|
+
|
|
511
|
+
# Getting message from the client over the socket using specific class.
|
|
512
|
+
client_message.timestamp = datetime.now()
|
|
513
|
+
received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
|
|
514
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
515
|
+
|
|
516
|
+
process_client_raw_data(received_raw_data, error_on_receive, client_message)
|
|
517
|
+
client_message.action = 'client_receive'
|
|
518
|
+
|
|
519
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
520
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
521
|
+
if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
|
|
522
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
523
|
+
logger_method='info')
|
|
524
|
+
return 'return'
|
|
525
|
+
|
|
526
|
+
# We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
|
|
527
|
+
# Meaning 'exception_or_close_in_receiving_thread=True'.
|
|
528
|
+
record_and_statistics_write(client_message)
|
|
529
|
+
if error_on_receive:
|
|
530
|
+
print_api(error_on_receive, logger=network_logger, logger_method='critical')
|
|
531
|
+
|
|
532
|
+
# At this point if the socket was closed on receive, then there's no point to send anything to the service.
|
|
533
|
+
# So, we check for 'is_socket_closed' here.
|
|
534
|
+
error_on_send: str = str()
|
|
535
|
+
if not is_socket_closed:
|
|
536
|
+
# Send to requester.
|
|
537
|
+
# THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
|
|
538
|
+
request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
|
|
539
|
+
# We will not process the raw data if requester didn't change anything.
|
|
540
|
+
if is_requester_worked:
|
|
541
|
+
client_message.reinitialize_dynamic_vars()
|
|
542
|
+
client_message.timestamp = datetime.now()
|
|
543
|
+
client_message.request_raw_bytes = request_custom_raw
|
|
544
|
+
client_message.action = 'client_requester'
|
|
545
|
+
process_client_raw_data(request_custom_raw, error_on_receive, client_message)
|
|
546
|
+
record_and_statistics_write(client_message)
|
|
547
|
+
|
|
548
|
+
error_on_send: str = sender.Sender(
|
|
549
|
+
ssl_socket=sending_socket, bytes_to_send=client_message.request_raw_bytes,
|
|
550
|
+
logger=network_logger).send()
|
|
551
|
+
|
|
552
|
+
if error_on_send:
|
|
553
|
+
client_message.reinitialize_dynamic_vars()
|
|
554
|
+
client_message.errors.append(error_on_send)
|
|
555
|
+
client_message.timestamp = datetime.now()
|
|
556
|
+
client_message.action = 'service_send'
|
|
557
|
+
record_and_statistics_write(client_message)
|
|
558
|
+
|
|
559
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
560
|
+
if is_socket_closed or error_on_send:
|
|
561
|
+
exception_or_close_in_receiving_thread = True
|
|
562
|
+
finish_thread()
|
|
563
|
+
return 'return'
|
|
564
|
+
|
|
565
|
+
return None
|
|
566
|
+
|
|
567
|
+
def receive_send_service(
|
|
568
|
+
client_message: ClientMessage,
|
|
569
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
570
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
571
|
+
) -> Literal['return'] | None:
|
|
572
|
+
|
|
573
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
574
|
+
nonlocal server_receive_count
|
|
575
|
+
|
|
576
|
+
server_receive_count += 1
|
|
577
|
+
|
|
578
|
+
network_logger.info(f"Initializing Receiver for Service cycle: {str(server_receive_count)}")
|
|
579
|
+
|
|
580
|
+
# Getting message from the client over the socket using specific class.
|
|
581
|
+
client_message.timestamp = datetime.now()
|
|
582
|
+
received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
|
|
583
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
584
|
+
|
|
585
|
+
process_server_raw_data(received_raw_data, error_on_receive, client_message)
|
|
586
|
+
client_message.action = 'service_receive'
|
|
587
|
+
|
|
588
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
589
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
590
|
+
if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
|
|
591
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
592
|
+
logger_method='info')
|
|
593
|
+
return 'return'
|
|
594
|
+
|
|
595
|
+
# We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
|
|
596
|
+
# Meaning 'exception_or_close_in_receiving_thread=True'.
|
|
597
|
+
record_and_statistics_write(client_message)
|
|
598
|
+
if error_on_receive:
|
|
599
|
+
print_api(error_on_receive, logger=network_logger, logger_method='critical')
|
|
600
|
+
|
|
601
|
+
# At this stage, we have received the response from the service, but there was an exception/error or the socket was simply closed.
|
|
602
|
+
# Meaning the 'received_raw_data' is None for exception or b'' for just closed socket.
|
|
603
|
+
# So, there's no point to send anything back to the client.
|
|
604
|
+
# Close both sockets and finish the threads.
|
|
605
|
+
# So, we check for 'is_socket_closed' here.
|
|
606
|
+
error_on_send: str = str()
|
|
607
|
+
if not is_socket_closed:
|
|
608
|
+
# Now send it to requester/responder.
|
|
609
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
610
|
+
|
|
611
|
+
# is_socket_closed: bool = False
|
|
612
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
613
|
+
client_message.reinitialize_dynamic_vars()
|
|
614
|
+
client_message.timestamp = datetime.now()
|
|
615
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
616
|
+
|
|
617
|
+
# This records the requester or responder output, only if it is not the same as the original
|
|
618
|
+
# message.
|
|
619
|
+
if bytes_to_send_single != received_raw_data:
|
|
620
|
+
client_message.action = 'service_responder'
|
|
621
|
+
record_and_statistics_write(client_message)
|
|
622
|
+
|
|
623
|
+
error_on_send: str = sender.Sender(
|
|
624
|
+
ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
|
|
625
|
+
logger=network_logger).send()
|
|
626
|
+
|
|
627
|
+
if error_on_send:
|
|
628
|
+
client_message.reinitialize_dynamic_vars()
|
|
629
|
+
client_message.errors.append(error_on_send)
|
|
630
|
+
client_message.timestamp = datetime.now()
|
|
631
|
+
client_message.action = 'client_send'
|
|
632
|
+
|
|
633
|
+
record_and_statistics_write(client_message)
|
|
634
|
+
|
|
635
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
636
|
+
if is_socket_closed or error_on_send:
|
|
637
|
+
exception_or_close_in_receiving_thread = True
|
|
638
|
+
|
|
639
|
+
if error_on_receive and 'Connection' in error_on_receive and 'Error' in error_on_receive:
|
|
640
|
+
# If there was a connection error on receive from client, and we're closing the socket to the client,
|
|
641
|
+
# then we'll send TCP RST flag to simulate ConnectionResetError on the client side.
|
|
642
|
+
finish_thread(send_connection_reset=True)
|
|
643
|
+
else:
|
|
644
|
+
finish_thread()
|
|
645
|
+
|
|
646
|
+
return 'return'
|
|
647
|
+
|
|
648
|
+
return None
|
|
649
|
+
|
|
650
|
+
|
|
338
651
|
def receive_send_start(
|
|
339
652
|
receiving_socket,
|
|
340
653
|
sending_socket = None,
|
|
@@ -360,146 +673,25 @@ def thread_worker_main(
|
|
|
360
673
|
raise ValueError(f"Unknown side of the socket: {receiving_socket}")
|
|
361
674
|
|
|
362
675
|
while True:
|
|
363
|
-
|
|
364
|
-
# pass the socket connect to responder.
|
|
365
|
-
if side == 'Service' and client_connection_message:
|
|
366
|
-
client_message = client_connection_message
|
|
367
|
-
|
|
368
|
-
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
369
|
-
print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]", logger=network_logger,
|
|
370
|
-
logger_method='info')
|
|
371
|
-
|
|
372
|
-
received_raw_data = None
|
|
373
|
-
else:
|
|
374
|
-
client_message.reinitialize_dynamic_vars()
|
|
676
|
+
client_message.reinitialize_dynamic_vars()
|
|
375
677
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
else:
|
|
380
|
-
server_receive_count += 1
|
|
381
|
-
current_count = server_receive_count
|
|
382
|
-
|
|
383
|
-
# Getting current time of message received, either from client or service.
|
|
384
|
-
client_message.timestamp = datetime.now()
|
|
385
|
-
|
|
386
|
-
# # No need to receive on service socket if we're in offline mode, because there is no service to connect to.
|
|
387
|
-
# if config_static.MainConfig.offline and side == 'Service':
|
|
388
|
-
# print_api("Offline Mode, skipping receiving on service socket.", logger=network_logger,
|
|
389
|
-
# logger_method='info')
|
|
390
|
-
# else:
|
|
391
|
-
|
|
392
|
-
network_logger.info(
|
|
393
|
-
f"Initializing Receiver for {side} cycle: {str(current_count)}")
|
|
394
|
-
|
|
395
|
-
# Getting message from the client over the socket using specific class.
|
|
396
|
-
received_raw_data, is_socket_closed, error_message = receiver.Receiver(
|
|
397
|
-
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
398
|
-
|
|
399
|
-
# In case of client socket, we'll process the raw data specifically for the client.
|
|
400
|
-
if side == 'Client':
|
|
401
|
-
process_client_raw_data(received_raw_data, error_message, client_message)
|
|
402
|
-
client_message.action = 'client_receive'
|
|
403
|
-
# In case of service socket, we'll process the raw data specifically for the service.
|
|
404
|
-
else:
|
|
405
|
-
process_server_raw_data(received_raw_data, error_message, client_message)
|
|
406
|
-
client_message.action = 'service_receive'
|
|
407
|
-
|
|
408
|
-
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
409
|
-
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
410
|
-
if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
|
|
411
|
-
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
412
|
-
logger_method='info')
|
|
413
|
-
return
|
|
414
|
-
|
|
415
|
-
# We will record only if there was no closing signal, because if there was, it means that we initiated
|
|
416
|
-
# the close on the opposite socket.
|
|
417
|
-
record_and_statistics_write(client_message)
|
|
418
|
-
|
|
419
|
-
# if is_socket_closed:
|
|
420
|
-
# exception_or_close_in_receiving_thread = True
|
|
421
|
-
# finish_thread()
|
|
422
|
-
# return
|
|
423
|
-
|
|
424
|
-
# Now send it to requester/responder.
|
|
425
|
-
if side == 'Client':
|
|
426
|
-
# Send to requester.
|
|
427
|
-
bytes_to_send_list: list[bytes] = create_requester_request(
|
|
428
|
-
client_message, sending_socket=sending_socket)
|
|
429
|
-
|
|
430
|
-
# If we're in offline mode, then we'll put the request to the responder right away.
|
|
431
|
-
if config_static.MainConfig.offline:
|
|
432
|
-
print_api("Offline Mode, sending to responder directly.", logger=network_logger,
|
|
433
|
-
logger_method='info')
|
|
434
|
-
process_client_raw_data(bytes_to_send_list[0], error_message, client_message)
|
|
435
|
-
bytes_to_send_list = create_responder_response(client_message)
|
|
436
|
-
elif side == 'Service':
|
|
437
|
-
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
438
|
-
print_api(f"Got responses from responder, count: [{len(bytes_to_send_list)}]",
|
|
439
|
-
logger=network_logger,
|
|
440
|
-
logger_method='info')
|
|
441
|
-
else:
|
|
442
|
-
raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
# If nothing was passed from the responder, and the client message is the connection message, then we'll skip to the next iteration.
|
|
446
|
-
if not bytes_to_send_list and client_connection_message:
|
|
678
|
+
if side == 'Service' and client_connection_message:
|
|
679
|
+
result: Literal['continue', 'return'] | None = (
|
|
680
|
+
receive_send_service_connect(client_connection_message, sending_socket))
|
|
447
681
|
client_connection_message = None
|
|
448
|
-
continue
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
client_message
|
|
454
|
-
|
|
682
|
+
if result == 'continue':
|
|
683
|
+
continue
|
|
684
|
+
elif side == 'Client' and config_static.MainConfig.is_offline:
|
|
685
|
+
result: Literal['return'] | None = receive_send_client_offline(client_message, receiving_socket, sending_socket)
|
|
686
|
+
elif side == 'Client':
|
|
687
|
+
result: Literal['return'] | None = receive_send_client(client_message, receiving_socket, sending_socket)
|
|
688
|
+
elif side == 'Service':
|
|
689
|
+
result: Literal['return'] | None = receive_send_service(client_message, receiving_socket, sending_socket)
|
|
690
|
+
else:
|
|
691
|
+
raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
|
|
455
692
|
|
|
456
|
-
|
|
457
|
-
client_message.request_raw_bytes = bytes_to_send_single
|
|
458
|
-
else:
|
|
459
|
-
client_message.response_raw_bytes = bytes_to_send_single
|
|
460
|
-
|
|
461
|
-
# This records the requester or responder output, only if it is not the same as the original
|
|
462
|
-
# message.
|
|
463
|
-
if bytes_to_send_single != received_raw_data:
|
|
464
|
-
if side == 'Client':
|
|
465
|
-
client_message.action = 'client_requester'
|
|
466
|
-
|
|
467
|
-
if config_static.MainConfig.offline:
|
|
468
|
-
client_message.action = 'client_responder_offline'
|
|
469
|
-
elif side == 'Service':
|
|
470
|
-
client_message.action = 'service_responder'
|
|
471
|
-
record_and_statistics_write(client_message)
|
|
472
|
-
|
|
473
|
-
# If we're in offline mode, it means we're in the client thread, and we'll send the
|
|
474
|
-
# bytes back to the client socket.
|
|
475
|
-
if config_static.MainConfig.offline:
|
|
476
|
-
error_on_send: str = sender.Sender(
|
|
477
|
-
ssl_socket=receiving_socket, class_message=bytes_to_send_single,
|
|
478
|
-
logger=network_logger).send()
|
|
479
|
-
else:
|
|
480
|
-
error_on_send: str = sender.Sender(
|
|
481
|
-
ssl_socket=sending_socket, class_message=bytes_to_send_single,
|
|
482
|
-
logger=network_logger).send()
|
|
483
|
-
|
|
484
|
-
if error_on_send:
|
|
485
|
-
client_message.reinitialize_dynamic_vars()
|
|
486
|
-
client_message.errors.append(error_on_send)
|
|
487
|
-
client_message.timestamp = datetime.now()
|
|
488
|
-
if side == 'Client':
|
|
489
|
-
client_message.action = 'service_send'
|
|
490
|
-
else:
|
|
491
|
-
client_message.action = 'client_send'
|
|
492
|
-
|
|
493
|
-
record_and_statistics_write(client_message)
|
|
494
|
-
|
|
495
|
-
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
496
|
-
if is_socket_closed or error_on_send:
|
|
497
|
-
exception_or_close_in_receiving_thread = True
|
|
498
|
-
finish_thread()
|
|
693
|
+
if result == 'return':
|
|
499
694
|
return
|
|
500
|
-
|
|
501
|
-
# For next iteration to start in case this iteration was responsible to process connection message, we need to set it to None.
|
|
502
|
-
client_connection_message = None
|
|
503
695
|
except Exception as exc:
|
|
504
696
|
# If the sockets were already closed, then there is nothing to do here besides log.
|
|
505
697
|
# if (isinstance(exc, OSError) and exc.errno == 10038 and
|
|
@@ -516,7 +708,7 @@ def thread_worker_main(
|
|
|
516
708
|
):
|
|
517
709
|
nonlocal exception_or_close_in_receiving_thread
|
|
518
710
|
|
|
519
|
-
exception_or_close_in_receiving_thread=True
|
|
711
|
+
exception_or_close_in_receiving_thread = True
|
|
520
712
|
# handle_exceptions(exc, client_message, recorded)
|
|
521
713
|
exception_message = tracebacks.get_as_string(one_line=True)
|
|
522
714
|
|
|
@@ -545,24 +737,34 @@ def thread_worker_main(
|
|
|
545
737
|
|
|
546
738
|
finish_thread()
|
|
547
739
|
|
|
740
|
+
# Add custom attribute to the exception.
|
|
741
|
+
exc.engine_name = client_message.engine_name
|
|
742
|
+
|
|
548
743
|
# After the socket clean up, we will still raise the exception to the main thread.
|
|
549
744
|
raise exc
|
|
550
745
|
|
|
551
746
|
# ================================================================================================================
|
|
552
747
|
# This is the start of the thread_worker_main function
|
|
553
|
-
|
|
554
748
|
network_logger = loggingw.get_logger_with_level(config_static.MainConfig.LOGGER_NAME)
|
|
555
749
|
|
|
556
750
|
# Only protocols that are encrypted with TLS have the server name attribute.
|
|
557
751
|
if is_tls:
|
|
558
752
|
# Get current destination domain
|
|
559
753
|
server_name = client_socket.server_hostname
|
|
754
|
+
# If there is no server name from the TLS handshake, then we'll use the domain from the DNS.
|
|
755
|
+
if not server_name:
|
|
756
|
+
server_name = domain_from_dns
|
|
560
757
|
# If the protocol is not TLS, then we'll use the domain from the DNS.
|
|
561
758
|
else:
|
|
562
759
|
server_name = domain_from_dns
|
|
563
760
|
|
|
564
761
|
thread_id = threads.current_thread_id()
|
|
565
762
|
|
|
763
|
+
process_name: str = multiprocessing.current_process().name
|
|
764
|
+
current_thread = threading.current_thread()
|
|
765
|
+
thread_process_name: str = f"{process_name} | {current_thread.name}"
|
|
766
|
+
current_thread.name = thread_process_name
|
|
767
|
+
|
|
566
768
|
# This is the main protocols.
|
|
567
769
|
protocol: str = str()
|
|
568
770
|
# This is the secondary protocol in the websocket.
|
|
@@ -585,6 +787,13 @@ def thread_worker_main(
|
|
|
585
787
|
responder = found_domain_module.responder_class_object()
|
|
586
788
|
recorder = found_domain_module.recorder_class_object(record_path=config_static.LogRec.recordings_path)
|
|
587
789
|
|
|
790
|
+
engine_name: str = recorder.engine_name
|
|
791
|
+
|
|
792
|
+
for engine in engines_list:
|
|
793
|
+
if engine.engine_name == engine_name:
|
|
794
|
+
responder.add_args(engine=engine)
|
|
795
|
+
break
|
|
796
|
+
|
|
588
797
|
network_logger.info(f"Assigned Modules for [{server_name}]: "
|
|
589
798
|
f"{parser.__name__}, "
|
|
590
799
|
f"{requester.__class__.__name__}, "
|
|
@@ -600,9 +809,14 @@ def thread_worker_main(
|
|
|
600
809
|
http_path_queue: queue.Queue = queue.Queue()
|
|
601
810
|
|
|
602
811
|
try:
|
|
603
|
-
engine_name: str = recorder.engine_name
|
|
604
812
|
client_ip, source_port = client_socket.getpeername()
|
|
605
|
-
|
|
813
|
+
|
|
814
|
+
try:
|
|
815
|
+
client_name: str = socket.gethostbyaddr(client_ip)[0]
|
|
816
|
+
# This can happen if the host changed IP address, but it wasn't propagated over DHCP.
|
|
817
|
+
except socket.herror:
|
|
818
|
+
client_name = ""
|
|
819
|
+
|
|
606
820
|
client_name = client_name.lower()
|
|
607
821
|
destination_port: int = client_socket.getsockname()[1]
|
|
608
822
|
destination_port_str: str = str(destination_port)
|
|
@@ -613,7 +827,7 @@ def thread_worker_main(
|
|
|
613
827
|
_, destination_port_str = initialize_engines.get_ipv4_from_engine_on_connect_port(on_port_connect_value)
|
|
614
828
|
destination_port: int = int(destination_port_str)
|
|
615
829
|
|
|
616
|
-
if config_static.MainConfig.
|
|
830
|
+
if config_static.MainConfig.is_offline:
|
|
617
831
|
# If in offline mode, then we'll get the TCP server's input address.
|
|
618
832
|
server_ip = client_socket.getsockname()[0]
|
|
619
833
|
else:
|
|
@@ -623,7 +837,7 @@ def thread_worker_main(
|
|
|
623
837
|
network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
|
|
624
838
|
f"Destination service: [{server_name}:{destination_port}]")
|
|
625
839
|
|
|
626
|
-
|
|
840
|
+
origin_service_client_instance = None
|
|
627
841
|
client_receive_count: int = 0
|
|
628
842
|
server_receive_count: int = 0
|
|
629
843
|
client_message_connection = client_message_first_start()
|
|
@@ -635,11 +849,11 @@ def thread_worker_main(
|
|
|
635
849
|
client_message_connection.action = 'service_connect'
|
|
636
850
|
client_message_connection.timestamp = datetime.now()
|
|
637
851
|
|
|
638
|
-
if config_static.MainConfig.
|
|
852
|
+
if config_static.MainConfig.is_offline:
|
|
639
853
|
client_message_connection.info = 'Offline Mode'
|
|
640
854
|
else:
|
|
641
|
-
|
|
642
|
-
service_socket_instance, connection_error =
|
|
855
|
+
origin_service_client_instance = create_client_socket(client_message_connection)
|
|
856
|
+
service_socket_instance, connection_error = origin_service_client_instance.service_connection()
|
|
643
857
|
|
|
644
858
|
if connection_error:
|
|
645
859
|
client_message_connection.errors.append(connection_error)
|
|
@@ -654,22 +868,22 @@ def thread_worker_main(
|
|
|
654
868
|
client_thread = threading.Thread(
|
|
655
869
|
target=receive_send_start,
|
|
656
870
|
args=(client_socket, service_socket_instance, client_exception_queue, None),
|
|
657
|
-
name=f"Thread-{thread_id}-Client",
|
|
871
|
+
name=f"{process_name} | Thread-{thread_id}-Client",
|
|
658
872
|
daemon=True)
|
|
659
873
|
client_thread.start()
|
|
660
874
|
|
|
661
875
|
service_exception_queue: queue.Queue = queue.Queue()
|
|
662
|
-
if not config_static.MainConfig.
|
|
876
|
+
if not config_static.MainConfig.is_offline:
|
|
663
877
|
service_thread = threading.Thread(
|
|
664
878
|
target=receive_send_start,
|
|
665
879
|
args=(service_socket_instance, client_socket, service_exception_queue, client_message_connection),
|
|
666
|
-
name=f"Thread-{thread_id}-Service",
|
|
880
|
+
name=f"{process_name} | Thread-{thread_id}-Service",
|
|
667
881
|
daemon=True)
|
|
668
882
|
service_thread.start()
|
|
669
883
|
|
|
670
884
|
client_thread.join()
|
|
671
885
|
# If we're in offline mode, then there is no service thread.
|
|
672
|
-
if not config_static.MainConfig.
|
|
886
|
+
if not config_static.MainConfig.is_offline:
|
|
673
887
|
# If we're not in offline mode, then we'll wait for the service thread to finish.
|
|
674
888
|
# noinspection PyUnboundLocalVariable
|
|
675
889
|
service_thread.join()
|