atomicshop 3.3.28__py3-none-any.whl → 3.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of atomicshop might be problematic. Click here for more details.

Files changed (99) hide show
  1. atomicshop/__init__.py +1 -1
  2. atomicshop/a_mains/get_local_tcp_ports.py +85 -0
  3. atomicshop/a_mains/install_ca_certificate.py +172 -0
  4. atomicshop/a_mains/process_from_port.py +119 -0
  5. atomicshop/a_mains/set_default_dns_gateway.py +90 -0
  6. atomicshop/basics/strings.py +1 -1
  7. atomicshop/certificates.py +2 -2
  8. atomicshop/dns.py +26 -28
  9. atomicshop/etws/traces/trace_tcp.py +1 -2
  10. atomicshop/mitm/centered_settings.py +133 -0
  11. atomicshop/mitm/config_static.py +18 -43
  12. atomicshop/mitm/connection_thread_worker.py +376 -162
  13. atomicshop/mitm/engines/__parent/recorder___parent.py +1 -1
  14. atomicshop/mitm/engines/__parent/requester___parent.py +1 -1
  15. atomicshop/mitm/engines/__parent/responder___parent.py +15 -2
  16. atomicshop/mitm/engines/create_module_template.py +1 -2
  17. atomicshop/mitm/import_config.py +79 -88
  18. atomicshop/mitm/initialize_engines.py +1 -2
  19. atomicshop/mitm/message.py +5 -4
  20. atomicshop/mitm/mitm_main.py +222 -121
  21. atomicshop/mitm/recs_files.py +61 -5
  22. atomicshop/mitm/ssh_tester.py +82 -0
  23. atomicshop/networks.py +108 -93
  24. atomicshop/package_mains_processor.py +84 -0
  25. atomicshop/permissions/ubuntu_permissions.py +47 -0
  26. atomicshop/print_api.py +3 -5
  27. atomicshop/python_functions.py +23 -108
  28. atomicshop/speech_recognize.py +8 -0
  29. atomicshop/ssh_remote.py +115 -51
  30. atomicshop/web.py +20 -7
  31. atomicshop/web_apis/google_llm.py +22 -14
  32. atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
  33. atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +2 -1
  34. atomicshop/wrappers/dockerw/dockerw.py +2 -2
  35. atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +5 -5
  36. atomicshop/wrappers/githubw.py +175 -63
  37. atomicshop/wrappers/loggingw/handlers.py +1 -1
  38. atomicshop/wrappers/loggingw/loggingw.py +17 -1
  39. atomicshop/wrappers/netshw.py +124 -3
  40. atomicshop/wrappers/playwrightw/scenarios.py +1 -1
  41. atomicshop/wrappers/powershell_networking.py +80 -0
  42. atomicshop/wrappers/psutilw/psutil_networks.py +9 -0
  43. atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
  44. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +3 -105
  45. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +3 -57
  46. atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +12 -27
  47. atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +15 -9
  48. atomicshop/wrappers/socketw/certificator.py +19 -9
  49. atomicshop/wrappers/socketw/creator.py +30 -7
  50. atomicshop/wrappers/socketw/dns_server.py +6 -6
  51. atomicshop/wrappers/socketw/exception_wrapper.py +3 -3
  52. atomicshop/wrappers/socketw/process_getter.py +86 -0
  53. atomicshop/wrappers/socketw/receiver.py +29 -9
  54. atomicshop/wrappers/socketw/sender.py +10 -9
  55. atomicshop/wrappers/socketw/sni.py +23 -6
  56. atomicshop/wrappers/socketw/{base.py → socket_base.py} +33 -1
  57. atomicshop/wrappers/socketw/socket_client.py +6 -8
  58. atomicshop/wrappers/socketw/socket_wrapper.py +82 -21
  59. atomicshop/wrappers/socketw/ssl_base.py +6 -2
  60. atomicshop/wrappers/win_auditw.py +189 -0
  61. {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/METADATA +25 -30
  62. {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/RECORD +74 -88
  63. atomicshop/_basics_temp.py +0 -101
  64. atomicshop/a_installs/ubuntu/docker_rootless.py +0 -11
  65. atomicshop/a_installs/ubuntu/docker_sudo.py +0 -11
  66. atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
  67. atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
  68. atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
  69. atomicshop/addons/package_setup/Setup.cmd +0 -7
  70. atomicshop/archiver/__init__.py +0 -0
  71. atomicshop/archiver/_search_in_zip.py +0 -189
  72. atomicshop/archiver/search_in_archive.py +0 -284
  73. atomicshop/archiver/sevenz_app_w.py +0 -86
  74. atomicshop/archiver/sevenzs.py +0 -73
  75. atomicshop/archiver/shutils.py +0 -34
  76. atomicshop/archiver/zips.py +0 -353
  77. atomicshop/file_types.py +0 -24
  78. atomicshop/pbtkmultifile_argparse.py +0 -88
  79. atomicshop/script_as_string_processor.py +0 -42
  80. atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
  81. atomicshop/ssh_scripts/process_from_port.py +0 -27
  82. atomicshop/wrappers/_process_wrapper_curl.py +0 -27
  83. atomicshop/wrappers/_process_wrapper_tar.py +0 -21
  84. atomicshop/wrappers/dockerw/install_docker.py +0 -449
  85. atomicshop/wrappers/ffmpegw.py +0 -125
  86. atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
  87. atomicshop/wrappers/socketw/get_process.py +0 -123
  88. /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
  89. /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
  90. /atomicshop/{addons → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
  91. /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
  92. /atomicshop/{addons → a_mains/addons}/process_list/compile.cmd +0 -0
  93. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.dll +0 -0
  94. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.exp +0 -0
  95. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.lib +0 -0
  96. /atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +0 -0
  97. {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/WHEEL +0 -0
  98. {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/licenses/LICENSE.txt +0 -0
  99. {atomicshop-3.3.28.dist-info → atomicshop-3.10.0.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,13 @@
1
+ import multiprocessing
1
2
  from datetime import datetime
2
3
  import threading
3
4
  import queue
4
5
  import socket
6
+ import ssl
7
+ from typing import Literal
8
+ import struct
5
9
 
6
- from ..wrappers.socketw import receiver, sender, socket_client, base
10
+ from ..wrappers.socketw import receiver, sender, socket_client, socket_base
7
11
  from .. import websocket_parse, ip_addresses
8
12
  from ..http_parse import HTTPRequestParse, HTTPResponseParse
9
13
  from ..basics import threads, tracebacks
@@ -25,7 +29,7 @@ def thread_worker_main(
25
29
  tls_version: str,
26
30
  domain_from_dns,
27
31
  statistics_writer,
28
- engines_list,
32
+ engines_list: list[initialize_engines.ModuleCategory],
29
33
 
30
34
  # These parameters come from the main mitm module.
31
35
  config_static: cf
@@ -168,32 +172,55 @@ def thread_worker_main(
168
172
  def parse_websocket(raw_bytes):
169
173
  return websocket_frame_parser.parse_frame_bytes(raw_bytes)
170
174
 
171
- def finish_thread():
175
+ def finish_thread(send_connection_reset: bool = False):
176
+ """
177
+ Finishing the thread, closing sockets.
178
+
179
+ :param send_connection_reset: Whether to send TCP RST flag to the client when closing the socket.
180
+ Basically what happens is that the server socket is closed abruptly sending us any Connection*Error exception.
181
+ We can simulate the ConnectionResetError on the client side by sending TCP RST flag when closing the socket.
182
+ But not the ConnectionAbortedError, since it is caused by other reasons (local TCP stack and has nothing
183
+ to do with the remote server).
184
+ """
172
185
  # At this stage there could be several times that the same socket was used to the service server - we need to
173
186
  # close this socket as well if it still opened.
174
187
  # The first part of the condition is to check if the service socket was connected at all.
175
188
  # If the service socket couldn't connect, then the instance will be None.
176
189
  if service_socket_instance and service_socket_instance.fileno() != -1:
177
- if service_client.socket_instance:
178
- service_client.close_socket()
190
+ if origin_service_client_instance.socket_instance:
191
+ origin_service_client_instance.close_socket()
179
192
 
180
193
  # If client socket is still opened - close
181
194
  if client_socket.fileno() != -1:
195
+ if send_connection_reset:
196
+ # Abort the connection
197
+ linger = struct.pack('ii', 1, 0)
198
+ client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger)
182
199
  client_socket.close()
183
- network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
200
+
201
+ if send_connection_reset:
202
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}] with TCP RST flag (Sent ConnectionResetError)...")
203
+ else:
204
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
184
205
 
185
206
  network_logger.info("Thread Finished. Will continue listening on the Main thread")
186
207
 
187
208
  def create_requester_request(
188
209
  client_message: ClientMessage,
189
210
  sending_socket: socket.socket
190
- ) -> list[bytes]:
211
+ ) -> tuple[bytes, bool]:
212
+ request_received_raw: bytes = client_message.request_raw_bytes
191
213
  request_custom_raw: bytes = requester.create_request(client_message, sending_socket=sending_socket)
192
214
 
193
- # Output first 100 characters of the request.
194
- requester.logger.info(f"{request_custom_raw[0: 100]}...")
215
+ if request_custom_raw is None or request_received_raw == request_custom_raw:
216
+ is_requester_worked: bool = False
217
+ else:
218
+ is_requester_worked: bool = True
219
+
220
+ # Output first 100 characters of the request.
221
+ requester.logger.info(f"{request_custom_raw[0: 100]}...")
195
222
 
196
- return [request_custom_raw]
223
+ return request_custom_raw, is_requester_worked
197
224
 
198
225
  def create_responder_response(client_message: ClientMessage) -> list[bytes]:
199
226
  if client_message.action == 'service_connect':
@@ -201,13 +228,19 @@ def thread_worker_main(
201
228
  else:
202
229
  # If we're in offline mode, and it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
203
230
  # response automatically.
204
- if config_static.MainConfig.offline and protocol == 'Websocket' and client_receive_count == 1:
231
+ if config_static.MainConfig.is_offline and protocol == 'Websocket' and client_receive_count == 1:
205
232
  responses: list = list()
206
233
  responses.append(
207
234
  websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
235
+ responder.logger.info(f"Generated automatic WebSocket response in Offline Mode.")
208
236
  else:
209
237
  # Creating response for parsed message and printing
210
- responses: list = responder.create_response(client_message)
238
+ responder_responses: list = responder.create_response(client_message)
239
+ if responder_responses is None:
240
+ responses: list = [client_message.response_raw_bytes]
241
+ else:
242
+ responses: list = responder_responses
243
+ responder.logger.info(f"Generated {len(responses)} responses from responder.")
211
244
 
212
245
  # Output first 100 characters of all the responses in the list.
213
246
  for response_raw_bytes_single in responses:
@@ -242,7 +275,7 @@ def thread_worker_main(
242
275
  else:
243
276
  # If we're on localhost, then use external services list in order to resolve the domain:
244
277
  # config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
245
- if client_message.client_ip in base.THIS_DEVICE_IP_LIST:
278
+ if client_message.client_ip in socket_base.THIS_DEVICE_IP_LIST:
246
279
  service_client_instance = socket_client.SocketClient(
247
280
  service_name=client_message.server_name,
248
281
  service_port=client_message.destination_port,
@@ -330,11 +363,291 @@ def thread_worker_main(
330
363
  client_message.destination_port = destination_port
331
364
  client_message.server_name = server_name
332
365
  client_message.thread_id = thread_id
366
+ client_message.thread_process = thread_process_name
333
367
  client_message.process_name = process_commandline
334
368
  client_message.engine_name = engine_name
335
369
 
336
370
  return client_message
337
371
 
372
+ def receive_send_service_connect(
373
+ client_connection_message: ClientMessage,
374
+ sending_socket: ssl.SSLSocket | socket.socket
375
+ ) -> Literal['continue', 'return'] | None:
376
+
377
+ nonlocal exception_or_close_in_receiving_thread
378
+
379
+ client_message = client_connection_message
380
+
381
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
382
+ print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]",
383
+ logger=network_logger, logger_method='info')
384
+
385
+ # If the client message is the connection message, then we'll skip to the next iteration.
386
+ if not bytes_to_send_list:
387
+ return 'continue'
388
+
389
+ # is_socket_closed: bool = False
390
+ error_on_send: str = str()
391
+ for bytes_to_send_single in bytes_to_send_list:
392
+ client_message.reinitialize_dynamic_vars()
393
+ client_message.timestamp = datetime.now()
394
+ client_message.response_raw_bytes = bytes_to_send_single
395
+ client_message.action = 'service_responder'
396
+ record_and_statistics_write(client_message)
397
+
398
+ # Send the bytes back to the client socket.
399
+ error_on_send: str = sender.Sender(
400
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
401
+ logger=network_logger).send()
402
+
403
+ if error_on_send:
404
+ client_message.reinitialize_dynamic_vars()
405
+ client_message.errors.append(error_on_send)
406
+ client_message.timestamp = datetime.now()
407
+ client_message.action = 'client_send'
408
+ record_and_statistics_write(client_message)
409
+
410
+ if error_on_send:
411
+ exception_or_close_in_receiving_thread = True
412
+ finish_thread()
413
+ return 'return'
414
+
415
+ return None
416
+
417
+ def receive_send_client_offline(
418
+ client_message: ClientMessage,
419
+ receiving_socket: ssl.SSLSocket | socket.socket,
420
+ sending_socket: ssl.SSLSocket | socket.socket
421
+ ) -> Literal['return'] | None:
422
+ nonlocal exception_or_close_in_receiving_thread
423
+ nonlocal client_receive_count
424
+
425
+ client_receive_count += 1
426
+
427
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
428
+ client_message.timestamp = datetime.now()
429
+
430
+ received_raw_data, is_socket_closed, error_message = receiver.Receiver(
431
+ ssl_socket=receiving_socket, logger=network_logger).receive()
432
+
433
+ process_client_raw_data(received_raw_data, error_message, client_message)
434
+ client_message.action = 'client_receive'
435
+ record_and_statistics_write(client_message)
436
+ if error_message:
437
+ print_api(error_message, logger=network_logger, logger_method='critical')
438
+
439
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
440
+ # the socket was closed by the other side, it means that the service thread closed the socket.
441
+ if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
442
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
443
+ logger_method='info')
444
+ return 'return'
445
+
446
+ # If the socket was closed on receive, and we're in offline mode, then we'll finish the thread right away.
447
+ # Since nothing more can be done, like responding to service or using requester.
448
+ if is_socket_closed:
449
+ exception_or_close_in_receiving_thread = True
450
+ finish_thread()
451
+ return 'return'
452
+
453
+ # Send to requester.
454
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
455
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
456
+ # We will not process the raw data if requester didn't change anything.
457
+ if is_requester_worked:
458
+ client_message.reinitialize_dynamic_vars()
459
+ client_message.timestamp = datetime.now()
460
+ client_message.request_raw_bytes = request_custom_raw
461
+ client_message.action = 'client_requester'
462
+ process_client_raw_data(request_custom_raw, error_message, client_message)
463
+ record_and_statistics_write(client_message)
464
+
465
+ print_api("Offline Mode, sending to responder directly.", logger=network_logger,
466
+ logger_method='info')
467
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
468
+
469
+ error_on_send: str = str()
470
+ for bytes_to_send_single in bytes_to_send_list:
471
+ client_message.reinitialize_dynamic_vars()
472
+ client_message.timestamp = datetime.now()
473
+ client_message.response_raw_bytes = bytes_to_send_single
474
+ client_message.action = 'client_responder_offline'
475
+ process_server_raw_data(bytes_to_send_single, '', client_message)
476
+ record_and_statistics_write(client_message)
477
+
478
+ error_on_send: str = sender.Sender(
479
+ ssl_socket=receiving_socket, bytes_to_send=bytes_to_send_single,
480
+ logger=network_logger).send()
481
+
482
+ if error_on_send:
483
+ client_message.reinitialize_dynamic_vars()
484
+ client_message.errors.append(error_on_send)
485
+ client_message.timestamp = datetime.now()
486
+ client_message.action = 'service_send'
487
+
488
+ record_and_statistics_write(client_message)
489
+
490
+ # If the socket was closed on message receive, then we'll break the loop only after send.
491
+ if is_socket_closed or error_on_send:
492
+ exception_or_close_in_receiving_thread = True
493
+ finish_thread()
494
+ return 'return'
495
+
496
+ return None
497
+
498
+ def receive_send_client(
499
+ client_message: ClientMessage,
500
+ receiving_socket: ssl.SSLSocket | socket.socket,
501
+ sending_socket: ssl.SSLSocket | socket.socket
502
+ ) -> Literal['return'] | None:
503
+
504
+ nonlocal exception_or_close_in_receiving_thread
505
+ nonlocal client_receive_count
506
+
507
+ client_receive_count += 1
508
+
509
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
510
+
511
+ # Getting message from the client over the socket using specific class.
512
+ client_message.timestamp = datetime.now()
513
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
514
+ ssl_socket=receiving_socket, logger=network_logger).receive()
515
+
516
+ process_client_raw_data(received_raw_data, error_on_receive, client_message)
517
+ client_message.action = 'client_receive'
518
+
519
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
520
+ # the socket was closed by the other side, it means that the service thread closed the socket.
521
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
522
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
523
+ logger_method='info')
524
+ return 'return'
525
+
526
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
527
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
528
+ record_and_statistics_write(client_message)
529
+ if error_on_receive:
530
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
531
+
532
+ # At this point if the socket was closed on receive, then there's no point to send anything to the service.
533
+ # So, we check for 'is_socket_closed' here.
534
+ error_on_send: str = str()
535
+ if not is_socket_closed:
536
+ # Send to requester.
537
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
538
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
539
+ # We will not process the raw data if requester didn't change anything.
540
+ if is_requester_worked:
541
+ client_message.reinitialize_dynamic_vars()
542
+ client_message.timestamp = datetime.now()
543
+ client_message.request_raw_bytes = request_custom_raw
544
+ client_message.action = 'client_requester'
545
+ process_client_raw_data(request_custom_raw, error_on_receive, client_message)
546
+ record_and_statistics_write(client_message)
547
+
548
+ error_on_send: str = sender.Sender(
549
+ ssl_socket=sending_socket, bytes_to_send=client_message.request_raw_bytes,
550
+ logger=network_logger).send()
551
+
552
+ if error_on_send:
553
+ client_message.reinitialize_dynamic_vars()
554
+ client_message.errors.append(error_on_send)
555
+ client_message.timestamp = datetime.now()
556
+ client_message.action = 'service_send'
557
+ record_and_statistics_write(client_message)
558
+
559
+ # If the socket was closed on message receive, then we'll break the loop only after send.
560
+ if is_socket_closed or error_on_send:
561
+ exception_or_close_in_receiving_thread = True
562
+ finish_thread()
563
+ return 'return'
564
+
565
+ return None
566
+
567
+ def receive_send_service(
568
+ client_message: ClientMessage,
569
+ receiving_socket: ssl.SSLSocket | socket.socket,
570
+ sending_socket: ssl.SSLSocket | socket.socket
571
+ ) -> Literal['return'] | None:
572
+
573
+ nonlocal exception_or_close_in_receiving_thread
574
+ nonlocal server_receive_count
575
+
576
+ server_receive_count += 1
577
+
578
+ network_logger.info(f"Initializing Receiver for Service cycle: {str(server_receive_count)}")
579
+
580
+ # Getting message from the client over the socket using specific class.
581
+ client_message.timestamp = datetime.now()
582
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
583
+ ssl_socket=receiving_socket, logger=network_logger).receive()
584
+
585
+ process_server_raw_data(received_raw_data, error_on_receive, client_message)
586
+ client_message.action = 'service_receive'
587
+
588
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
589
+ # the socket was closed by the other side, it means that the service thread closed the socket.
590
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
591
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
592
+ logger_method='info')
593
+ return 'return'
594
+
595
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
596
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
597
+ record_and_statistics_write(client_message)
598
+ if error_on_receive:
599
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
600
+
601
+ # At this stage, we have received the response from the service, but there was an exception/error or the socket was simply closed.
602
+ # Meaning the 'received_raw_data' is None for exception or b'' for just closed socket.
603
+ # So, there's no point to send anything back to the client.
604
+ # Close both sockets and finish the threads.
605
+ # So, we check for 'is_socket_closed' here.
606
+ error_on_send: str = str()
607
+ if not is_socket_closed:
608
+ # Now send it to requester/responder.
609
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
610
+
611
+ # is_socket_closed: bool = False
612
+ for bytes_to_send_single in bytes_to_send_list:
613
+ client_message.reinitialize_dynamic_vars()
614
+ client_message.timestamp = datetime.now()
615
+ client_message.response_raw_bytes = bytes_to_send_single
616
+
617
+ # This records the requester or responder output, only if it is not the same as the original
618
+ # message.
619
+ if bytes_to_send_single != received_raw_data:
620
+ client_message.action = 'service_responder'
621
+ record_and_statistics_write(client_message)
622
+
623
+ error_on_send: str = sender.Sender(
624
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
625
+ logger=network_logger).send()
626
+
627
+ if error_on_send:
628
+ client_message.reinitialize_dynamic_vars()
629
+ client_message.errors.append(error_on_send)
630
+ client_message.timestamp = datetime.now()
631
+ client_message.action = 'client_send'
632
+
633
+ record_and_statistics_write(client_message)
634
+
635
+ # If the socket was closed on message receive, then we'll break the loop only after send.
636
+ if is_socket_closed or error_on_send:
637
+ exception_or_close_in_receiving_thread = True
638
+
639
+ if error_on_receive and 'Connection' in error_on_receive and 'Error' in error_on_receive:
640
+ # If there was a connection error on receive from client, and we're closing the socket to the client,
641
+ # then we'll send TCP RST flag to simulate ConnectionResetError on the client side.
642
+ finish_thread(send_connection_reset=True)
643
+ else:
644
+ finish_thread()
645
+
646
+ return 'return'
647
+
648
+ return None
649
+
650
+
338
651
  def receive_send_start(
339
652
  receiving_socket,
340
653
  sending_socket = None,
@@ -360,146 +673,25 @@ def thread_worker_main(
360
673
  raise ValueError(f"Unknown side of the socket: {receiving_socket}")
361
674
 
362
675
  while True:
363
- is_socket_closed: bool = False
364
- # pass the socket connect to responder.
365
- if side == 'Service' and client_connection_message:
366
- client_message = client_connection_message
367
-
368
- bytes_to_send_list: list[bytes] = create_responder_response(client_message)
369
- print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]", logger=network_logger,
370
- logger_method='info')
371
-
372
- received_raw_data = None
373
- else:
374
- client_message.reinitialize_dynamic_vars()
676
+ client_message.reinitialize_dynamic_vars()
375
677
 
376
- if side == 'Client':
377
- client_receive_count += 1
378
- current_count = client_receive_count
379
- else:
380
- server_receive_count += 1
381
- current_count = server_receive_count
382
-
383
- # Getting current time of message received, either from client or service.
384
- client_message.timestamp = datetime.now()
385
-
386
- # # No need to receive on service socket if we're in offline mode, because there is no service to connect to.
387
- # if config_static.MainConfig.offline and side == 'Service':
388
- # print_api("Offline Mode, skipping receiving on service socket.", logger=network_logger,
389
- # logger_method='info')
390
- # else:
391
-
392
- network_logger.info(
393
- f"Initializing Receiver for {side} cycle: {str(current_count)}")
394
-
395
- # Getting message from the client over the socket using specific class.
396
- received_raw_data, is_socket_closed, error_message = receiver.Receiver(
397
- ssl_socket=receiving_socket, logger=network_logger).receive()
398
-
399
- # In case of client socket, we'll process the raw data specifically for the client.
400
- if side == 'Client':
401
- process_client_raw_data(received_raw_data, error_message, client_message)
402
- client_message.action = 'client_receive'
403
- # In case of service socket, we'll process the raw data specifically for the service.
404
- else:
405
- process_server_raw_data(received_raw_data, error_message, client_message)
406
- client_message.action = 'service_receive'
407
-
408
- # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
409
- # the socket was closed by the other side, it means that the service thread closed the socket.
410
- if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
411
- print_api("Both sockets are closed, breaking the loop", logger=network_logger,
412
- logger_method='info')
413
- return
414
-
415
- # We will record only if there was no closing signal, because if there was, it means that we initiated
416
- # the close on the opposite socket.
417
- record_and_statistics_write(client_message)
418
-
419
- # if is_socket_closed:
420
- # exception_or_close_in_receiving_thread = True
421
- # finish_thread()
422
- # return
423
-
424
- # Now send it to requester/responder.
425
- if side == 'Client':
426
- # Send to requester.
427
- bytes_to_send_list: list[bytes] = create_requester_request(
428
- client_message, sending_socket=sending_socket)
429
-
430
- # If we're in offline mode, then we'll put the request to the responder right away.
431
- if config_static.MainConfig.offline:
432
- print_api("Offline Mode, sending to responder directly.", logger=network_logger,
433
- logger_method='info')
434
- process_client_raw_data(bytes_to_send_list[0], error_message, client_message)
435
- bytes_to_send_list = create_responder_response(client_message)
436
- elif side == 'Service':
437
- bytes_to_send_list: list[bytes] = create_responder_response(client_message)
438
- print_api(f"Got responses from responder, count: [{len(bytes_to_send_list)}]",
439
- logger=network_logger,
440
- logger_method='info')
441
- else:
442
- raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
443
-
444
-
445
- # If nothing was passed from the responder, and the client message is the connection message, then we'll skip to the next iteration.
446
- if not bytes_to_send_list and client_connection_message:
678
+ if side == 'Service' and client_connection_message:
679
+ result: Literal['continue', 'return'] | None = (
680
+ receive_send_service_connect(client_connection_message, sending_socket))
447
681
  client_connection_message = None
448
- continue
449
-
450
- # is_socket_closed: bool = False
451
- error_on_send: str = str()
452
- for bytes_to_send_single in bytes_to_send_list:
453
- client_message.reinitialize_dynamic_vars()
454
- client_message.timestamp = datetime.now()
682
+ if result == 'continue':
683
+ continue
684
+ elif side == 'Client' and config_static.MainConfig.is_offline:
685
+ result: Literal['return'] | None = receive_send_client_offline(client_message, receiving_socket, sending_socket)
686
+ elif side == 'Client':
687
+ result: Literal['return'] | None = receive_send_client(client_message, receiving_socket, sending_socket)
688
+ elif side == 'Service':
689
+ result: Literal['return'] | None = receive_send_service(client_message, receiving_socket, sending_socket)
690
+ else:
691
+ raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
455
692
 
456
- if side == 'Client':
457
- client_message.request_raw_bytes = bytes_to_send_single
458
- else:
459
- client_message.response_raw_bytes = bytes_to_send_single
460
-
461
- # This records the requester or responder output, only if it is not the same as the original
462
- # message.
463
- if bytes_to_send_single != received_raw_data:
464
- if side == 'Client':
465
- client_message.action = 'client_requester'
466
-
467
- if config_static.MainConfig.offline:
468
- client_message.action = 'client_responder_offline'
469
- elif side == 'Service':
470
- client_message.action = 'service_responder'
471
- record_and_statistics_write(client_message)
472
-
473
- # If we're in offline mode, it means we're in the client thread, and we'll send the
474
- # bytes back to the client socket.
475
- if config_static.MainConfig.offline:
476
- error_on_send: str = sender.Sender(
477
- ssl_socket=receiving_socket, class_message=bytes_to_send_single,
478
- logger=network_logger).send()
479
- else:
480
- error_on_send: str = sender.Sender(
481
- ssl_socket=sending_socket, class_message=bytes_to_send_single,
482
- logger=network_logger).send()
483
-
484
- if error_on_send:
485
- client_message.reinitialize_dynamic_vars()
486
- client_message.errors.append(error_on_send)
487
- client_message.timestamp = datetime.now()
488
- if side == 'Client':
489
- client_message.action = 'service_send'
490
- else:
491
- client_message.action = 'client_send'
492
-
493
- record_and_statistics_write(client_message)
494
-
495
- # If the socket was closed on message receive, then we'll break the loop only after send.
496
- if is_socket_closed or error_on_send:
497
- exception_or_close_in_receiving_thread = True
498
- finish_thread()
693
+ if result == 'return':
499
694
  return
500
-
501
- # For next iteration to start in case this iteration was responsible to process connection message, we need to set it to None.
502
- client_connection_message = None
503
695
  except Exception as exc:
504
696
  # If the sockets were already closed, then there is nothing to do here besides log.
505
697
  # if (isinstance(exc, OSError) and exc.errno == 10038 and
@@ -516,7 +708,7 @@ def thread_worker_main(
516
708
  ):
517
709
  nonlocal exception_or_close_in_receiving_thread
518
710
 
519
- exception_or_close_in_receiving_thread=True
711
+ exception_or_close_in_receiving_thread = True
520
712
  # handle_exceptions(exc, client_message, recorded)
521
713
  exception_message = tracebacks.get_as_string(one_line=True)
522
714
 
@@ -545,24 +737,34 @@ def thread_worker_main(
545
737
 
546
738
  finish_thread()
547
739
 
740
+ # Add custom attribute to the exception.
741
+ exc.engine_name = client_message.engine_name
742
+
548
743
  # After the socket clean up, we will still raise the exception to the main thread.
549
744
  raise exc
550
745
 
551
746
  # ================================================================================================================
552
747
  # This is the start of the thread_worker_main function
553
-
554
748
  network_logger = loggingw.get_logger_with_level(config_static.MainConfig.LOGGER_NAME)
555
749
 
556
750
  # Only protocols that are encrypted with TLS have the server name attribute.
557
751
  if is_tls:
558
752
  # Get current destination domain
559
753
  server_name = client_socket.server_hostname
754
+ # If there is no server name from the TLS handshake, then we'll use the domain from the DNS.
755
+ if not server_name:
756
+ server_name = domain_from_dns
560
757
  # If the protocol is not TLS, then we'll use the domain from the DNS.
561
758
  else:
562
759
  server_name = domain_from_dns
563
760
 
564
761
  thread_id = threads.current_thread_id()
565
762
 
763
+ process_name: str = multiprocessing.current_process().name
764
+ current_thread = threading.current_thread()
765
+ thread_process_name: str = f"{process_name} | {current_thread.name}"
766
+ current_thread.name = thread_process_name
767
+
566
768
  # This is the main protocols.
567
769
  protocol: str = str()
568
770
  # This is the secondary protocol in the websocket.
@@ -585,6 +787,13 @@ def thread_worker_main(
585
787
  responder = found_domain_module.responder_class_object()
586
788
  recorder = found_domain_module.recorder_class_object(record_path=config_static.LogRec.recordings_path)
587
789
 
790
+ engine_name: str = recorder.engine_name
791
+
792
+ for engine in engines_list:
793
+ if engine.engine_name == engine_name:
794
+ responder.add_args(engine=engine)
795
+ break
796
+
588
797
  network_logger.info(f"Assigned Modules for [{server_name}]: "
589
798
  f"{parser.__name__}, "
590
799
  f"{requester.__class__.__name__}, "
@@ -600,9 +809,14 @@ def thread_worker_main(
600
809
  http_path_queue: queue.Queue = queue.Queue()
601
810
 
602
811
  try:
603
- engine_name: str = recorder.engine_name
604
812
  client_ip, source_port = client_socket.getpeername()
605
- client_name: str = socket.gethostbyaddr(client_ip)[0]
813
+
814
+ try:
815
+ client_name: str = socket.gethostbyaddr(client_ip)[0]
816
+ # This can happen if the host changed IP address, but it wasn't propagated over DHCP.
817
+ except socket.herror:
818
+ client_name = ""
819
+
606
820
  client_name = client_name.lower()
607
821
  destination_port: int = client_socket.getsockname()[1]
608
822
  destination_port_str: str = str(destination_port)
@@ -613,7 +827,7 @@ def thread_worker_main(
613
827
  _, destination_port_str = initialize_engines.get_ipv4_from_engine_on_connect_port(on_port_connect_value)
614
828
  destination_port: int = int(destination_port_str)
615
829
 
616
- if config_static.MainConfig.offline:
830
+ if config_static.MainConfig.is_offline:
617
831
  # If in offline mode, then we'll get the TCP server's input address.
618
832
  server_ip = client_socket.getsockname()[0]
619
833
  else:
@@ -623,7 +837,7 @@ def thread_worker_main(
623
837
  network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
624
838
  f"Destination service: [{server_name}:{destination_port}]")
625
839
 
626
- service_client = None
840
+ origin_service_client_instance = None
627
841
  client_receive_count: int = 0
628
842
  server_receive_count: int = 0
629
843
  client_message_connection = client_message_first_start()
@@ -635,11 +849,11 @@ def thread_worker_main(
635
849
  client_message_connection.action = 'service_connect'
636
850
  client_message_connection.timestamp = datetime.now()
637
851
 
638
- if config_static.MainConfig.offline:
852
+ if config_static.MainConfig.is_offline:
639
853
  client_message_connection.info = 'Offline Mode'
640
854
  else:
641
- service_client = create_client_socket(client_message_connection)
642
- service_socket_instance, connection_error = service_client.service_connection()
855
+ origin_service_client_instance = create_client_socket(client_message_connection)
856
+ service_socket_instance, connection_error = origin_service_client_instance.service_connection()
643
857
 
644
858
  if connection_error:
645
859
  client_message_connection.errors.append(connection_error)
@@ -654,22 +868,22 @@ def thread_worker_main(
654
868
  client_thread = threading.Thread(
655
869
  target=receive_send_start,
656
870
  args=(client_socket, service_socket_instance, client_exception_queue, None),
657
- name=f"Thread-{thread_id}-Client",
871
+ name=f"{process_name} | Thread-{thread_id}-Client",
658
872
  daemon=True)
659
873
  client_thread.start()
660
874
 
661
875
  service_exception_queue: queue.Queue = queue.Queue()
662
- if not config_static.MainConfig.offline:
876
+ if not config_static.MainConfig.is_offline:
663
877
  service_thread = threading.Thread(
664
878
  target=receive_send_start,
665
879
  args=(service_socket_instance, client_socket, service_exception_queue, client_message_connection),
666
- name=f"Thread-{thread_id}-Service",
880
+ name=f"{process_name} | Thread-{thread_id}-Service",
667
881
  daemon=True)
668
882
  service_thread.start()
669
883
 
670
884
  client_thread.join()
671
885
  # If we're in offline mode, then there is no service thread.
672
- if not config_static.MainConfig.offline:
886
+ if not config_static.MainConfig.is_offline:
673
887
  # If we're not in offline mode, then we'll wait for the service thread to finish.
674
888
  # noinspection PyUnboundLocalVariable
675
889
  service_thread.join()