atomicshop 3.3.8__py3-none-any.whl → 3.10.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of atomicshop might be problematic. Click here for more details.

Files changed (120) hide show
  1. atomicshop/__init__.py +1 -1
  2. atomicshop/a_mains/get_local_tcp_ports.py +85 -0
  3. atomicshop/a_mains/install_ca_certificate.py +172 -0
  4. atomicshop/a_mains/process_from_port.py +119 -0
  5. atomicshop/a_mains/set_default_dns_gateway.py +90 -0
  6. atomicshop/basics/strings.py +1 -1
  7. atomicshop/certificates.py +2 -2
  8. atomicshop/dns.py +26 -28
  9. atomicshop/etws/traces/trace_tcp.py +1 -2
  10. atomicshop/mitm/centered_settings.py +133 -0
  11. atomicshop/mitm/config_static.py +22 -44
  12. atomicshop/mitm/connection_thread_worker.py +383 -165
  13. atomicshop/mitm/engines/__parent/recorder___parent.py +1 -1
  14. atomicshop/mitm/engines/__parent/requester___parent.py +1 -1
  15. atomicshop/mitm/engines/__parent/responder___parent.py +15 -2
  16. atomicshop/mitm/engines/create_module_template.py +1 -2
  17. atomicshop/mitm/import_config.py +91 -89
  18. atomicshop/mitm/initialize_engines.py +1 -2
  19. atomicshop/mitm/message.py +5 -4
  20. atomicshop/mitm/mitm_main.py +238 -122
  21. atomicshop/mitm/recs_files.py +61 -5
  22. atomicshop/mitm/ssh_tester.py +82 -0
  23. atomicshop/mitm/statistic_analyzer.py +33 -12
  24. atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +104 -31
  25. atomicshop/networks.py +160 -92
  26. atomicshop/package_mains_processor.py +84 -0
  27. atomicshop/permissions/ubuntu_permissions.py +47 -0
  28. atomicshop/print_api.py +3 -5
  29. atomicshop/process.py +11 -4
  30. atomicshop/python_functions.py +23 -108
  31. atomicshop/speech_recognize.py +8 -0
  32. atomicshop/ssh_remote.py +140 -164
  33. atomicshop/web.py +63 -22
  34. atomicshop/web_apis/google_llm.py +22 -14
  35. atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
  36. atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +2 -1
  37. atomicshop/wrappers/dockerw/dockerw.py +2 -2
  38. atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
  39. atomicshop/wrappers/elasticsearchw/elastic_infra.py +0 -190
  40. atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +5 -5
  41. atomicshop/wrappers/githubw.py +180 -68
  42. atomicshop/wrappers/loggingw/consts.py +1 -1
  43. atomicshop/wrappers/loggingw/handlers.py +1 -1
  44. atomicshop/wrappers/loggingw/loggingw.py +20 -4
  45. atomicshop/wrappers/loggingw/reading.py +18 -0
  46. atomicshop/wrappers/mongodbw/mongo_infra.py +0 -38
  47. atomicshop/wrappers/netshw.py +124 -3
  48. atomicshop/wrappers/playwrightw/scenarios.py +1 -1
  49. atomicshop/wrappers/powershell_networking.py +80 -0
  50. atomicshop/wrappers/psutilw/psutil_networks.py +9 -0
  51. atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
  52. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +3 -105
  53. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +3 -57
  54. atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +12 -27
  55. atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +15 -9
  56. atomicshop/wrappers/socketw/certificator.py +19 -9
  57. atomicshop/wrappers/socketw/creator.py +101 -14
  58. atomicshop/wrappers/socketw/dns_server.py +17 -5
  59. atomicshop/wrappers/socketw/exception_wrapper.py +21 -16
  60. atomicshop/wrappers/socketw/process_getter.py +86 -0
  61. atomicshop/wrappers/socketw/receiver.py +29 -9
  62. atomicshop/wrappers/socketw/sender.py +10 -9
  63. atomicshop/wrappers/socketw/sni.py +31 -10
  64. atomicshop/wrappers/socketw/{base.py → socket_base.py} +33 -1
  65. atomicshop/wrappers/socketw/socket_client.py +11 -10
  66. atomicshop/wrappers/socketw/socket_wrapper.py +125 -32
  67. atomicshop/wrappers/socketw/ssl_base.py +6 -2
  68. atomicshop/wrappers/ubuntu_terminal.py +21 -18
  69. atomicshop/wrappers/win_auditw.py +189 -0
  70. {atomicshop-3.3.8.dist-info → atomicshop-3.10.0.dist-info}/METADATA +25 -30
  71. {atomicshop-3.3.8.dist-info → atomicshop-3.10.0.dist-info}/RECORD +83 -109
  72. atomicshop/_basics_temp.py +0 -101
  73. atomicshop/a_installs/ubuntu/docker_rootless.py +0 -11
  74. atomicshop/a_installs/ubuntu/docker_sudo.py +0 -11
  75. atomicshop/a_installs/ubuntu/elastic_search_and_kibana.py +0 -10
  76. atomicshop/a_installs/ubuntu/mongodb.py +0 -12
  77. atomicshop/a_installs/win/fibratus.py +0 -9
  78. atomicshop/a_installs/win/mongodb.py +0 -9
  79. atomicshop/a_installs/win/wsl_ubuntu_lts.py +0 -10
  80. atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
  81. atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
  82. atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
  83. atomicshop/addons/package_setup/Setup.cmd +0 -7
  84. atomicshop/archiver/__init__.py +0 -0
  85. atomicshop/archiver/_search_in_zip.py +0 -189
  86. atomicshop/archiver/search_in_archive.py +0 -284
  87. atomicshop/archiver/sevenz_app_w.py +0 -86
  88. atomicshop/archiver/sevenzs.py +0 -73
  89. atomicshop/archiver/shutils.py +0 -34
  90. atomicshop/archiver/zips.py +0 -353
  91. atomicshop/file_types.py +0 -24
  92. atomicshop/pbtkmultifile_argparse.py +0 -88
  93. atomicshop/script_as_string_processor.py +0 -42
  94. atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
  95. atomicshop/ssh_scripts/process_from_port.py +0 -27
  96. atomicshop/wrappers/_process_wrapper_curl.py +0 -27
  97. atomicshop/wrappers/_process_wrapper_tar.py +0 -21
  98. atomicshop/wrappers/dockerw/install_docker.py +0 -449
  99. atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -233
  100. atomicshop/wrappers/ffmpegw.py +0 -125
  101. atomicshop/wrappers/fibratusw/__init__.py +0 -0
  102. atomicshop/wrappers/fibratusw/install.py +0 -80
  103. atomicshop/wrappers/mongodbw/install_mongodb_ubuntu.py +0 -100
  104. atomicshop/wrappers/mongodbw/install_mongodb_win.py +0 -244
  105. atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
  106. atomicshop/wrappers/socketw/get_process.py +0 -123
  107. atomicshop/wrappers/wslw.py +0 -192
  108. atomicshop-3.3.8.dist-info/entry_points.txt +0 -2
  109. /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
  110. /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
  111. /atomicshop/{addons → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
  112. /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
  113. /atomicshop/{addons → a_mains/addons}/process_list/compile.cmd +0 -0
  114. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.dll +0 -0
  115. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.exp +0 -0
  116. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.lib +0 -0
  117. /atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +0 -0
  118. {atomicshop-3.3.8.dist-info → atomicshop-3.10.0.dist-info}/WHEEL +0 -0
  119. {atomicshop-3.3.8.dist-info → atomicshop-3.10.0.dist-info}/licenses/LICENSE.txt +0 -0
  120. {atomicshop-3.3.8.dist-info → atomicshop-3.10.0.dist-info}/top_level.txt +0 -0
@@ -1,9 +1,13 @@
1
+ import multiprocessing
1
2
  from datetime import datetime
2
3
  import threading
3
4
  import queue
4
5
  import socket
6
+ import ssl
7
+ from typing import Literal
8
+ import struct
5
9
 
6
- from ..wrappers.socketw import receiver, sender, socket_client, base
10
+ from ..wrappers.socketw import receiver, sender, socket_client, socket_base
7
11
  from .. import websocket_parse, ip_addresses
8
12
  from ..http_parse import HTTPRequestParse, HTTPResponseParse
9
13
  from ..basics import threads, tracebacks
@@ -25,7 +29,7 @@ def thread_worker_main(
25
29
  tls_version: str,
26
30
  domain_from_dns,
27
31
  statistics_writer,
28
- engines_list,
32
+ engines_list: list[initialize_engines.ModuleCategory],
29
33
 
30
34
  # These parameters come from the main mitm module.
31
35
  config_static: cf
@@ -168,32 +172,55 @@ def thread_worker_main(
168
172
  def parse_websocket(raw_bytes):
169
173
  return websocket_frame_parser.parse_frame_bytes(raw_bytes)
170
174
 
171
- def finish_thread():
175
+ def finish_thread(send_connection_reset: bool = False):
176
+ """
177
+ Finishing the thread, closing sockets.
178
+
179
+ :param send_connection_reset: Whether to send TCP RST flag to the client when closing the socket.
180
+ Basically what happens is that the server socket is closed abruptly sending us any Connection*Error exception.
181
+ We can simulate the ConnectionResetError on the client side by sending TCP RST flag when closing the socket.
182
+ But not the ConnectionAbortedError, since it is caused by other reasons (local TCP stack and has nothing
183
+ to do with the remote server).
184
+ """
172
185
  # At this stage there could be several times that the same socket was used to the service server - we need to
173
186
  # close this socket as well if it still opened.
174
187
  # The first part of the condition is to check if the service socket was connected at all.
175
188
  # If the service socket couldn't connect, then the instance will be None.
176
189
  if service_socket_instance and service_socket_instance.fileno() != -1:
177
- if service_client.socket_instance:
178
- service_client.close_socket()
190
+ if origin_service_client_instance.socket_instance:
191
+ origin_service_client_instance.close_socket()
179
192
 
180
193
  # If client socket is still opened - close
181
194
  if client_socket.fileno() != -1:
195
+ if send_connection_reset:
196
+ # Abort the connection
197
+ linger = struct.pack('ii', 1, 0)
198
+ client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger)
182
199
  client_socket.close()
183
- network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
200
+
201
+ if send_connection_reset:
202
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}] with TCP RST flag (Sent ConnectionResetError)...")
203
+ else:
204
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
184
205
 
185
206
  network_logger.info("Thread Finished. Will continue listening on the Main thread")
186
207
 
187
208
  def create_requester_request(
188
209
  client_message: ClientMessage,
189
210
  sending_socket: socket.socket
190
- ) -> list[bytes]:
211
+ ) -> tuple[bytes, bool]:
212
+ request_received_raw: bytes = client_message.request_raw_bytes
191
213
  request_custom_raw: bytes = requester.create_request(client_message, sending_socket=sending_socket)
192
214
 
193
- # Output first 100 characters of the request.
194
- requester.logger.info(f"{request_custom_raw[0: 100]}...")
215
+ if request_custom_raw is None or request_received_raw == request_custom_raw:
216
+ is_requester_worked: bool = False
217
+ else:
218
+ is_requester_worked: bool = True
219
+
220
+ # Output first 100 characters of the request.
221
+ requester.logger.info(f"{request_custom_raw[0: 100]}...")
195
222
 
196
- return [request_custom_raw]
223
+ return request_custom_raw, is_requester_worked
197
224
 
198
225
  def create_responder_response(client_message: ClientMessage) -> list[bytes]:
199
226
  if client_message.action == 'service_connect':
@@ -201,13 +228,19 @@ def thread_worker_main(
201
228
  else:
202
229
  # If we're in offline mode, and it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
203
230
  # response automatically.
204
- if config_static.MainConfig.offline and protocol == 'Websocket' and client_receive_count == 1:
231
+ if config_static.MainConfig.is_offline and protocol == 'Websocket' and client_receive_count == 1:
205
232
  responses: list = list()
206
233
  responses.append(
207
234
  websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
235
+ responder.logger.info(f"Generated automatic WebSocket response in Offline Mode.")
208
236
  else:
209
237
  # Creating response for parsed message and printing
210
- responses: list = responder.create_response(client_message)
238
+ responder_responses: list = responder.create_response(client_message)
239
+ if responder_responses is None:
240
+ responses: list = [client_message.response_raw_bytes]
241
+ else:
242
+ responses: list = responder_responses
243
+ responder.logger.info(f"Generated {len(responses)} responses from responder.")
211
244
 
212
245
  # Output first 100 characters of all the responses in the list.
213
246
  for response_raw_bytes_single in responses:
@@ -235,13 +268,14 @@ def thread_worker_main(
235
268
  logger=network_logger,
236
269
  custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
237
270
  enable_sslkeylogfile_env_to_client_ssl_context=(
238
- config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context)
271
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
272
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
239
273
  )
240
274
  # If it's a domain name, then we'll use the DNS to resolve it.
241
275
  else:
242
276
  # If we're on localhost, then use external services list in order to resolve the domain:
243
277
  # config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
244
- if client_message.client_ip in base.THIS_DEVICE_IP_LIST:
278
+ if client_message.client_ip in socket_base.THIS_DEVICE_IP_LIST:
245
279
  service_client_instance = socket_client.SocketClient(
246
280
  service_name=client_message.server_name,
247
281
  service_port=client_message.destination_port,
@@ -250,7 +284,8 @@ def thread_worker_main(
250
284
  logger=network_logger,
251
285
  custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
252
286
  enable_sslkeylogfile_env_to_client_ssl_context=(
253
- config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context)
287
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
288
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
254
289
  )
255
290
  # If we're not on localhost, then connect to domain directly.
256
291
  else:
@@ -261,7 +296,8 @@ def thread_worker_main(
261
296
  logger=network_logger,
262
297
  custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
263
298
  enable_sslkeylogfile_env_to_client_ssl_context=(
264
- config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context)
299
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
300
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
265
301
  )
266
302
 
267
303
  return service_client_instance
@@ -327,11 +363,291 @@ def thread_worker_main(
327
363
  client_message.destination_port = destination_port
328
364
  client_message.server_name = server_name
329
365
  client_message.thread_id = thread_id
366
+ client_message.thread_process = thread_process_name
330
367
  client_message.process_name = process_commandline
331
368
  client_message.engine_name = engine_name
332
369
 
333
370
  return client_message
334
371
 
372
+ def receive_send_service_connect(
373
+ client_connection_message: ClientMessage,
374
+ sending_socket: ssl.SSLSocket | socket.socket
375
+ ) -> Literal['continue', 'return'] | None:
376
+
377
+ nonlocal exception_or_close_in_receiving_thread
378
+
379
+ client_message = client_connection_message
380
+
381
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
382
+ print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]",
383
+ logger=network_logger, logger_method='info')
384
+
385
+ # If the client message is the connection message, then we'll skip to the next iteration.
386
+ if not bytes_to_send_list:
387
+ return 'continue'
388
+
389
+ # is_socket_closed: bool = False
390
+ error_on_send: str = str()
391
+ for bytes_to_send_single in bytes_to_send_list:
392
+ client_message.reinitialize_dynamic_vars()
393
+ client_message.timestamp = datetime.now()
394
+ client_message.response_raw_bytes = bytes_to_send_single
395
+ client_message.action = 'service_responder'
396
+ record_and_statistics_write(client_message)
397
+
398
+ # Send the bytes back to the client socket.
399
+ error_on_send: str = sender.Sender(
400
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
401
+ logger=network_logger).send()
402
+
403
+ if error_on_send:
404
+ client_message.reinitialize_dynamic_vars()
405
+ client_message.errors.append(error_on_send)
406
+ client_message.timestamp = datetime.now()
407
+ client_message.action = 'client_send'
408
+ record_and_statistics_write(client_message)
409
+
410
+ if error_on_send:
411
+ exception_or_close_in_receiving_thread = True
412
+ finish_thread()
413
+ return 'return'
414
+
415
+ return None
416
+
417
+ def receive_send_client_offline(
418
+ client_message: ClientMessage,
419
+ receiving_socket: ssl.SSLSocket | socket.socket,
420
+ sending_socket: ssl.SSLSocket | socket.socket
421
+ ) -> Literal['return'] | None:
422
+ nonlocal exception_or_close_in_receiving_thread
423
+ nonlocal client_receive_count
424
+
425
+ client_receive_count += 1
426
+
427
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
428
+ client_message.timestamp = datetime.now()
429
+
430
+ received_raw_data, is_socket_closed, error_message = receiver.Receiver(
431
+ ssl_socket=receiving_socket, logger=network_logger).receive()
432
+
433
+ process_client_raw_data(received_raw_data, error_message, client_message)
434
+ client_message.action = 'client_receive'
435
+ record_and_statistics_write(client_message)
436
+ if error_message:
437
+ print_api(error_message, logger=network_logger, logger_method='critical')
438
+
439
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
440
+ # the socket was closed by the other side, it means that the service thread closed the socket.
441
+ if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
442
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
443
+ logger_method='info')
444
+ return 'return'
445
+
446
+ # If the socket was closed on receive, and we're in offline mode, then we'll finish the thread right away.
447
+ # Since nothing more can be done, like responding to service or using requester.
448
+ if is_socket_closed:
449
+ exception_or_close_in_receiving_thread = True
450
+ finish_thread()
451
+ return 'return'
452
+
453
+ # Send to requester.
454
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
455
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
456
+ # We will not process the raw data if requester didn't change anything.
457
+ if is_requester_worked:
458
+ client_message.reinitialize_dynamic_vars()
459
+ client_message.timestamp = datetime.now()
460
+ client_message.request_raw_bytes = request_custom_raw
461
+ client_message.action = 'client_requester'
462
+ process_client_raw_data(request_custom_raw, error_message, client_message)
463
+ record_and_statistics_write(client_message)
464
+
465
+ print_api("Offline Mode, sending to responder directly.", logger=network_logger,
466
+ logger_method='info')
467
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
468
+
469
+ error_on_send: str = str()
470
+ for bytes_to_send_single in bytes_to_send_list:
471
+ client_message.reinitialize_dynamic_vars()
472
+ client_message.timestamp = datetime.now()
473
+ client_message.response_raw_bytes = bytes_to_send_single
474
+ client_message.action = 'client_responder_offline'
475
+ process_server_raw_data(bytes_to_send_single, '', client_message)
476
+ record_and_statistics_write(client_message)
477
+
478
+ error_on_send: str = sender.Sender(
479
+ ssl_socket=receiving_socket, bytes_to_send=bytes_to_send_single,
480
+ logger=network_logger).send()
481
+
482
+ if error_on_send:
483
+ client_message.reinitialize_dynamic_vars()
484
+ client_message.errors.append(error_on_send)
485
+ client_message.timestamp = datetime.now()
486
+ client_message.action = 'service_send'
487
+
488
+ record_and_statistics_write(client_message)
489
+
490
+ # If the socket was closed on message receive, then we'll break the loop only after send.
491
+ if is_socket_closed or error_on_send:
492
+ exception_or_close_in_receiving_thread = True
493
+ finish_thread()
494
+ return 'return'
495
+
496
+ return None
497
+
498
+ def receive_send_client(
499
+ client_message: ClientMessage,
500
+ receiving_socket: ssl.SSLSocket | socket.socket,
501
+ sending_socket: ssl.SSLSocket | socket.socket
502
+ ) -> Literal['return'] | None:
503
+
504
+ nonlocal exception_or_close_in_receiving_thread
505
+ nonlocal client_receive_count
506
+
507
+ client_receive_count += 1
508
+
509
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
510
+
511
+ # Getting message from the client over the socket using specific class.
512
+ client_message.timestamp = datetime.now()
513
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
514
+ ssl_socket=receiving_socket, logger=network_logger).receive()
515
+
516
+ process_client_raw_data(received_raw_data, error_on_receive, client_message)
517
+ client_message.action = 'client_receive'
518
+
519
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
520
+ # the socket was closed by the other side, it means that the service thread closed the socket.
521
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
522
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
523
+ logger_method='info')
524
+ return 'return'
525
+
526
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
527
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
528
+ record_and_statistics_write(client_message)
529
+ if error_on_receive:
530
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
531
+
532
+ # At this point if the socket was closed on receive, then there's no point to send anything to the service.
533
+ # So, we check for 'is_socket_closed' here.
534
+ error_on_send: str = str()
535
+ if not is_socket_closed:
536
+ # Send to requester.
537
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
538
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
539
+ # We will not process the raw data if requester didn't change anything.
540
+ if is_requester_worked:
541
+ client_message.reinitialize_dynamic_vars()
542
+ client_message.timestamp = datetime.now()
543
+ client_message.request_raw_bytes = request_custom_raw
544
+ client_message.action = 'client_requester'
545
+ process_client_raw_data(request_custom_raw, error_on_receive, client_message)
546
+ record_and_statistics_write(client_message)
547
+
548
+ error_on_send: str = sender.Sender(
549
+ ssl_socket=sending_socket, bytes_to_send=client_message.request_raw_bytes,
550
+ logger=network_logger).send()
551
+
552
+ if error_on_send:
553
+ client_message.reinitialize_dynamic_vars()
554
+ client_message.errors.append(error_on_send)
555
+ client_message.timestamp = datetime.now()
556
+ client_message.action = 'service_send'
557
+ record_and_statistics_write(client_message)
558
+
559
+ # If the socket was closed on message receive, then we'll break the loop only after send.
560
+ if is_socket_closed or error_on_send:
561
+ exception_or_close_in_receiving_thread = True
562
+ finish_thread()
563
+ return 'return'
564
+
565
+ return None
566
+
567
+ def receive_send_service(
568
+ client_message: ClientMessage,
569
+ receiving_socket: ssl.SSLSocket | socket.socket,
570
+ sending_socket: ssl.SSLSocket | socket.socket
571
+ ) -> Literal['return'] | None:
572
+
573
+ nonlocal exception_or_close_in_receiving_thread
574
+ nonlocal server_receive_count
575
+
576
+ server_receive_count += 1
577
+
578
+ network_logger.info(f"Initializing Receiver for Service cycle: {str(server_receive_count)}")
579
+
580
+ # Getting message from the client over the socket using specific class.
581
+ client_message.timestamp = datetime.now()
582
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
583
+ ssl_socket=receiving_socket, logger=network_logger).receive()
584
+
585
+ process_server_raw_data(received_raw_data, error_on_receive, client_message)
586
+ client_message.action = 'service_receive'
587
+
588
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
589
+ # the socket was closed by the other side, it means that the service thread closed the socket.
590
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
591
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
592
+ logger_method='info')
593
+ return 'return'
594
+
595
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
596
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
597
+ record_and_statistics_write(client_message)
598
+ if error_on_receive:
599
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
600
+
601
+ # At this stage, we have received the response from the service, but there was an exception/error or the socket was simply closed.
602
+ # Meaning the 'received_raw_data' is None for exception or b'' for just closed socket.
603
+ # So, there's no point to send anything back to the client.
604
+ # Close both sockets and finish the threads.
605
+ # So, we check for 'is_socket_closed' here.
606
+ error_on_send: str = str()
607
+ if not is_socket_closed:
608
+ # Now send it to requester/responder.
609
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
610
+
611
+ # is_socket_closed: bool = False
612
+ for bytes_to_send_single in bytes_to_send_list:
613
+ client_message.reinitialize_dynamic_vars()
614
+ client_message.timestamp = datetime.now()
615
+ client_message.response_raw_bytes = bytes_to_send_single
616
+
617
+ # This records the requester or responder output, only if it is not the same as the original
618
+ # message.
619
+ if bytes_to_send_single != received_raw_data:
620
+ client_message.action = 'service_responder'
621
+ record_and_statistics_write(client_message)
622
+
623
+ error_on_send: str = sender.Sender(
624
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
625
+ logger=network_logger).send()
626
+
627
+ if error_on_send:
628
+ client_message.reinitialize_dynamic_vars()
629
+ client_message.errors.append(error_on_send)
630
+ client_message.timestamp = datetime.now()
631
+ client_message.action = 'client_send'
632
+
633
+ record_and_statistics_write(client_message)
634
+
635
+ # If the socket was closed on message receive, then we'll break the loop only after send.
636
+ if is_socket_closed or error_on_send:
637
+ exception_or_close_in_receiving_thread = True
638
+
639
+ if error_on_receive and 'Connection' in error_on_receive and 'Error' in error_on_receive:
640
+ # If there was a connection error on receive from client, and we're closing the socket to the client,
641
+ # then we'll send TCP RST flag to simulate ConnectionResetError on the client side.
642
+ finish_thread(send_connection_reset=True)
643
+ else:
644
+ finish_thread()
645
+
646
+ return 'return'
647
+
648
+ return None
649
+
650
+
335
651
  def receive_send_start(
336
652
  receiving_socket,
337
653
  sending_socket = None,
@@ -357,146 +673,25 @@ def thread_worker_main(
357
673
  raise ValueError(f"Unknown side of the socket: {receiving_socket}")
358
674
 
359
675
  while True:
360
- is_socket_closed: bool = False
361
- # pass the socket connect to responder.
362
- if side == 'Service' and client_connection_message:
363
- client_message = client_connection_message
364
-
365
- bytes_to_send_list: list[bytes] = create_responder_response(client_message)
366
- print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]", logger=network_logger,
367
- logger_method='info')
368
-
369
- received_raw_data = None
370
- else:
371
- client_message.reinitialize_dynamic_vars()
676
+ client_message.reinitialize_dynamic_vars()
372
677
 
373
- if side == 'Client':
374
- client_receive_count += 1
375
- current_count = client_receive_count
376
- else:
377
- server_receive_count += 1
378
- current_count = server_receive_count
379
-
380
- # Getting current time of message received, either from client or service.
381
- client_message.timestamp = datetime.now()
382
-
383
- # # No need to receive on service socket if we're in offline mode, because there is no service to connect to.
384
- # if config_static.MainConfig.offline and side == 'Service':
385
- # print_api("Offline Mode, skipping receiving on service socket.", logger=network_logger,
386
- # logger_method='info')
387
- # else:
388
-
389
- network_logger.info(
390
- f"Initializing Receiver for {side} cycle: {str(current_count)}")
391
-
392
- # Getting message from the client over the socket using specific class.
393
- received_raw_data, is_socket_closed, error_message = receiver.Receiver(
394
- ssl_socket=receiving_socket, logger=network_logger).receive()
395
-
396
- # In case of client socket, we'll process the raw data specifically for the client.
397
- if side == 'Client':
398
- process_client_raw_data(received_raw_data, error_message, client_message)
399
- client_message.action = 'client_receive'
400
- # In case of service socket, we'll process the raw data specifically for the service.
401
- else:
402
- process_server_raw_data(received_raw_data, error_message, client_message)
403
- client_message.action = 'service_receive'
404
-
405
- # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
406
- # the socket was closed by the other side, it means that the service thread closed the socket.
407
- if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
408
- print_api("Both sockets are closed, breaking the loop", logger=network_logger,
409
- logger_method='info')
410
- return
411
-
412
- # We will record only if there was no closing signal, because if there was, it means that we initiated
413
- # the close on the opposite socket.
414
- record_and_statistics_write(client_message)
415
-
416
- # if is_socket_closed:
417
- # exception_or_close_in_receiving_thread = True
418
- # finish_thread()
419
- # return
420
-
421
- # Now send it to requester/responder.
422
- if side == 'Client':
423
- # Send to requester.
424
- bytes_to_send_list: list[bytes] = create_requester_request(
425
- client_message, sending_socket=sending_socket)
426
-
427
- # If we're in offline mode, then we'll put the request to the responder right away.
428
- if config_static.MainConfig.offline:
429
- print_api("Offline Mode, sending to responder directly.", logger=network_logger,
430
- logger_method='info')
431
- process_client_raw_data(bytes_to_send_list[0], error_message, client_message)
432
- bytes_to_send_list = create_responder_response(client_message)
433
- elif side == 'Service':
434
- bytes_to_send_list: list[bytes] = create_responder_response(client_message)
435
- print_api(f"Got responses from responder, count: [{len(bytes_to_send_list)}]",
436
- logger=network_logger,
437
- logger_method='info')
438
- else:
439
- raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
440
-
441
-
442
- # If nothing was passed from the responder, and the client message is the connection message, then we'll skip to the next iteration.
443
- if not bytes_to_send_list and client_connection_message:
678
+ if side == 'Service' and client_connection_message:
679
+ result: Literal['continue', 'return'] | None = (
680
+ receive_send_service_connect(client_connection_message, sending_socket))
444
681
  client_connection_message = None
445
- continue
446
-
447
- # is_socket_closed: bool = False
448
- error_on_send: str = str()
449
- for bytes_to_send_single in bytes_to_send_list:
450
- client_message.reinitialize_dynamic_vars()
451
- client_message.timestamp = datetime.now()
682
+ if result == 'continue':
683
+ continue
684
+ elif side == 'Client' and config_static.MainConfig.is_offline:
685
+ result: Literal['return'] | None = receive_send_client_offline(client_message, receiving_socket, sending_socket)
686
+ elif side == 'Client':
687
+ result: Literal['return'] | None = receive_send_client(client_message, receiving_socket, sending_socket)
688
+ elif side == 'Service':
689
+ result: Literal['return'] | None = receive_send_service(client_message, receiving_socket, sending_socket)
690
+ else:
691
+ raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
452
692
 
453
- if side == 'Client':
454
- client_message.request_raw_bytes = bytes_to_send_single
455
- else:
456
- client_message.response_raw_bytes = bytes_to_send_single
457
-
458
- # This records the requester or responder output, only if it is not the same as the original
459
- # message.
460
- if bytes_to_send_single != received_raw_data:
461
- if side == 'Client':
462
- client_message.action = 'client_requester'
463
-
464
- if config_static.MainConfig.offline:
465
- client_message.action = 'client_responder_offline'
466
- elif side == 'Service':
467
- client_message.action = 'service_responder'
468
- record_and_statistics_write(client_message)
469
-
470
- # If we're in offline mode, it means we're in the client thread, and we'll send the
471
- # bytes back to the client socket.
472
- if config_static.MainConfig.offline:
473
- error_on_send: str = sender.Sender(
474
- ssl_socket=receiving_socket, class_message=bytes_to_send_single,
475
- logger=network_logger).send()
476
- else:
477
- error_on_send: str = sender.Sender(
478
- ssl_socket=sending_socket, class_message=bytes_to_send_single,
479
- logger=network_logger).send()
480
-
481
- if error_on_send:
482
- client_message.reinitialize_dynamic_vars()
483
- client_message.errors.append(error_on_send)
484
- client_message.timestamp = datetime.now()
485
- if side == 'Client':
486
- client_message.action = 'service_send'
487
- else:
488
- client_message.action = 'client_send'
489
-
490
- record_and_statistics_write(client_message)
491
-
492
- # If the socket was closed on message receive, then we'll break the loop only after send.
493
- if is_socket_closed or error_on_send:
494
- exception_or_close_in_receiving_thread = True
495
- finish_thread()
693
+ if result == 'return':
496
694
  return
497
-
498
- # For next iteration to start in case this iteration was responsible to process connection message, we need to set it to None.
499
- client_connection_message = None
500
695
  except Exception as exc:
501
696
  # If the sockets were already closed, then there is nothing to do here besides log.
502
697
  # if (isinstance(exc, OSError) and exc.errno == 10038 and
@@ -513,7 +708,7 @@ def thread_worker_main(
513
708
  ):
514
709
  nonlocal exception_or_close_in_receiving_thread
515
710
 
516
- exception_or_close_in_receiving_thread=True
711
+ exception_or_close_in_receiving_thread = True
517
712
  # handle_exceptions(exc, client_message, recorded)
518
713
  exception_message = tracebacks.get_as_string(one_line=True)
519
714
 
@@ -542,24 +737,34 @@ def thread_worker_main(
542
737
 
543
738
  finish_thread()
544
739
 
740
+ # Add custom attribute to the exception.
741
+ exc.engine_name = client_message.engine_name
742
+
545
743
  # After the socket clean up, we will still raise the exception to the main thread.
546
744
  raise exc
547
745
 
548
746
  # ================================================================================================================
549
747
  # This is the start of the thread_worker_main function
550
-
551
748
  network_logger = loggingw.get_logger_with_level(config_static.MainConfig.LOGGER_NAME)
552
749
 
553
750
  # Only protocols that are encrypted with TLS have the server name attribute.
554
751
  if is_tls:
555
752
  # Get current destination domain
556
753
  server_name = client_socket.server_hostname
754
+ # If there is no server name from the TLS handshake, then we'll use the domain from the DNS.
755
+ if not server_name:
756
+ server_name = domain_from_dns
557
757
  # If the protocol is not TLS, then we'll use the domain from the DNS.
558
758
  else:
559
759
  server_name = domain_from_dns
560
760
 
561
761
  thread_id = threads.current_thread_id()
562
762
 
763
+ process_name: str = multiprocessing.current_process().name
764
+ current_thread = threading.current_thread()
765
+ thread_process_name: str = f"{process_name} | {current_thread.name}"
766
+ current_thread.name = thread_process_name
767
+
563
768
  # This is the main protocols.
564
769
  protocol: str = str()
565
770
  # This is the secondary protocol in the websocket.
@@ -582,6 +787,13 @@ def thread_worker_main(
582
787
  responder = found_domain_module.responder_class_object()
583
788
  recorder = found_domain_module.recorder_class_object(record_path=config_static.LogRec.recordings_path)
584
789
 
790
+ engine_name: str = recorder.engine_name
791
+
792
+ for engine in engines_list:
793
+ if engine.engine_name == engine_name:
794
+ responder.add_args(engine=engine)
795
+ break
796
+
585
797
  network_logger.info(f"Assigned Modules for [{server_name}]: "
586
798
  f"{parser.__name__}, "
587
799
  f"{requester.__class__.__name__}, "
@@ -597,9 +809,15 @@ def thread_worker_main(
597
809
  http_path_queue: queue.Queue = queue.Queue()
598
810
 
599
811
  try:
600
- engine_name: str = recorder.engine_name
601
812
  client_ip, source_port = client_socket.getpeername()
602
- client_name: str = socket.gethostbyaddr(client_ip)[0]
813
+
814
+ try:
815
+ client_name: str = socket.gethostbyaddr(client_ip)[0]
816
+ # This can happen if the host changed IP address, but it wasn't propagated over DHCP.
817
+ except socket.herror:
818
+ client_name = ""
819
+
820
+ client_name = client_name.lower()
603
821
  destination_port: int = client_socket.getsockname()[1]
604
822
  destination_port_str: str = str(destination_port)
605
823
 
@@ -609,7 +827,7 @@ def thread_worker_main(
609
827
  _, destination_port_str = initialize_engines.get_ipv4_from_engine_on_connect_port(on_port_connect_value)
610
828
  destination_port: int = int(destination_port_str)
611
829
 
612
- if config_static.MainConfig.offline:
830
+ if config_static.MainConfig.is_offline:
613
831
  # If in offline mode, then we'll get the TCP server's input address.
614
832
  server_ip = client_socket.getsockname()[0]
615
833
  else:
@@ -619,7 +837,7 @@ def thread_worker_main(
619
837
  network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
620
838
  f"Destination service: [{server_name}:{destination_port}]")
621
839
 
622
- service_client = None
840
+ origin_service_client_instance = None
623
841
  client_receive_count: int = 0
624
842
  server_receive_count: int = 0
625
843
  client_message_connection = client_message_first_start()
@@ -631,11 +849,11 @@ def thread_worker_main(
631
849
  client_message_connection.action = 'service_connect'
632
850
  client_message_connection.timestamp = datetime.now()
633
851
 
634
- if config_static.MainConfig.offline:
852
+ if config_static.MainConfig.is_offline:
635
853
  client_message_connection.info = 'Offline Mode'
636
854
  else:
637
- service_client = create_client_socket(client_message_connection)
638
- service_socket_instance, connection_error = service_client.service_connection()
855
+ origin_service_client_instance = create_client_socket(client_message_connection)
856
+ service_socket_instance, connection_error = origin_service_client_instance.service_connection()
639
857
 
640
858
  if connection_error:
641
859
  client_message_connection.errors.append(connection_error)
@@ -650,22 +868,22 @@ def thread_worker_main(
650
868
  client_thread = threading.Thread(
651
869
  target=receive_send_start,
652
870
  args=(client_socket, service_socket_instance, client_exception_queue, None),
653
- name=f"Thread-{thread_id}-Client",
871
+ name=f"{process_name} | Thread-{thread_id}-Client",
654
872
  daemon=True)
655
873
  client_thread.start()
656
874
 
657
875
  service_exception_queue: queue.Queue = queue.Queue()
658
- if not config_static.MainConfig.offline:
876
+ if not config_static.MainConfig.is_offline:
659
877
  service_thread = threading.Thread(
660
878
  target=receive_send_start,
661
879
  args=(service_socket_instance, client_socket, service_exception_queue, client_message_connection),
662
- name=f"Thread-{thread_id}-Service",
880
+ name=f"{process_name} | Thread-{thread_id}-Service",
663
881
  daemon=True)
664
882
  service_thread.start()
665
883
 
666
884
  client_thread.join()
667
885
  # If we're in offline mode, then there is no service thread.
668
- if not config_static.MainConfig.offline:
886
+ if not config_static.MainConfig.is_offline:
669
887
  # If we're not in offline mode, then we'll wait for the service thread to finish.
670
888
  # noinspection PyUnboundLocalVariable
671
889
  service_thread.join()