atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. atomicshop/__init__.py +1 -1
  2. atomicshop/{addons/mains → a_mains}/FACT/update_extract.py +3 -2
  3. atomicshop/a_mains/dns_gateway_setting.py +11 -0
  4. atomicshop/a_mains/get_local_tcp_ports.py +85 -0
  5. atomicshop/a_mains/github_wrapper.py +11 -0
  6. atomicshop/a_mains/install_ca_certificate.py +172 -0
  7. atomicshop/a_mains/process_from_port.py +119 -0
  8. atomicshop/a_mains/set_default_dns_gateway.py +90 -0
  9. atomicshop/a_mains/update_config_toml.py +38 -0
  10. atomicshop/basics/ansi_escape_codes.py +3 -1
  11. atomicshop/basics/argparse_template.py +2 -0
  12. atomicshop/basics/booleans.py +27 -30
  13. atomicshop/basics/bytes_arrays.py +43 -0
  14. atomicshop/basics/classes.py +149 -1
  15. atomicshop/basics/enums.py +2 -2
  16. atomicshop/basics/exceptions.py +5 -1
  17. atomicshop/basics/list_of_classes.py +29 -0
  18. atomicshop/basics/multiprocesses.py +374 -50
  19. atomicshop/basics/strings.py +72 -3
  20. atomicshop/basics/threads.py +14 -0
  21. atomicshop/basics/tracebacks.py +13 -3
  22. atomicshop/certificates.py +153 -52
  23. atomicshop/config_init.py +11 -6
  24. atomicshop/console_user_response.py +7 -14
  25. atomicshop/consoles.py +9 -0
  26. atomicshop/datetimes.py +1 -1
  27. atomicshop/diff_check.py +3 -3
  28. atomicshop/dns.py +128 -3
  29. atomicshop/etws/_pywintrace_fix.py +17 -0
  30. atomicshop/etws/trace.py +40 -42
  31. atomicshop/etws/traces/trace_dns.py +56 -44
  32. atomicshop/etws/traces/trace_tcp.py +130 -0
  33. atomicshop/file_io/csvs.py +27 -5
  34. atomicshop/file_io/docxs.py +34 -17
  35. atomicshop/file_io/file_io.py +31 -17
  36. atomicshop/file_io/jsons.py +49 -0
  37. atomicshop/file_io/tomls.py +139 -0
  38. atomicshop/filesystem.py +616 -291
  39. atomicshop/get_process_list.py +3 -3
  40. atomicshop/http_parse.py +149 -93
  41. atomicshop/ip_addresses.py +6 -1
  42. atomicshop/mitm/centered_settings.py +132 -0
  43. atomicshop/mitm/config_static.py +207 -0
  44. atomicshop/mitm/config_toml_editor.py +55 -0
  45. atomicshop/mitm/connection_thread_worker.py +875 -357
  46. atomicshop/mitm/engines/__parent/parser___parent.py +4 -17
  47. atomicshop/mitm/engines/__parent/recorder___parent.py +108 -51
  48. atomicshop/mitm/engines/__parent/requester___parent.py +116 -0
  49. atomicshop/mitm/engines/__parent/responder___parent.py +75 -114
  50. atomicshop/mitm/engines/__reference_general/parser___reference_general.py +10 -7
  51. atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +5 -5
  52. atomicshop/mitm/engines/__reference_general/requester___reference_general.py +47 -0
  53. atomicshop/mitm/engines/__reference_general/responder___reference_general.py +95 -13
  54. atomicshop/mitm/engines/create_module_template.py +58 -14
  55. atomicshop/mitm/import_config.py +359 -139
  56. atomicshop/mitm/initialize_engines.py +160 -80
  57. atomicshop/mitm/message.py +64 -23
  58. atomicshop/mitm/mitm_main.py +892 -0
  59. atomicshop/mitm/recs_files.py +183 -0
  60. atomicshop/mitm/shared_functions.py +4 -10
  61. atomicshop/mitm/ssh_tester.py +82 -0
  62. atomicshop/mitm/statistic_analyzer.py +136 -40
  63. atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +265 -83
  64. atomicshop/monitor/checks/dns.py +1 -1
  65. atomicshop/networks.py +671 -0
  66. atomicshop/on_exit.py +39 -9
  67. atomicshop/package_mains_processor.py +84 -0
  68. atomicshop/permissions/permissions.py +22 -0
  69. atomicshop/permissions/ubuntu_permissions.py +239 -0
  70. atomicshop/permissions/win_permissions.py +33 -0
  71. atomicshop/print_api.py +24 -42
  72. atomicshop/process.py +24 -6
  73. atomicshop/process_poller/process_pool.py +0 -1
  74. atomicshop/process_poller/simple_process_pool.py +204 -5
  75. atomicshop/python_file_patcher.py +1 -1
  76. atomicshop/python_functions.py +27 -75
  77. atomicshop/speech_recognize.py +8 -0
  78. atomicshop/ssh_remote.py +158 -172
  79. atomicshop/system_resource_monitor.py +61 -47
  80. atomicshop/system_resources.py +8 -8
  81. atomicshop/tempfiles.py +1 -2
  82. atomicshop/urls.py +6 -0
  83. atomicshop/venvs.py +28 -0
  84. atomicshop/versioning.py +27 -0
  85. atomicshop/web.py +98 -27
  86. atomicshop/web_apis/google_custom_search.py +44 -0
  87. atomicshop/web_apis/google_llm.py +188 -0
  88. atomicshop/websocket_parse.py +450 -0
  89. atomicshop/wrappers/certauthw/certauth.py +1 -0
  90. atomicshop/wrappers/cryptographyw.py +29 -8
  91. atomicshop/wrappers/ctyping/etw_winapi/const.py +97 -47
  92. atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +178 -49
  93. atomicshop/wrappers/ctyping/file_details_winapi.py +67 -0
  94. atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
  95. atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +2 -2
  96. atomicshop/wrappers/ctyping/setup_device.py +466 -0
  97. atomicshop/wrappers/ctyping/win_console.py +39 -0
  98. atomicshop/wrappers/dockerw/dockerw.py +113 -2
  99. atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
  100. atomicshop/wrappers/elasticsearchw/elastic_infra.py +75 -0
  101. atomicshop/wrappers/elasticsearchw/elasticsearchw.py +2 -20
  102. atomicshop/wrappers/factw/get_file_data.py +12 -5
  103. atomicshop/wrappers/factw/install/install_after_restart.py +89 -5
  104. atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +20 -14
  105. atomicshop/wrappers/githubw.py +537 -54
  106. atomicshop/wrappers/loggingw/consts.py +1 -1
  107. atomicshop/wrappers/loggingw/filters.py +23 -0
  108. atomicshop/wrappers/loggingw/formatters.py +12 -0
  109. atomicshop/wrappers/loggingw/handlers.py +214 -107
  110. atomicshop/wrappers/loggingw/loggers.py +19 -0
  111. atomicshop/wrappers/loggingw/loggingw.py +860 -22
  112. atomicshop/wrappers/loggingw/reading.py +134 -112
  113. atomicshop/wrappers/mongodbw/mongo_infra.py +31 -0
  114. atomicshop/wrappers/mongodbw/mongodbw.py +1324 -36
  115. atomicshop/wrappers/netshw.py +271 -0
  116. atomicshop/wrappers/playwrightw/engine.py +34 -19
  117. atomicshop/wrappers/playwrightw/infra.py +5 -0
  118. atomicshop/wrappers/playwrightw/javascript.py +7 -3
  119. atomicshop/wrappers/playwrightw/keyboard.py +14 -0
  120. atomicshop/wrappers/playwrightw/scenarios.py +172 -5
  121. atomicshop/wrappers/playwrightw/waits.py +9 -7
  122. atomicshop/wrappers/powershell_networking.py +80 -0
  123. atomicshop/wrappers/psutilw/processes.py +37 -1
  124. atomicshop/wrappers/psutilw/psutil_networks.py +85 -0
  125. atomicshop/wrappers/pyopensslw.py +9 -2
  126. atomicshop/wrappers/pywin32w/cert_store.py +116 -0
  127. atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
  128. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +3 -105
  129. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +3 -57
  130. atomicshop/wrappers/pywin32w/wmis/msft_netipaddress.py +113 -0
  131. atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +259 -0
  132. atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +112 -0
  133. atomicshop/wrappers/pywin32w/wmis/wmi_helpers.py +236 -0
  134. atomicshop/wrappers/socketw/accepter.py +21 -7
  135. atomicshop/wrappers/socketw/certificator.py +216 -150
  136. atomicshop/wrappers/socketw/creator.py +190 -50
  137. atomicshop/wrappers/socketw/dns_server.py +491 -182
  138. atomicshop/wrappers/socketw/exception_wrapper.py +45 -52
  139. atomicshop/wrappers/socketw/process_getter.py +86 -0
  140. atomicshop/wrappers/socketw/receiver.py +144 -102
  141. atomicshop/wrappers/socketw/sender.py +65 -35
  142. atomicshop/wrappers/socketw/sni.py +334 -165
  143. atomicshop/wrappers/socketw/socket_base.py +134 -0
  144. atomicshop/wrappers/socketw/socket_client.py +137 -95
  145. atomicshop/wrappers/socketw/socket_server_tester.py +11 -7
  146. atomicshop/wrappers/socketw/socket_wrapper.py +717 -116
  147. atomicshop/wrappers/socketw/ssl_base.py +15 -14
  148. atomicshop/wrappers/socketw/statistics_csv.py +148 -17
  149. atomicshop/wrappers/sysmonw.py +1 -1
  150. atomicshop/wrappers/ubuntu_terminal.py +65 -26
  151. atomicshop/wrappers/win_auditw.py +189 -0
  152. atomicshop/wrappers/winregw/__init__.py +0 -0
  153. atomicshop/wrappers/winregw/winreg_installed_software.py +58 -0
  154. atomicshop/wrappers/winregw/winreg_network.py +232 -0
  155. {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/METADATA +31 -51
  156. atomicshop-3.10.5.dist-info/RECORD +306 -0
  157. {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/WHEEL +1 -1
  158. atomicshop/_basics_temp.py +0 -101
  159. atomicshop/a_installs/win/fibratus.py +0 -9
  160. atomicshop/a_installs/win/mongodb.py +0 -9
  161. atomicshop/a_installs/win/pycharm.py +0 -9
  162. atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
  163. atomicshop/addons/a_setup_scripts/install_pywintrace_0.3.cmd +0 -2
  164. atomicshop/addons/mains/__pycache__/install_fibratus_windows.cpython-312.pyc +0 -0
  165. atomicshop/addons/mains/__pycache__/msi_unpacker.cpython-312.pyc +0 -0
  166. atomicshop/addons/mains/install_docker_rootless_ubuntu.py +0 -11
  167. atomicshop/addons/mains/install_docker_ubuntu_main_sudo.py +0 -11
  168. atomicshop/addons/mains/install_elastic_search_and_kibana_ubuntu.py +0 -10
  169. atomicshop/addons/mains/install_wsl_ubuntu_lts_admin.py +0 -9
  170. atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
  171. atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
  172. atomicshop/addons/package_setup/Setup.cmd +0 -7
  173. atomicshop/archiver/_search_in_zip.py +0 -189
  174. atomicshop/archiver/archiver.py +0 -34
  175. atomicshop/archiver/search_in_archive.py +0 -250
  176. atomicshop/archiver/sevenz_app_w.py +0 -86
  177. atomicshop/archiver/sevenzs.py +0 -44
  178. atomicshop/archiver/zips.py +0 -293
  179. atomicshop/file_types.py +0 -24
  180. atomicshop/mitm/config_editor.py +0 -37
  181. atomicshop/mitm/engines/create_module_template_example.py +0 -13
  182. atomicshop/mitm/initialize_mitm_server.py +0 -268
  183. atomicshop/pbtkmultifile_argparse.py +0 -88
  184. atomicshop/permissions.py +0 -151
  185. atomicshop/script_as_string_processor.py +0 -38
  186. atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
  187. atomicshop/ssh_scripts/process_from_port.py +0 -27
  188. atomicshop/wrappers/_process_wrapper_curl.py +0 -27
  189. atomicshop/wrappers/_process_wrapper_tar.py +0 -21
  190. atomicshop/wrappers/dockerw/install_docker.py +0 -209
  191. atomicshop/wrappers/elasticsearchw/infrastructure.py +0 -265
  192. atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -232
  193. atomicshop/wrappers/ffmpegw.py +0 -125
  194. atomicshop/wrappers/fibratusw/install.py +0 -81
  195. atomicshop/wrappers/mongodbw/infrastructure.py +0 -53
  196. atomicshop/wrappers/mongodbw/install_mongodb.py +0 -190
  197. atomicshop/wrappers/msiw.py +0 -149
  198. atomicshop/wrappers/nodejsw/install_nodejs.py +0 -139
  199. atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
  200. atomicshop/wrappers/psutilw/networks.py +0 -45
  201. atomicshop/wrappers/pycharmw.py +0 -81
  202. atomicshop/wrappers/socketw/base.py +0 -59
  203. atomicshop/wrappers/socketw/get_process.py +0 -107
  204. atomicshop/wrappers/wslw.py +0 -191
  205. atomicshop-2.15.11.dist-info/RECORD +0 -302
  206. /atomicshop/{addons/mains → a_mains}/FACT/factw_fact_extractor_docker_image_main_sudo.py +0 -0
  207. /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
  208. /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
  209. /atomicshop/{addons → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
  210. /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
  211. /atomicshop/{addons → a_mains/addons}/process_list/compile.cmd +0 -0
  212. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.dll +0 -0
  213. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.exp +0 -0
  214. /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.lib +0 -0
  215. /atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +0 -0
  216. /atomicshop/{archiver → permissions}/__init__.py +0 -0
  217. /atomicshop/{wrappers/fibratusw → web_apis}/__init__.py +0 -0
  218. /atomicshop/wrappers/{nodejsw → pywin32w/wmis}/__init__.py +0 -0
  219. /atomicshop/wrappers/pywin32w/{wmi_win32process.py → wmis/win32process.py} +0 -0
  220. {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info/licenses}/LICENSE.txt +0 -0
  221. {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/top_level.txt +0 -0
@@ -1,381 +1,899 @@
1
+ import multiprocessing
1
2
  from datetime import datetime
2
-
3
- from .message import ClientMessage
4
- from .initialize_engines import assign_class_by_domain
5
- from ..wrappers.socketw.receiver import Receiver
6
- from ..wrappers.socketw.sender import Sender
7
- from ..wrappers.socketw.socket_client import SocketClient
3
+ import threading
4
+ import queue
5
+ import socket
6
+ import ssl
7
+ from typing import Literal
8
+ import struct
9
+
10
+ from ..wrappers.socketw import receiver, sender, socket_client, socket_base
11
+ from .. import websocket_parse, ip_addresses
8
12
  from ..http_parse import HTTPRequestParse, HTTPResponseParse
9
- from ..basics.threads import current_thread_id
13
+ from ..basics import threads, tracebacks
10
14
  from ..print_api import print_api
11
15
 
16
+ from .message import ClientMessage
17
+ from . import initialize_engines
18
+ from ..wrappers.loggingw import loggingw
19
+ # This is needed only for the data typing.
20
+ from . import config_static as cf
21
+
12
22
 
13
- # Thread function on client connect.
14
23
  def thread_worker_main(
15
- function_client_socket_object,
24
+ # These parameters come from the SocketWrapper.
25
+ client_socket,
16
26
  process_commandline: str,
17
27
  is_tls: bool,
28
+ tls_type: str,
29
+ tls_version: str,
18
30
  domain_from_dns,
19
- network_logger,
20
- statistics_logger,
21
- engines_list,
22
- reference_module,
23
- config):
24
- def output_statistics_csv_row():
25
- statistics_dict['host'] = client_message.server_name
26
- try:
27
- statistics_dict['path'] = client_message.request_raw_decoded.path
28
- except Exception:
29
- pass
30
- try:
31
- statistics_dict['status_code'] = \
32
- ','.join([str(x.code) for x in client_message.response_list_of_raw_decoded])
33
- except Exception:
34
- pass
35
- try:
36
- statistics_dict['command'] = client_message.request_raw_decoded.command
37
- except Exception:
38
- pass
39
- try:
40
- statistics_dict['request_time_sent'] = client_message.request_time_received
41
- except Exception:
42
- pass
31
+ statistics_writer,
32
+ engines_list: list[initialize_engines.ModuleCategory],
33
+
34
+ # These parameters come from the main mitm module.
35
+ config_static: cf
36
+ ):
37
+ def output_statistics_csv_row(client_message: ClientMessage):
38
+ # If there is no '.code' attribute in HTTPResponse, this means that this is not an HTTP message, so there is no
39
+ # status code.
43
40
  try:
44
- statistics_dict['request_size_bytes'] = len(client_message.request_raw_bytes)
45
- except Exception:
46
- pass
47
- try:
48
- statistics_dict['response_size_bytes'] = \
49
- ','.join([str(len(x)) for x in client_message.response_list_of_raw_bytes])
50
- except Exception:
51
- pass
52
- # try:
53
- # statistics_dict['request_hex'] = client_message.request_raw_hex
54
- # except Exception:
55
- # pass
56
- # try:
57
- # statistics_dict['response_hex'] = \
58
- # f'"' + ','.join([x for x in client_message.response_list_of_raw_hex]) + '"'
59
- # except Exception:
60
- # pass
41
+ http_status_code: str = str(client_message.response_auto_parsed.code)
42
+ except AttributeError:
43
+ http_status_code: str = str()
44
+
45
+ # Same goes for the '.path' attribute, if it is not HTTP message then there will be no path.
61
46
  try:
62
- statistics_dict['file_path'] = recorded_file
63
- except Exception:
64
- pass
47
+ if client_message.request_auto_parsed and client_message.request_auto_parsed.path:
48
+ http_path: str = client_message.request_auto_parsed.path
49
+ elif client_message.response_auto_parsed.path:
50
+ http_path: str = client_message.response_auto_parsed.path
51
+ else:
52
+ http_path: str = str()
53
+ except AttributeError:
54
+ http_path: str = str()
55
+
56
+ # Same goes for the '.command' attribute, if it is not HTTP message then there will be no command.
65
57
  try:
66
- statistics_dict['process_cmd'] = process_commandline
67
- except Exception:
68
- pass
69
- statistics_dict['error'] = str()
70
-
71
- statistics_logger.info(f"{statistics_dict['request_time_sent']},"
72
- f"{statistics_dict['host']},"
73
- f"\"{statistics_dict['path']}\","
74
- f"{statistics_dict['command']},"
75
- f"{statistics_dict['status_code']},"
76
- f"{statistics_dict['request_size_bytes']},"
77
- f"{statistics_dict['response_size_bytes']},"
78
- # f"{statistics_dict['request_hex']},"
79
- # f"{statistics_dict['response_hex']},"
80
- f"\"{statistics_dict['file_path']}\","
81
- f"\"{statistics_dict['process_cmd']}\","
82
- f"{statistics_dict['error']}"
83
- )
58
+ http_command: str = client_message.request_auto_parsed.command
59
+ except AttributeError:
60
+ http_command: str = str()
84
61
 
85
- try:
86
- # Defining variables before assignment
87
- function_recorded: bool = False
88
- client_message: ClientMessage = ClientMessage()
89
- request_decoded = None
90
- service_client = None
91
-
92
- # Getting thread ID of the current thread and putting to the client message class
93
- client_message.thread_id = current_thread_id()
94
- # Get client ip and port
95
- client_message.client_ip, client_message.source_port = function_client_socket_object.getpeername()
96
- # Get destination port
97
- client_message.destination_port = function_client_socket_object.getsockname()[1]
98
- # Putting the process command line.
99
- client_message.process_name = process_commandline
62
+ if client_message.request_raw_bytes is None:
63
+ request_size_bytes = ''
64
+ else:
65
+ request_size_bytes = str(len(client_message.request_raw_bytes))
66
+
67
+ if client_message.response_raw_bytes is None:
68
+ response_size_bytes = ''
69
+ else:
70
+ response_size_bytes = str(len(client_message.response_raw_bytes))
71
+
72
+ if client_message.errors and len(client_message.errors) > 1:
73
+ error_string = '||'.join(client_message.errors)
74
+ error_string = f'Error count: {len(client_message.errors)} | Errors: {error_string}'
75
+ elif client_message.errors and len(client_message.errors) == 1:
76
+ error_string = client_message.errors[0]
77
+ elif not client_message.errors:
78
+ error_string = str()
79
+ else:
80
+ raise ValueError(f"Error in statistics error list. Values: {client_message.errors}")
81
+
82
+ statistics_writer.write_row(
83
+ thread_id=str(thread_id),
84
+ engine=client_message.engine_name,
85
+ source_host=client_message.client_name,
86
+ source_ip=client_message.client_ip,
87
+ tls_type=tls_type,
88
+ tls_version=tls_version,
89
+ protocol=client_message.protocol,
90
+ protocol2=client_message.protocol2,
91
+ protocol3=client_message.protocol3,
92
+ dest_port=client_message.destination_port,
93
+ host=client_message.server_name,
94
+ path=http_path,
95
+ status_code=http_status_code,
96
+ command=http_command,
97
+ timestamp=client_message.timestamp,
98
+ request_size_bytes=request_size_bytes,
99
+ response_size_bytes=response_size_bytes,
100
+ recorded_file_path=client_message.recorded_file_path,
101
+ process_cmd=process_commandline,
102
+ action=client_message.action,
103
+ error=error_string
104
+ )
105
+
106
+ def record_and_statistics_write(client_message: ClientMessage):
107
+ # If recorder wasn't executed before, then execute it now
108
+ if config_static.LogRec.enable_request_response_recordings_in_logs:
109
+ recorded_file = recorder.record(class_client_message=client_message)
110
+ client_message.recorded_file_path = recorded_file
111
+
112
+ # Save statistics file.
113
+ output_statistics_csv_row(client_message)
114
+
115
+ def parse_http(
116
+ raw_bytes: bytes,
117
+ client_message: ClientMessage):
118
+ nonlocal protocol
119
+
120
+ # Parsing the raw bytes as HTTP.
121
+ request_http_parsed, is_http_request, request_parsing_error = (
122
+ HTTPRequestParse(raw_bytes).parse())
123
+
124
+ response_http_parsed, is_http_response, response_parsing_error = (
125
+ HTTPResponseParse(raw_bytes).parse())
126
+
127
+ if is_http_request:
128
+ if protocol == '':
129
+ protocol = 'HTTP'
130
+
131
+ auto_parsed = request_http_parsed
132
+ network_logger.info(
133
+ f"HTTP Request Parsed: Method: {request_http_parsed.command} | Path: {request_http_parsed.path}")
134
+ http_path_queue.put(request_http_parsed.path)
135
+ network_logger.info(f"HTTP Request Parsed: Putting PATH to queue.")
136
+
137
+ is_http_request_a_websocket(auto_parsed, client_message)
138
+ elif is_http_response:
139
+ auto_parsed = response_http_parsed
140
+ network_logger.info(
141
+ f"HTTP Response Parsed: Status: {response_http_parsed.code}")
142
+
143
+ auto_parsed.path = http_path_queue.get()
144
+ network_logger.info(f"HTTP Response Parsed: Got PATH from queue: [{auto_parsed.path}]")
145
+ elif protocol == 'Websocket':
146
+ client_message.protocol2 = 'Frame'
147
+ auto_parsed = parse_websocket(raw_bytes)
148
+ if protocol3:
149
+ client_message.protocol3 = protocol3
150
+ else:
151
+ auto_parsed = None
152
+
153
+ return auto_parsed
154
+
155
+ def is_http_request_a_websocket(
156
+ auto_parsed,
157
+ client_message: ClientMessage):
158
+ nonlocal protocol
159
+ nonlocal protocol3
160
+
161
+ if protocol == 'HTTP':
162
+ if auto_parsed and hasattr(auto_parsed, 'headers') and 'Upgrade' in auto_parsed.headers:
163
+ if auto_parsed.headers['Upgrade'] == 'websocket':
164
+ protocol = 'Websocket'
165
+ client_message.protocol2 = 'Handshake'
166
+ protocol3 = auto_parsed.headers.get('Sec-WebSocket-Protocol', None)
167
+ if protocol3:
168
+ client_message.protocol3 = protocol3
169
+
170
+ network_logger.info(f'Protocol upgraded to Websocket')
171
+
172
+ def parse_websocket(raw_bytes):
173
+ return websocket_frame_parser.parse_frame_bytes(raw_bytes)
174
+
175
+ def finish_thread(send_connection_reset: bool = False):
176
+ """
177
+ Finishing the thread, closing sockets.
178
+
179
+ :param send_connection_reset: Whether to send TCP RST flag to the client when closing the socket.
180
+ Basically what happens is that the server socket is closed abruptly sending us any Connection*Error exception.
181
+ We can simulate the ConnectionResetError on the client side by sending TCP RST flag when closing the socket.
182
+ But not the ConnectionAbortedError, since it is caused by other reasons (local TCP stack and has nothing
183
+ to do with the remote server).
184
+ """
185
+ # At this stage there could be several times that the same socket was used to the service server - we need to
186
+ # close this socket as well if it still opened.
187
+ # The first part of the condition is to check if the service socket was connected at all.
188
+ # If the service socket couldn't connect, then the instance will be None.
189
+ if service_socket_instance and service_socket_instance.fileno() != -1:
190
+ if origin_service_client_instance.socket_instance:
191
+ origin_service_client_instance.close_socket()
192
+
193
+ # If client socket is still opened - close
194
+ if client_socket.fileno() != -1:
195
+ if send_connection_reset:
196
+ # Abort the connection
197
+ linger = struct.pack('ii', 1, 0)
198
+ client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger)
199
+ client_socket.close()
200
+
201
+ if send_connection_reset:
202
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}] with TCP RST flag (Sent ConnectionResetError)...")
203
+ else:
204
+ network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
205
+
206
+ network_logger.info("Thread Finished. Will continue listening on the Main thread")
207
+
208
+ def create_requester_request(
209
+ client_message: ClientMessage,
210
+ sending_socket: socket.socket
211
+ ) -> tuple[bytes, bool]:
212
+ request_received_raw: bytes = client_message.request_raw_bytes
213
+ request_custom_raw: bytes = requester.create_request(client_message, sending_socket=sending_socket)
100
214
 
101
- # Only protocols that are encrypted with TLS have the server name attribute.
102
- if is_tls:
103
- # Get current destination domain
104
- client_message.server_name = function_client_socket_object.server_hostname
105
- # client_message.server_name = domain_from_dns
106
- # If the protocol is not TLS, then we'll use the domain from the DNS.
215
+ if request_custom_raw is None or request_received_raw == request_custom_raw:
216
+ is_requester_worked: bool = False
107
217
  else:
108
- client_message.server_name = domain_from_dns
109
-
110
- network_logger.info(f"Thread Created - Client [{client_message.client_ip}:{client_message.source_port}] | "
111
- f"Destination service: [{client_message.server_name}:{client_message.destination_port}]")
112
-
113
- # Loading parser by domain, if there is no parser for current domain - general reference parser is loaded.
114
- # These should be outside any loop and initialized only once entering the thread.
115
- parser, responder, recorder = assign_class_by_domain(
116
- engines_list, client_message.server_name, reference_module=reference_module, config=config,
117
- logger=network_logger)
118
-
119
- # Defining client connection boolean variable to enter the loop
120
- client_connection_boolean: bool = True
121
-
122
- # Loop while received message is not empty, if so, close socket, since other side already closed.
123
- while client_connection_boolean:
124
- # Don't forget that 'ClientMessage' object is being reused at this step.
125
- # Meaning, that each list / dictionary that is used to update at
126
- # this loop section needs to be reinitialized.
127
- # Add any variables that need reinitializing in the 'ClientMessage' class 'reinitialize' function.
128
- client_message.reinitialize()
129
-
130
- # Initialize statistics_dict for the same reason as 'client_message.reinitialize()'.
131
- statistics_dict: dict = dict()
132
- statistics_dict['host'] = client_message.server_name
133
- statistics_dict['path'] = str()
134
- statistics_dict['status_code'] = str()
135
- statistics_dict['command'] = str()
136
- statistics_dict['request_time_sent'] = str()
137
- statistics_dict['request_size_bytes'] = str()
138
- # statistics_dict['response_time_sent'] = str()
139
- statistics_dict['response_size_bytes'] = str()
140
- statistics_dict['file_path'] = str()
141
- statistics_dict['process_cmd'] = str()
142
- statistics_dict['error'] = str()
143
-
144
- network_logger.info("Initializing Receiver")
145
- # Getting message from the client over the socket using specific class.
146
- client_received_raw_data = Receiver(function_client_socket_object).receive()
147
-
148
- # If the message is empty, then the connection was closed already by the other side,
149
- # so we can close the socket as well.
150
- # If the received message from the client is not empty, then continue.
151
- if client_received_raw_data:
152
- # Putting the received message to the aggregating message class.
153
- client_message.request_raw_bytes = client_received_raw_data
154
- # Getting current time of message received from client.
155
- client_message.request_time_received = datetime.now()
156
-
157
- # HTTP Parsing section =================================================================================
158
- # Parsing the raw bytes as HTTP.
159
- try:
160
- request_decoded = HTTPRequestParse(client_message.request_raw_bytes)
161
- except Exception:
162
- message = "There was an exception in HTTP Parsing module!"
163
- print_api(
164
- message, error_type=True, logger=network_logger, logger_method='critical',
165
- traceback_string=True, oneline=True)
166
- # Socket connection can be closed since we have a problem in current thread and break the loop
167
- client_connection_boolean = False
168
- break
169
-
170
- # Getting the status of http parsing
171
- request_is_http, http_parsing_reason, http_parsing_error = request_decoded.check_if_http()
172
-
173
- # Currently, we don't care if it's HTTP. If there was no error we can continue. Just log the reason.
174
- if not http_parsing_error:
175
- network_logger.info(http_parsing_reason)
176
- # If there was error - the request is really HTTP, but there's a problem with its structure.
177
- # So, we'll stop the loop.
218
+ is_requester_worked: bool = True
219
+
220
+ # Output first 100 characters of the request.
221
+ requester.logger.info(f"{request_custom_raw[0: 100]}...")
222
+
223
+ return request_custom_raw, is_requester_worked
224
+
225
+ def create_responder_response(client_message: ClientMessage) -> list[bytes]:
226
+ if client_message.action == 'service_connect':
227
+ return responder.create_connect_response(client_message)
228
+ else:
229
+ # If we're in offline mode, and it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
230
+ # response automatically.
231
+ if config_static.MainConfig.is_offline and protocol == 'Websocket' and client_receive_count == 1:
232
+ responses: list = list()
233
+ responses.append(
234
+ websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
235
+ responder.logger.info(f"Generated automatic WebSocket response in Offline Mode.")
236
+ else:
237
+ # Creating response for parsed message and printing
238
+ responder_responses: list = responder.create_response(client_message)
239
+ if responder_responses is None:
240
+ responses: list = [client_message.response_raw_bytes]
178
241
  else:
179
- client_message.error = http_parsing_reason
180
- network_logger.critical(client_message.error)
181
- break
182
-
183
- # If the request is HTTP protocol.
184
- if request_is_http:
185
- network_logger.info(f"Method: {request_decoded.command}")
186
- network_logger.info(f"Path: {request_decoded.path}")
187
- # statistics.dict['path'] = request_decoded.path
188
- client_message.request_raw_decoded = request_decoded
189
- # HTTP Parsing section EOF =============================================================================
190
-
191
- # Catching exceptions in the parser
192
- try:
193
- parser(client_message).parse()
194
- except Exception:
195
- message = "Exception in Parser"
196
- print_api(
197
- message, error_type=True, logger=parser.logger, logger_method='critical',
198
- traceback_string=True, oneline=True)
199
- print_api(
200
- message, error_type=True, logger=network_logger, logger_method='critical',
201
- traceback_string=True, oneline=True)
202
- # At this point we can pass the exception and continue the script.
203
- pass
204
- # Socket connection can be closed since we have a problem in current thread and break the loop
205
- client_connection_boolean = False
206
- break
207
-
208
- # Converting body parsed to string, since there is no strict rule for the parameter to be string.
209
- # Still going to put exception on it, since it is not critical for the server.
210
- # Won't even log the exception.
211
- try:
212
- parser.logger.info(f"{str(client_message.request_body_parsed)[0: 100]}...")
213
- except Exception:
214
- pass
215
-
216
- # If we're in response mode, execute responder.
217
- if config['tcp']['server_response_mode']:
218
- # Since we're in response mode, we'll record the request anyway, after the responder did its job.
219
- client_message.info = "In Server Response Mode"
220
-
221
- # Re-initiate the 'client_message.response_list_of_raw_bytes' list, since we'll be appending
222
- # new entries for empty list.
223
- client_message.response_list_of_raw_bytes = list()
224
- # Creating response for parsed message and printing
225
- try:
226
- responder.create_response(client_message)
227
- except Exception:
228
- message = "Exception in Responder"
229
- print_api(
230
- message, error_type=True, logger=responder.logger, logger_method='critical',
231
- traceback_string=True, oneline=True)
232
- print_api(
233
- message, error_type=True, logger=network_logger, logger_method='critical',
234
- traceback_string=True, oneline=True)
235
- pass
236
- # Socket connection can be closed since we have a problem in current thread and break the loop.
237
- client_connection_boolean = False
238
- break
239
-
240
- # Output first 100 characters of all the responses in the list.
241
- for response_raw_bytes in client_message.response_list_of_raw_bytes:
242
- if response_raw_bytes:
243
- responder.logger.info(f"{response_raw_bytes[0: 100]}...")
244
- else:
245
- responder.logger.info(f"Response empty...")
246
- # Else, we're not in response mode, then execute client connect and record section.
242
+ responses: list = responder_responses
243
+ responder.logger.info(f"Generated {len(responses)} responses from responder.")
244
+
245
+ # Output first 100 characters of all the responses in the list.
246
+ for response_raw_bytes_single in responses:
247
+ responder.logger.info(f"{response_raw_bytes_single[0: 100]}...")
248
+
249
+ return responses
250
+
251
+ def create_client_socket(client_message: ClientMessage):
252
+ # If there is a custom certificate for the client for this domain, then we'll use it.
253
+ # noinspection PyTypeChecker
254
+ custom_client_pem_certificate_path: str = None
255
+ for subdomain, pem_file_path in found_domain_module.mtls.items():
256
+ if subdomain == client_message.server_name:
257
+ custom_client_pem_certificate_path = pem_file_path
258
+ break
259
+
260
+ # Check if the destination service is an ip address or a domain name.
261
+ if ip_addresses.is_ip_address(client_message.server_name, ip_type='ipv4'):
262
+ # If it's an ip address, connect to the ip address directly.
263
+ service_client_instance = socket_client.SocketClient(
264
+ service_name=client_message.server_name,
265
+ connection_ip=client_message.server_name,
266
+ service_port=client_message.destination_port,
267
+ tls=is_tls,
268
+ logger=network_logger,
269
+ custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
270
+ enable_sslkeylogfile_env_to_client_ssl_context=(
271
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
272
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
273
+ )
274
+ # If it's a domain name, then we'll use the DNS to resolve it.
275
+ else:
276
+ # If we're on localhost, then use external services list in order to resolve the domain:
277
+ # config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
278
+ if client_message.client_ip in socket_base.THIS_DEVICE_IP_LIST:
279
+ service_client_instance = socket_client.SocketClient(
280
+ service_name=client_message.server_name,
281
+ service_port=client_message.destination_port,
282
+ tls=is_tls,
283
+ dns_servers_list=[config_static.DNSServer.forwarding_dns_service_ipv4],
284
+ logger=network_logger,
285
+ custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
286
+ enable_sslkeylogfile_env_to_client_ssl_context=(
287
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
288
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
289
+ )
290
+ # If we're not on localhost, then connect to domain directly.
291
+ else:
292
+ service_client_instance = socket_client.SocketClient(
293
+ service_name=client_message.server_name,
294
+ service_port=client_message.destination_port,
295
+ tls=is_tls,
296
+ logger=network_logger,
297
+ custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
298
+ enable_sslkeylogfile_env_to_client_ssl_context=(
299
+ config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
300
+ sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
301
+ )
302
+
303
+ return service_client_instance
304
+
305
+ def process_client_raw_data(
306
+ client_received_raw_data: bytes,
307
+ error_string: str,
308
+ client_message: ClientMessage):
309
+ """
310
+ Process the client raw data request.
311
+ """
312
+ nonlocal protocol
313
+
314
+ client_message.request_raw_bytes = client_received_raw_data
315
+
316
+ if error_string:
317
+ client_message.errors.append(error_string)
318
+
319
+ if client_received_raw_data == b'' or client_received_raw_data is None:
320
+ return
321
+
322
+ client_message.request_auto_parsed = parse_http(client_message.request_raw_bytes, client_message)
323
+ # This is needed for each cycle that is not HTTP, but its protocol maybe set by HTTP, like websocket.
324
+ if protocol != '':
325
+ client_message.protocol = protocol
326
+
327
+ # Parse websocket frames only if it is not the first protocol upgrade request.
328
+ if protocol == 'Websocket' and client_receive_count != 1:
329
+ client_message.request_auto_parsed = parse_websocket(client_message.request_raw_bytes)
330
+
331
+ # Custom parser, should parse HTTP body or the whole message if not HTTP.
332
+ parser_instance = parser(client_message)
333
+ parser_instance.parse()
334
+
335
+ # Converting body parsed to string on logging, there is no strict rule for the parameter to be string.
336
+ parser_instance.logger.info(f"{str(client_message.request_custom_parsed)[0: 100]}...")
337
+
338
+ def process_server_raw_data(
339
+ service_received_raw_data: bytes,
340
+ error_string: str,
341
+ client_message: ClientMessage
342
+ ):
343
+ nonlocal protocol
344
+
345
+ client_message.response_raw_bytes = service_received_raw_data
346
+
347
+ if error_string:
348
+ client_message.errors.append(error_string)
349
+
350
+ if service_received_raw_data == b'' or service_received_raw_data is None:
351
+ return
352
+
353
+ client_message.response_auto_parsed = parse_http(client_message.response_raw_bytes, client_message)
354
+ if protocol != '':
355
+ client_message.protocol = protocol
356
+
357
+ def client_message_first_start() -> ClientMessage:
358
+ client_message: ClientMessage = ClientMessage()
359
+ client_message.client_name = client_name
360
+ client_message.client_ip = client_ip
361
+ client_message.server_ip = server_ip
362
+ client_message.source_port = source_port
363
+ client_message.destination_port = destination_port
364
+ client_message.server_name = server_name
365
+ client_message.thread_id = thread_id
366
+ client_message.thread_process = thread_process_name
367
+ client_message.process_name = process_commandline
368
+ client_message.engine_name = engine_name
369
+
370
+ return client_message
371
+
372
+ def receive_send_service_connect(
373
+ client_connection_message: ClientMessage,
374
+ sending_socket: ssl.SSLSocket | socket.socket
375
+ ) -> Literal['continue', 'return'] | None:
376
+
377
+ nonlocal exception_or_close_in_receiving_thread
378
+
379
+ client_message = client_connection_message
380
+
381
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
382
+ print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]",
383
+ logger=network_logger, logger_method='info')
384
+
385
+ # If the client message is the connection message, then we'll skip to the next iteration.
386
+ if not bytes_to_send_list:
387
+ return 'continue'
388
+
389
+ # is_socket_closed: bool = False
390
+ error_on_send: str = str()
391
+ for bytes_to_send_single in bytes_to_send_list:
392
+ client_message.reinitialize_dynamic_vars()
393
+ client_message.timestamp = datetime.now()
394
+ client_message.response_raw_bytes = bytes_to_send_single
395
+ client_message.action = 'service_responder'
396
+ record_and_statistics_write(client_message)
397
+
398
+ # Send the bytes back to the client socket.
399
+ error_on_send: str = sender.Sender(
400
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
401
+ logger=network_logger).send()
402
+
403
+ if error_on_send:
404
+ client_message.reinitialize_dynamic_vars()
405
+ client_message.errors.append(error_on_send)
406
+ client_message.timestamp = datetime.now()
407
+ client_message.action = 'client_send'
408
+ record_and_statistics_write(client_message)
409
+
410
+ if error_on_send:
411
+ exception_or_close_in_receiving_thread = True
412
+ finish_thread()
413
+ return 'return'
414
+
415
+ return None
416
+
417
+ def receive_send_client_offline(
418
+ client_message: ClientMessage,
419
+ receiving_socket: ssl.SSLSocket | socket.socket,
420
+ sending_socket: ssl.SSLSocket | socket.socket
421
+ ) -> Literal['return'] | None:
422
+ nonlocal exception_or_close_in_receiving_thread
423
+ nonlocal client_receive_count
424
+
425
+ client_receive_count += 1
426
+
427
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
428
+ client_message.timestamp = datetime.now()
429
+
430
+ received_raw_data, is_socket_closed, error_message = receiver.Receiver(
431
+ ssl_socket=receiving_socket, logger=network_logger).receive()
432
+
433
+ process_client_raw_data(received_raw_data, error_message, client_message)
434
+ client_message.action = 'client_receive'
435
+ record_and_statistics_write(client_message)
436
+ if error_message:
437
+ print_api(error_message, logger=network_logger, logger_method='critical')
438
+
439
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
440
+ # the socket was closed by the other side, it means that the service thread closed the socket.
441
+ if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
442
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
443
+ logger_method='info')
444
+ return 'return'
445
+
446
+ # If the socket was closed on receive, and we're in offline mode, then we'll finish the thread right away.
447
+ # Since nothing more can be done, like responding to service or using requester.
448
+ if is_socket_closed:
449
+ exception_or_close_in_receiving_thread = True
450
+ finish_thread()
451
+ return 'return'
452
+
453
+ # Send to requester.
454
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
455
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
456
+ # We will not process the raw data if requester didn't change anything.
457
+ if is_requester_worked:
458
+ client_message.reinitialize_dynamic_vars()
459
+ client_message.timestamp = datetime.now()
460
+ client_message.request_raw_bytes = request_custom_raw
461
+ client_message.action = 'client_requester'
462
+ process_client_raw_data(request_custom_raw, error_message, client_message)
463
+ record_and_statistics_write(client_message)
464
+
465
+ print_api("Offline Mode, sending to responder directly.", logger=network_logger,
466
+ logger_method='info')
467
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
468
+
469
+ error_on_send: str = str()
470
+ for bytes_to_send_single in bytes_to_send_list:
471
+ client_message.reinitialize_dynamic_vars()
472
+ client_message.timestamp = datetime.now()
473
+ client_message.response_raw_bytes = bytes_to_send_single
474
+ client_message.action = 'client_responder_offline'
475
+ process_server_raw_data(bytes_to_send_single, '', client_message)
476
+ record_and_statistics_write(client_message)
477
+
478
+ error_on_send: str = sender.Sender(
479
+ ssl_socket=receiving_socket, bytes_to_send=bytes_to_send_single,
480
+ logger=network_logger).send()
481
+
482
+ if error_on_send:
483
+ client_message.reinitialize_dynamic_vars()
484
+ client_message.errors.append(error_on_send)
485
+ client_message.timestamp = datetime.now()
486
+ client_message.action = 'service_send'
487
+
488
+ record_and_statistics_write(client_message)
489
+
490
+ # If the socket was closed on message receive, then we'll break the loop only after send.
491
+ if is_socket_closed or error_on_send:
492
+ exception_or_close_in_receiving_thread = True
493
+ finish_thread()
494
+ return 'return'
495
+
496
+ return None
497
+
498
+ def receive_send_client(
499
+ client_message: ClientMessage,
500
+ receiving_socket: ssl.SSLSocket | socket.socket,
501
+ sending_socket: ssl.SSLSocket | socket.socket
502
+ ) -> Literal['return'] | None:
503
+
504
+ nonlocal exception_or_close_in_receiving_thread
505
+ nonlocal client_receive_count
506
+
507
+ client_receive_count += 1
508
+
509
+ network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
510
+
511
+ # Getting message from the client over the socket using specific class.
512
+ client_message.timestamp = datetime.now()
513
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
514
+ ssl_socket=receiving_socket, logger=network_logger).receive()
515
+
516
+ process_client_raw_data(received_raw_data, error_on_receive, client_message)
517
+ client_message.action = 'client_receive'
518
+
519
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
520
+ # the socket was closed by the other side, it means that the service thread closed the socket.
521
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
522
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
523
+ logger_method='info')
524
+ return 'return'
525
+
526
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
527
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
528
+ record_and_statistics_write(client_message)
529
+ if error_on_receive:
530
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
531
+
532
+ # At this point if the socket was closed on receive, then there's no point to send anything to the service.
533
+ # But if the data was received and then the socket was closed, we first send the data and then close the socket.
534
+ error_on_send: str = str()
535
+ if received_raw_data != b'' and received_raw_data is not None:
536
+ # Send to requester.
537
+ # THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
538
+ request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
539
+ # We will not process the raw data if requester didn't change anything.
540
+ if is_requester_worked:
541
+ client_message.reinitialize_dynamic_vars()
542
+ client_message.timestamp = datetime.now()
543
+ client_message.request_raw_bytes = request_custom_raw
544
+ client_message.action = 'client_requester'
545
+ process_client_raw_data(request_custom_raw, error_on_receive, client_message)
546
+ record_and_statistics_write(client_message)
547
+
548
+ error_on_send: str = sender.Sender(
549
+ ssl_socket=sending_socket, bytes_to_send=client_message.request_raw_bytes,
550
+ logger=network_logger).send()
551
+
552
+ if error_on_send:
553
+ client_message.reinitialize_dynamic_vars()
554
+ client_message.errors.append(error_on_send)
555
+ client_message.timestamp = datetime.now()
556
+ client_message.action = 'service_send'
557
+ record_and_statistics_write(client_message)
558
+
559
+ # If the socket was closed on message receive, then we'll break the loop only after send.
560
+ if is_socket_closed or error_on_send:
561
+ exception_or_close_in_receiving_thread = True
562
+ finish_thread()
563
+ return 'return'
564
+
565
+ return None
566
+
567
+ def receive_send_service(
568
+ client_message: ClientMessage,
569
+ receiving_socket: ssl.SSLSocket | socket.socket,
570
+ sending_socket: ssl.SSLSocket | socket.socket
571
+ ) -> Literal['return'] | None:
572
+
573
+ nonlocal exception_or_close_in_receiving_thread
574
+ nonlocal server_receive_count
575
+
576
+ server_receive_count += 1
577
+
578
+ network_logger.info(f"Initializing Receiver for Service cycle: {str(server_receive_count)}")
579
+
580
+ # Getting message from the client over the socket using specific class.
581
+ client_message.timestamp = datetime.now()
582
+ received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
583
+ ssl_socket=receiving_socket, logger=network_logger).receive()
584
+
585
+ process_server_raw_data(received_raw_data, error_on_receive, client_message)
586
+ client_message.action = 'service_receive'
587
+
588
+ # If there was an exception in the service thread, then receiving empty bytes doesn't mean that
589
+ # the socket was closed by the other side, it means that the service thread closed the socket.
590
+ if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
591
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger,
592
+ logger_method='info')
593
+ return 'return'
594
+
595
+ # We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
596
+ # Meaning 'exception_or_close_in_receiving_thread=True'.
597
+ record_and_statistics_write(client_message)
598
+ if error_on_receive:
599
+ print_api(error_on_receive, logger=network_logger, logger_method='critical')
600
+
601
+ # At this stage, we have received the response from the service, but there was an exception/error or the socket was simply closed.
602
+ # Meaning the 'received_raw_data' is None for exception or b'' for just closed socket.
603
+ # So, there's no point to send anything back to the client.
604
+ # Close both sockets and finish the threads.
605
+ # But if the data was received and then the socket was closed, we first send the data and then close the socket.
606
+ error_on_send: str = str()
607
+ if received_raw_data != b'' and received_raw_data is not None:
608
+ # Now send it to requester/responder.
609
+ bytes_to_send_list: list[bytes] = create_responder_response(client_message)
610
+
611
+ # is_socket_closed: bool = False
612
+ for bytes_to_send_single in bytes_to_send_list:
613
+ client_message.reinitialize_dynamic_vars()
614
+ client_message.timestamp = datetime.now()
615
+ client_message.response_raw_bytes = bytes_to_send_single
616
+
617
+ # This records the requester or responder output, only if it is not the same as the original
618
+ # message.
619
+ if bytes_to_send_single != received_raw_data:
620
+ client_message.action = 'service_responder'
621
+ record_and_statistics_write(client_message)
622
+
623
+ error_on_send: str = sender.Sender(
624
+ ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
625
+ logger=network_logger).send()
626
+
627
+ if error_on_send:
628
+ client_message.reinitialize_dynamic_vars()
629
+ client_message.errors.append(error_on_send)
630
+ client_message.timestamp = datetime.now()
631
+ client_message.action = 'client_send'
632
+
633
+ record_and_statistics_write(client_message)
634
+
635
+ # If the socket was closed on message receive, then we'll break the loop only after send.
636
+ if is_socket_closed or error_on_send:
637
+ exception_or_close_in_receiving_thread = True
638
+
639
+ if error_on_receive and 'Connection' in error_on_receive and 'Error' in error_on_receive:
640
+ # If there was a connection error on receive from client, and we're closing the socket to the client,
641
+ # then we'll send TCP RST flag to simulate ConnectionResetError on the client side.
642
+ finish_thread(send_connection_reset=True)
643
+ else:
644
+ finish_thread()
645
+
646
+ return 'return'
647
+
648
+ return None
649
+
650
+
651
+ def receive_send_start(
652
+ receiving_socket,
653
+ sending_socket = None,
654
+ exception_queue: queue.Queue = None,
655
+ client_connection_message: ClientMessage = None
656
+ ):
657
+ nonlocal client_receive_count
658
+ nonlocal server_receive_count
659
+ nonlocal exception_or_close_in_receiving_thread
660
+
661
+ # Set the thread name to the custom name for logging
662
+ # threading.current_thread().name = thread_name
663
+
664
+ # Initialize the client message object with current thread's data.
665
+ client_message: ClientMessage = client_message_first_start()
666
+
667
+ try:
668
+ if receiving_socket is client_socket:
669
+ side: str = 'Client'
670
+ elif receiving_socket is service_socket_instance:
671
+ side: str = 'Service'
672
+ else:
673
+ raise ValueError(f"Unknown side of the socket: {receiving_socket}")
674
+
675
+ while True:
676
+ client_message.reinitialize_dynamic_vars()
677
+
678
+ if side == 'Service' and client_connection_message:
679
+ result: Literal['continue', 'return'] | None = (
680
+ receive_send_service_connect(client_connection_message, sending_socket))
681
+ client_connection_message = None
682
+ if result == 'continue':
683
+ continue
684
+ elif side == 'Client' and config_static.MainConfig.is_offline:
685
+ result: Literal['return'] | None = receive_send_client_offline(client_message, receiving_socket, sending_socket)
686
+ elif side == 'Client':
687
+ result: Literal['return'] | None = receive_send_client(client_message, receiving_socket, sending_socket)
688
+ elif side == 'Service':
689
+ result: Literal['return'] | None = receive_send_service(client_message, receiving_socket, sending_socket)
247
690
  else:
248
- # If "service_client" object is not defined, we'll define it.
249
- # If it's defined, then it means there's still active "ssl_socket" with connection to the service
250
- # domain.
251
- if not service_client:
252
- # If we're on localhost, then use external services list in order to resolve the domain:
253
- # config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
254
- if client_message.client_ip == "127.0.0.1":
255
- service_client = SocketClient(
256
- service_name=client_message.server_name, service_port=client_message.destination_port,
257
- tls=is_tls,
258
- dns_servers_list=config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost'])
259
- # If we're not on localhost, then connect to domain directly.
260
- else:
261
- service_client = SocketClient(
262
- service_name=client_message.server_name, service_port=client_message.destination_port,
263
- tls=is_tls)
264
-
265
- # Sending current client message and receiving a response.
266
- # If there was an error it will be passed to "client_message" object class and if not, "None" will
267
- # be passed.
268
- # If there was connection error or socket close, then "ssl_socket" of the "service_client"
269
- # will be empty.
270
- response_raw_bytes, client_message.error, client_message.server_ip, service_ssl_socket =\
271
- service_client.send_receive_to_service(client_message.request_raw_bytes)
272
-
273
- # Since we need a list for raw bytes, we'll add the 'response_raw_bytes' to our list object.
274
- # But we need to re-initiate it first.
275
- client_message.response_list_of_raw_bytes = list()
276
- client_message.response_list_of_raw_bytes.append(response_raw_bytes)
277
-
278
- client_message.response_list_of_raw_decoded = list()
279
- # Make HTTP Response parsing only if there was response at all.
280
- if response_raw_bytes:
281
- response_raw_decoded = HTTPResponseParse(response_raw_bytes).response_raw_decoded
282
- client_message.response_list_of_raw_decoded.append(response_raw_decoded)
283
-
284
- # So if the socket was closed and there was an error we can break the loop
285
- if not service_ssl_socket:
286
- break
287
-
288
- # This is the point after the response mode check was finished.
289
- # Recording the message, doesn't matter what type of mode this is.
290
- try:
291
- recorded_file = recorder(class_client_message=client_message,
292
- record_path=config['recorder']['recordings_path']).record()
293
- except Exception:
294
- message = "Exception in Recorder"
295
- print_api(
296
- message, error_type=True, logger=recorder.logger, logger_method='critical',
297
- traceback_string=True, oneline=True)
298
- print_api(
299
- message, error_type=True, logger=network_logger, logger_method='critical',
300
- traceback_string=True, oneline=True)
301
- pass
302
-
303
- function_recorded = True
304
-
305
- # Save statistics file.
306
- output_statistics_csv_row()
307
-
308
- try:
309
- # If there is a response, then send it.
310
- if response_raw_bytes:
311
- # Sending response/s to client no matter if in record mode or not.
312
- network_logger.info(
313
- f"Sending messages to client: {len(client_message.response_list_of_raw_bytes)}")
314
- function_data_sent = None
315
-
316
- # Iterate through the list of byte responses.
317
- for response_raw_bytes in client_message.response_list_of_raw_bytes:
318
- function_data_sent = Sender(function_client_socket_object, response_raw_bytes).send()
319
-
320
- # If there was problem with sending data, we'll break current loop.
321
- if not function_data_sent:
322
- break
323
- # If there is no response, close the socket.
324
- else:
325
- function_data_sent = None
326
- network_logger.info(f"Response empty, nothing to send to client.")
327
- except Exception:
328
- message = "Not sending anything to the client, since there is no response available"
329
- print_api(
330
- message, error_type=True, logger=network_logger, logger_method='critical',
331
- traceback_string=True, oneline=True)
332
- # Pass the exception
333
- pass
334
- # Break the while loop
335
- break
336
-
337
- # If there was problem with sending data, we'll break the while loop
338
- if not function_data_sent:
339
- break
691
+ raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
692
+
693
+ if result == 'return':
694
+ return
695
+ except Exception as exc:
696
+ # If the sockets were already closed, then there is nothing to do here besides log.
697
+ # if (isinstance(exc, OSError) and exc.errno == 10038 and
698
+ # client_socket.fileno() == -1 and service_socket_instance.fileno() == -1):
699
+ if isinstance(exc, OSError) and exc.errno == 10038:
700
+ print_api("Both sockets are closed, breaking the loop", logger=network_logger, logger_method='info')
340
701
  else:
341
- # Ending the while loop, basically we can use 'break'
342
- client_connection_boolean = False
343
- # We don't need to record empty message so setting the recorder state to recorded
344
- function_recorded = True
702
+ handle_exceptions_on_sub_connection_thread(client_message, exception_queue, exc)
703
+
704
+ def handle_exceptions_on_sub_connection_thread(
705
+ client_message: ClientMessage,
706
+ exception_queue: queue.Queue,
707
+ exc: Exception
708
+ ):
709
+ nonlocal exception_or_close_in_receiving_thread
710
+
711
+ exception_or_close_in_receiving_thread = True
712
+ # handle_exceptions(exc, client_message, recorded)
713
+ exception_message = tracebacks.get_as_string(one_line=True)
714
+
715
+ error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
716
+ print_api("Exception in a thread, forwarding to parent thread.", logger_method='info', logger=network_logger)
717
+ client_message.errors.append(error_message)
718
+
719
+ # if not recorded:
720
+ # record_and_statistics_write(client_message)
721
+
722
+ finish_thread()
723
+ exception_queue.put(exc)
724
+
725
+ def handle_exceptions_on_main_connection_thread(
726
+ exc: Exception,
727
+ client_message: ClientMessage
728
+ ):
729
+ exception_message = tracebacks.get_as_string(one_line=True)
730
+ error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
731
+ print_api(error_message, logger_method='critical', logger=network_logger)
732
+ client_message.errors.append(error_message)
345
733
 
346
734
  # === At this point while loop of 'client_connection_boolean' was broken =======================================
347
735
  # If recorder wasn't executed before, then execute it now
348
- if not function_recorded:
349
- try:
350
- recorded_file = recorder(
351
- class_client_message=client_message, record_path=config['recorder']['recordings_path']).record()
352
- except Exception:
353
- message = "Exception in Recorder"
354
- print_api(
355
- message, error_type=True, logger=recorder.logger, logger_method='critical',
356
- traceback_string=True, oneline=True)
357
- print_api(
358
- message, error_type=True, logger=network_logger, logger_method='critical',
359
- traceback_string=True, oneline=True)
360
- pass
361
-
362
- # Save statistics file.
363
- output_statistics_csv_row()
736
+ record_and_statistics_write(client_message)
737
+
738
+ finish_thread()
739
+
740
+ # Add custom attribute to the exception.
741
+ exc.engine_name = client_message.engine_name
742
+
743
+ # After the socket clean up, we will still raise the exception to the main thread.
744
+ raise exc
745
+
746
+ # ================================================================================================================
747
+ # This is the start of the thread_worker_main function
748
+ network_logger = loggingw.get_logger_with_level(config_static.MainConfig.LOGGER_NAME)
749
+
750
+ # Only protocols that are encrypted with TLS have the server name attribute.
751
+ if is_tls:
752
+ # Get current destination domain
753
+ server_name = client_socket.server_hostname
754
+ # If there is no server name from the TLS handshake, then we'll use the domain from the DNS.
755
+ if not server_name:
756
+ server_name = domain_from_dns
757
+ # If the protocol is not TLS, then we'll use the domain from the DNS.
758
+ else:
759
+ server_name = domain_from_dns
760
+
761
+ thread_id = threads.current_thread_id()
762
+
763
+ process_name: str = multiprocessing.current_process().name
764
+ current_thread = threading.current_thread()
765
+ thread_process_name: str = f"{process_name} | {current_thread.name}"
766
+ current_thread.name = thread_process_name
767
+
768
+ # This is the main protocols.
769
+ protocol: str = str()
770
+ # This is the secondary protocol in the websocket.
771
+ protocol3: str = str()
772
+ # # This is Client Masked Frame Parser.
773
+ # websocket_masked_frame_parser = websocket_parse.WebsocketFrameParser()
774
+ # # This is Server UnMasked Frame Parser.
775
+ # websocket_unmasked_frame_parser = websocket_parse.WebsocketFrameParser()
776
+ websocket_frame_parser = websocket_parse.WebsocketFrameParser()
777
+
778
+ # Loading parser by domain, if there is no parser for current domain - general reference parser is loaded.
779
+ # These should be outside any loop and initialized only once entering the thread.
780
+ found_domain_module = initialize_engines.assign_class_by_domain(
781
+ engines_list=engines_list,
782
+ message_domain_name=server_name,
783
+ reference_module=config_static.REFERENCE_MODULE
784
+ )
785
+ parser = found_domain_module.parser_class_object
786
+ requester = found_domain_module.requester_class_object()
787
+ responder = found_domain_module.responder_class_object()
788
+ recorder = found_domain_module.recorder_class_object(record_path=config_static.LogRec.recordings_path)
789
+
790
+ engine_name: str = recorder.engine_name
791
+
792
+ for engine in engines_list:
793
+ if engine.engine_name == engine_name:
794
+ responder.add_args(engine=engine)
795
+ break
796
+
797
+ network_logger.info(f"Assigned Modules for [{server_name}]: "
798
+ f"{parser.__name__}, "
799
+ f"{requester.__class__.__name__}, "
800
+ f"{responder.__class__.__name__}, "
801
+ f"{recorder.__class__.__name__}")
802
+
803
+ # Initializing the client message object with current thread's data.
804
+ # This is needed only to skip error alerts after 'try'.
805
+ client_message_connection: ClientMessage = ClientMessage()
806
+ # This is needed to indicate if there was an exception or socket was closed in any of the receiving thread.
807
+ exception_or_close_in_receiving_thread: bool = False
808
+ # Queue for http request URI paths.
809
+ http_path_queue: queue.Queue = queue.Queue()
364
810
 
365
- # At this stage there could be several times that the same socket was used to the service server - we need to
366
- # close this socket as well if it still opened.
367
- if service_client:
368
- if service_client.socket_instance:
369
- service_client.close_socket()
811
+ try:
812
+ client_ip, source_port = client_socket.getpeername()
370
813
 
371
- # If client socket is still opened - close
372
- if function_client_socket_object:
373
- function_client_socket_object.close()
374
- network_logger.info(f"Closed client socket [{client_message.client_ip}:{client_message.source_port}]...")
814
+ try:
815
+ client_name: str = socket.gethostbyaddr(client_ip)[0]
816
+ # This can happen if the host changed IP address, but it wasn't propagated over DHCP.
817
+ except socket.herror:
818
+ client_name = ""
819
+
820
+ client_name = client_name.lower()
821
+ destination_port: int = client_socket.getsockname()[1]
822
+ destination_port_str: str = str(destination_port)
823
+
824
+ # If the destination port is in the on_port_connect dictionary, then we'll get the port from there.
825
+ if destination_port_str in found_domain_module.on_port_connect:
826
+ on_port_connect_value = found_domain_module.on_port_connect[destination_port_str]
827
+ _, destination_port_str = initialize_engines.get_ipv4_from_engine_on_connect_port(on_port_connect_value)
828
+ destination_port: int = int(destination_port_str)
829
+
830
+ if config_static.MainConfig.is_offline:
831
+ # If in offline mode, then we'll get the TCP server's input address.
832
+ server_ip = client_socket.getsockname()[0]
833
+ else:
834
+ # If not in offline mode, we will get the ip from the socket that will connect later to the service.
835
+ server_ip = ""
836
+
837
+ network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
838
+ f"Destination service: [{server_name}:{destination_port}]")
839
+
840
+ origin_service_client_instance = None
841
+ client_receive_count: int = 0
842
+ server_receive_count: int = 0
843
+ client_message_connection = client_message_first_start()
844
+
845
+ # If we're not in offline mode, then we'll create the client socket to the service.
846
+ # noinspection PyTypeChecker
847
+ connection_error: str = None
848
+ service_socket_instance = None
849
+ client_message_connection.action = 'service_connect'
850
+ client_message_connection.timestamp = datetime.now()
851
+
852
+ if config_static.MainConfig.is_offline:
853
+ client_message_connection.info = 'Offline Mode'
854
+ else:
855
+ origin_service_client_instance = create_client_socket(client_message_connection)
856
+ service_socket_instance, connection_error = origin_service_client_instance.service_connection()
375
857
 
376
- network_logger.info("Thread Finished. Will continue listening on the Main thread")
377
- except Exception:
378
- message = "Undocumented exception in thread worker"
379
- print_api(
380
- message, error_type=True, logger=network_logger, logger_method='critical',
381
- traceback_string=True, oneline=True)
858
+ if connection_error:
859
+ client_message_connection.errors.append(connection_error)
860
+ record_and_statistics_write(client_message_connection)
861
+ else:
862
+ # Now we'll update the server IP with the IP of the service.
863
+ server_ip = service_socket_instance.getpeername()[0]
864
+ client_message_connection.server_ip = server_ip
865
+
866
+ if not connection_error:
867
+ client_exception_queue: queue.Queue = queue.Queue()
868
+ client_thread = threading.Thread(
869
+ target=receive_send_start,
870
+ args=(client_socket, service_socket_instance, client_exception_queue, None),
871
+ name=f"{process_name} | Thread-{thread_id}-Client",
872
+ daemon=True)
873
+ client_thread.start()
874
+
875
+ service_exception_queue: queue.Queue = queue.Queue()
876
+ if not config_static.MainConfig.is_offline:
877
+ service_thread = threading.Thread(
878
+ target=receive_send_start,
879
+ args=(service_socket_instance, client_socket, service_exception_queue, client_message_connection),
880
+ name=f"{process_name} | Thread-{thread_id}-Service",
881
+ daemon=True)
882
+ service_thread.start()
883
+
884
+ client_thread.join()
885
+ # If we're in offline mode, then there is no service thread.
886
+ if not config_static.MainConfig.is_offline:
887
+ # If we're not in offline mode, then we'll wait for the service thread to finish.
888
+ # noinspection PyUnboundLocalVariable
889
+ service_thread.join()
890
+
891
+ # If there was an exception in any of the threads, then we'll raise it here.
892
+ if not client_exception_queue.empty():
893
+ raise client_exception_queue.get()
894
+ if not service_exception_queue.empty():
895
+ raise service_exception_queue.get()
896
+
897
+ finish_thread()
898
+ except Exception as e:
899
+ handle_exceptions_on_main_connection_thread(e, client_message_connection)