atomicshop 2.15.11__py3-none-any.whl → 3.10.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- atomicshop/__init__.py +1 -1
- atomicshop/{addons/mains → a_mains}/FACT/update_extract.py +3 -2
- atomicshop/a_mains/dns_gateway_setting.py +11 -0
- atomicshop/a_mains/get_local_tcp_ports.py +85 -0
- atomicshop/a_mains/github_wrapper.py +11 -0
- atomicshop/a_mains/install_ca_certificate.py +172 -0
- atomicshop/a_mains/process_from_port.py +119 -0
- atomicshop/a_mains/set_default_dns_gateway.py +90 -0
- atomicshop/a_mains/update_config_toml.py +38 -0
- atomicshop/basics/ansi_escape_codes.py +3 -1
- atomicshop/basics/argparse_template.py +2 -0
- atomicshop/basics/booleans.py +27 -30
- atomicshop/basics/bytes_arrays.py +43 -0
- atomicshop/basics/classes.py +149 -1
- atomicshop/basics/enums.py +2 -2
- atomicshop/basics/exceptions.py +5 -1
- atomicshop/basics/list_of_classes.py +29 -0
- atomicshop/basics/multiprocesses.py +374 -50
- atomicshop/basics/strings.py +72 -3
- atomicshop/basics/threads.py +14 -0
- atomicshop/basics/tracebacks.py +13 -3
- atomicshop/certificates.py +153 -52
- atomicshop/config_init.py +11 -6
- atomicshop/console_user_response.py +7 -14
- atomicshop/consoles.py +9 -0
- atomicshop/datetimes.py +1 -1
- atomicshop/diff_check.py +3 -3
- atomicshop/dns.py +128 -3
- atomicshop/etws/_pywintrace_fix.py +17 -0
- atomicshop/etws/trace.py +40 -42
- atomicshop/etws/traces/trace_dns.py +56 -44
- atomicshop/etws/traces/trace_tcp.py +130 -0
- atomicshop/file_io/csvs.py +27 -5
- atomicshop/file_io/docxs.py +34 -17
- atomicshop/file_io/file_io.py +31 -17
- atomicshop/file_io/jsons.py +49 -0
- atomicshop/file_io/tomls.py +139 -0
- atomicshop/filesystem.py +616 -291
- atomicshop/get_process_list.py +3 -3
- atomicshop/http_parse.py +149 -93
- atomicshop/ip_addresses.py +6 -1
- atomicshop/mitm/centered_settings.py +132 -0
- atomicshop/mitm/config_static.py +207 -0
- atomicshop/mitm/config_toml_editor.py +55 -0
- atomicshop/mitm/connection_thread_worker.py +875 -357
- atomicshop/mitm/engines/__parent/parser___parent.py +4 -17
- atomicshop/mitm/engines/__parent/recorder___parent.py +108 -51
- atomicshop/mitm/engines/__parent/requester___parent.py +116 -0
- atomicshop/mitm/engines/__parent/responder___parent.py +75 -114
- atomicshop/mitm/engines/__reference_general/parser___reference_general.py +10 -7
- atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +5 -5
- atomicshop/mitm/engines/__reference_general/requester___reference_general.py +47 -0
- atomicshop/mitm/engines/__reference_general/responder___reference_general.py +95 -13
- atomicshop/mitm/engines/create_module_template.py +58 -14
- atomicshop/mitm/import_config.py +359 -139
- atomicshop/mitm/initialize_engines.py +160 -80
- atomicshop/mitm/message.py +64 -23
- atomicshop/mitm/mitm_main.py +892 -0
- atomicshop/mitm/recs_files.py +183 -0
- atomicshop/mitm/shared_functions.py +4 -10
- atomicshop/mitm/ssh_tester.py +82 -0
- atomicshop/mitm/statistic_analyzer.py +136 -40
- atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +265 -83
- atomicshop/monitor/checks/dns.py +1 -1
- atomicshop/networks.py +671 -0
- atomicshop/on_exit.py +39 -9
- atomicshop/package_mains_processor.py +84 -0
- atomicshop/permissions/permissions.py +22 -0
- atomicshop/permissions/ubuntu_permissions.py +239 -0
- atomicshop/permissions/win_permissions.py +33 -0
- atomicshop/print_api.py +24 -42
- atomicshop/process.py +24 -6
- atomicshop/process_poller/process_pool.py +0 -1
- atomicshop/process_poller/simple_process_pool.py +204 -5
- atomicshop/python_file_patcher.py +1 -1
- atomicshop/python_functions.py +27 -75
- atomicshop/speech_recognize.py +8 -0
- atomicshop/ssh_remote.py +158 -172
- atomicshop/system_resource_monitor.py +61 -47
- atomicshop/system_resources.py +8 -8
- atomicshop/tempfiles.py +1 -2
- atomicshop/urls.py +6 -0
- atomicshop/venvs.py +28 -0
- atomicshop/versioning.py +27 -0
- atomicshop/web.py +98 -27
- atomicshop/web_apis/google_custom_search.py +44 -0
- atomicshop/web_apis/google_llm.py +188 -0
- atomicshop/websocket_parse.py +450 -0
- atomicshop/wrappers/certauthw/certauth.py +1 -0
- atomicshop/wrappers/cryptographyw.py +29 -8
- atomicshop/wrappers/ctyping/etw_winapi/const.py +97 -47
- atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +178 -49
- atomicshop/wrappers/ctyping/file_details_winapi.py +67 -0
- atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
- atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +2 -2
- atomicshop/wrappers/ctyping/setup_device.py +466 -0
- atomicshop/wrappers/ctyping/win_console.py +39 -0
- atomicshop/wrappers/dockerw/dockerw.py +113 -2
- atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
- atomicshop/wrappers/elasticsearchw/elastic_infra.py +75 -0
- atomicshop/wrappers/elasticsearchw/elasticsearchw.py +2 -20
- atomicshop/wrappers/factw/get_file_data.py +12 -5
- atomicshop/wrappers/factw/install/install_after_restart.py +89 -5
- atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +20 -14
- atomicshop/wrappers/githubw.py +537 -54
- atomicshop/wrappers/loggingw/consts.py +1 -1
- atomicshop/wrappers/loggingw/filters.py +23 -0
- atomicshop/wrappers/loggingw/formatters.py +12 -0
- atomicshop/wrappers/loggingw/handlers.py +214 -107
- atomicshop/wrappers/loggingw/loggers.py +19 -0
- atomicshop/wrappers/loggingw/loggingw.py +860 -22
- atomicshop/wrappers/loggingw/reading.py +134 -112
- atomicshop/wrappers/mongodbw/mongo_infra.py +31 -0
- atomicshop/wrappers/mongodbw/mongodbw.py +1324 -36
- atomicshop/wrappers/netshw.py +271 -0
- atomicshop/wrappers/playwrightw/engine.py +34 -19
- atomicshop/wrappers/playwrightw/infra.py +5 -0
- atomicshop/wrappers/playwrightw/javascript.py +7 -3
- atomicshop/wrappers/playwrightw/keyboard.py +14 -0
- atomicshop/wrappers/playwrightw/scenarios.py +172 -5
- atomicshop/wrappers/playwrightw/waits.py +9 -7
- atomicshop/wrappers/powershell_networking.py +80 -0
- atomicshop/wrappers/psutilw/processes.py +37 -1
- atomicshop/wrappers/psutilw/psutil_networks.py +85 -0
- atomicshop/wrappers/pyopensslw.py +9 -2
- atomicshop/wrappers/pywin32w/cert_store.py +116 -0
- atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +3 -105
- atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +3 -57
- atomicshop/wrappers/pywin32w/wmis/msft_netipaddress.py +113 -0
- atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +259 -0
- atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +112 -0
- atomicshop/wrappers/pywin32w/wmis/wmi_helpers.py +236 -0
- atomicshop/wrappers/socketw/accepter.py +21 -7
- atomicshop/wrappers/socketw/certificator.py +216 -150
- atomicshop/wrappers/socketw/creator.py +190 -50
- atomicshop/wrappers/socketw/dns_server.py +491 -182
- atomicshop/wrappers/socketw/exception_wrapper.py +45 -52
- atomicshop/wrappers/socketw/process_getter.py +86 -0
- atomicshop/wrappers/socketw/receiver.py +144 -102
- atomicshop/wrappers/socketw/sender.py +65 -35
- atomicshop/wrappers/socketw/sni.py +334 -165
- atomicshop/wrappers/socketw/socket_base.py +134 -0
- atomicshop/wrappers/socketw/socket_client.py +137 -95
- atomicshop/wrappers/socketw/socket_server_tester.py +11 -7
- atomicshop/wrappers/socketw/socket_wrapper.py +717 -116
- atomicshop/wrappers/socketw/ssl_base.py +15 -14
- atomicshop/wrappers/socketw/statistics_csv.py +148 -17
- atomicshop/wrappers/sysmonw.py +1 -1
- atomicshop/wrappers/ubuntu_terminal.py +65 -26
- atomicshop/wrappers/win_auditw.py +189 -0
- atomicshop/wrappers/winregw/__init__.py +0 -0
- atomicshop/wrappers/winregw/winreg_installed_software.py +58 -0
- atomicshop/wrappers/winregw/winreg_network.py +232 -0
- {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/METADATA +31 -51
- atomicshop-3.10.5.dist-info/RECORD +306 -0
- {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/WHEEL +1 -1
- atomicshop/_basics_temp.py +0 -101
- atomicshop/a_installs/win/fibratus.py +0 -9
- atomicshop/a_installs/win/mongodb.py +0 -9
- atomicshop/a_installs/win/pycharm.py +0 -9
- atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
- atomicshop/addons/a_setup_scripts/install_pywintrace_0.3.cmd +0 -2
- atomicshop/addons/mains/__pycache__/install_fibratus_windows.cpython-312.pyc +0 -0
- atomicshop/addons/mains/__pycache__/msi_unpacker.cpython-312.pyc +0 -0
- atomicshop/addons/mains/install_docker_rootless_ubuntu.py +0 -11
- atomicshop/addons/mains/install_docker_ubuntu_main_sudo.py +0 -11
- atomicshop/addons/mains/install_elastic_search_and_kibana_ubuntu.py +0 -10
- atomicshop/addons/mains/install_wsl_ubuntu_lts_admin.py +0 -9
- atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
- atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
- atomicshop/addons/package_setup/Setup.cmd +0 -7
- atomicshop/archiver/_search_in_zip.py +0 -189
- atomicshop/archiver/archiver.py +0 -34
- atomicshop/archiver/search_in_archive.py +0 -250
- atomicshop/archiver/sevenz_app_w.py +0 -86
- atomicshop/archiver/sevenzs.py +0 -44
- atomicshop/archiver/zips.py +0 -293
- atomicshop/file_types.py +0 -24
- atomicshop/mitm/config_editor.py +0 -37
- atomicshop/mitm/engines/create_module_template_example.py +0 -13
- atomicshop/mitm/initialize_mitm_server.py +0 -268
- atomicshop/pbtkmultifile_argparse.py +0 -88
- atomicshop/permissions.py +0 -151
- atomicshop/script_as_string_processor.py +0 -38
- atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
- atomicshop/ssh_scripts/process_from_port.py +0 -27
- atomicshop/wrappers/_process_wrapper_curl.py +0 -27
- atomicshop/wrappers/_process_wrapper_tar.py +0 -21
- atomicshop/wrappers/dockerw/install_docker.py +0 -209
- atomicshop/wrappers/elasticsearchw/infrastructure.py +0 -265
- atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -232
- atomicshop/wrappers/ffmpegw.py +0 -125
- atomicshop/wrappers/fibratusw/install.py +0 -81
- atomicshop/wrappers/mongodbw/infrastructure.py +0 -53
- atomicshop/wrappers/mongodbw/install_mongodb.py +0 -190
- atomicshop/wrappers/msiw.py +0 -149
- atomicshop/wrappers/nodejsw/install_nodejs.py +0 -139
- atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
- atomicshop/wrappers/psutilw/networks.py +0 -45
- atomicshop/wrappers/pycharmw.py +0 -81
- atomicshop/wrappers/socketw/base.py +0 -59
- atomicshop/wrappers/socketw/get_process.py +0 -107
- atomicshop/wrappers/wslw.py +0 -191
- atomicshop-2.15.11.dist-info/RECORD +0 -302
- /atomicshop/{addons/mains → a_mains}/FACT/factw_fact_extractor_docker_image_main_sudo.py +0 -0
- /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compile.cmd +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.dll +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.exp +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/compiled/Win10x64/process_list.lib +0 -0
- /atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +0 -0
- /atomicshop/{archiver → permissions}/__init__.py +0 -0
- /atomicshop/{wrappers/fibratusw → web_apis}/__init__.py +0 -0
- /atomicshop/wrappers/{nodejsw → pywin32w/wmis}/__init__.py +0 -0
- /atomicshop/wrappers/pywin32w/{wmi_win32process.py → wmis/win32process.py} +0 -0
- {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info/licenses}/LICENSE.txt +0 -0
- {atomicshop-2.15.11.dist-info → atomicshop-3.10.5.dist-info}/top_level.txt +0 -0
|
@@ -1,381 +1,899 @@
|
|
|
1
|
+
import multiprocessing
|
|
1
2
|
from datetime import datetime
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
from
|
|
7
|
-
|
|
3
|
+
import threading
|
|
4
|
+
import queue
|
|
5
|
+
import socket
|
|
6
|
+
import ssl
|
|
7
|
+
from typing import Literal
|
|
8
|
+
import struct
|
|
9
|
+
|
|
10
|
+
from ..wrappers.socketw import receiver, sender, socket_client, socket_base
|
|
11
|
+
from .. import websocket_parse, ip_addresses
|
|
8
12
|
from ..http_parse import HTTPRequestParse, HTTPResponseParse
|
|
9
|
-
from ..basics
|
|
13
|
+
from ..basics import threads, tracebacks
|
|
10
14
|
from ..print_api import print_api
|
|
11
15
|
|
|
16
|
+
from .message import ClientMessage
|
|
17
|
+
from . import initialize_engines
|
|
18
|
+
from ..wrappers.loggingw import loggingw
|
|
19
|
+
# This is needed only for the data typing.
|
|
20
|
+
from . import config_static as cf
|
|
21
|
+
|
|
12
22
|
|
|
13
|
-
# Thread function on client connect.
|
|
14
23
|
def thread_worker_main(
|
|
15
|
-
|
|
24
|
+
# These parameters come from the SocketWrapper.
|
|
25
|
+
client_socket,
|
|
16
26
|
process_commandline: str,
|
|
17
27
|
is_tls: bool,
|
|
28
|
+
tls_type: str,
|
|
29
|
+
tls_version: str,
|
|
18
30
|
domain_from_dns,
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
except Exception:
|
|
29
|
-
pass
|
|
30
|
-
try:
|
|
31
|
-
statistics_dict['status_code'] = \
|
|
32
|
-
','.join([str(x.code) for x in client_message.response_list_of_raw_decoded])
|
|
33
|
-
except Exception:
|
|
34
|
-
pass
|
|
35
|
-
try:
|
|
36
|
-
statistics_dict['command'] = client_message.request_raw_decoded.command
|
|
37
|
-
except Exception:
|
|
38
|
-
pass
|
|
39
|
-
try:
|
|
40
|
-
statistics_dict['request_time_sent'] = client_message.request_time_received
|
|
41
|
-
except Exception:
|
|
42
|
-
pass
|
|
31
|
+
statistics_writer,
|
|
32
|
+
engines_list: list[initialize_engines.ModuleCategory],
|
|
33
|
+
|
|
34
|
+
# These parameters come from the main mitm module.
|
|
35
|
+
config_static: cf
|
|
36
|
+
):
|
|
37
|
+
def output_statistics_csv_row(client_message: ClientMessage):
|
|
38
|
+
# If there is no '.code' attribute in HTTPResponse, this means that this is not an HTTP message, so there is no
|
|
39
|
+
# status code.
|
|
43
40
|
try:
|
|
44
|
-
|
|
45
|
-
except
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
','.join([str(len(x)) for x in client_message.response_list_of_raw_bytes])
|
|
50
|
-
except Exception:
|
|
51
|
-
pass
|
|
52
|
-
# try:
|
|
53
|
-
# statistics_dict['request_hex'] = client_message.request_raw_hex
|
|
54
|
-
# except Exception:
|
|
55
|
-
# pass
|
|
56
|
-
# try:
|
|
57
|
-
# statistics_dict['response_hex'] = \
|
|
58
|
-
# f'"' + ','.join([x for x in client_message.response_list_of_raw_hex]) + '"'
|
|
59
|
-
# except Exception:
|
|
60
|
-
# pass
|
|
41
|
+
http_status_code: str = str(client_message.response_auto_parsed.code)
|
|
42
|
+
except AttributeError:
|
|
43
|
+
http_status_code: str = str()
|
|
44
|
+
|
|
45
|
+
# Same goes for the '.path' attribute, if it is not HTTP message then there will be no path.
|
|
61
46
|
try:
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
47
|
+
if client_message.request_auto_parsed and client_message.request_auto_parsed.path:
|
|
48
|
+
http_path: str = client_message.request_auto_parsed.path
|
|
49
|
+
elif client_message.response_auto_parsed.path:
|
|
50
|
+
http_path: str = client_message.response_auto_parsed.path
|
|
51
|
+
else:
|
|
52
|
+
http_path: str = str()
|
|
53
|
+
except AttributeError:
|
|
54
|
+
http_path: str = str()
|
|
55
|
+
|
|
56
|
+
# Same goes for the '.command' attribute, if it is not HTTP message then there will be no command.
|
|
65
57
|
try:
|
|
66
|
-
|
|
67
|
-
except
|
|
68
|
-
|
|
69
|
-
statistics_dict['error'] = str()
|
|
70
|
-
|
|
71
|
-
statistics_logger.info(f"{statistics_dict['request_time_sent']},"
|
|
72
|
-
f"{statistics_dict['host']},"
|
|
73
|
-
f"\"{statistics_dict['path']}\","
|
|
74
|
-
f"{statistics_dict['command']},"
|
|
75
|
-
f"{statistics_dict['status_code']},"
|
|
76
|
-
f"{statistics_dict['request_size_bytes']},"
|
|
77
|
-
f"{statistics_dict['response_size_bytes']},"
|
|
78
|
-
# f"{statistics_dict['request_hex']},"
|
|
79
|
-
# f"{statistics_dict['response_hex']},"
|
|
80
|
-
f"\"{statistics_dict['file_path']}\","
|
|
81
|
-
f"\"{statistics_dict['process_cmd']}\","
|
|
82
|
-
f"{statistics_dict['error']}"
|
|
83
|
-
)
|
|
58
|
+
http_command: str = client_message.request_auto_parsed.command
|
|
59
|
+
except AttributeError:
|
|
60
|
+
http_command: str = str()
|
|
84
61
|
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
client_message.
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
62
|
+
if client_message.request_raw_bytes is None:
|
|
63
|
+
request_size_bytes = ''
|
|
64
|
+
else:
|
|
65
|
+
request_size_bytes = str(len(client_message.request_raw_bytes))
|
|
66
|
+
|
|
67
|
+
if client_message.response_raw_bytes is None:
|
|
68
|
+
response_size_bytes = ''
|
|
69
|
+
else:
|
|
70
|
+
response_size_bytes = str(len(client_message.response_raw_bytes))
|
|
71
|
+
|
|
72
|
+
if client_message.errors and len(client_message.errors) > 1:
|
|
73
|
+
error_string = '||'.join(client_message.errors)
|
|
74
|
+
error_string = f'Error count: {len(client_message.errors)} | Errors: {error_string}'
|
|
75
|
+
elif client_message.errors and len(client_message.errors) == 1:
|
|
76
|
+
error_string = client_message.errors[0]
|
|
77
|
+
elif not client_message.errors:
|
|
78
|
+
error_string = str()
|
|
79
|
+
else:
|
|
80
|
+
raise ValueError(f"Error in statistics error list. Values: {client_message.errors}")
|
|
81
|
+
|
|
82
|
+
statistics_writer.write_row(
|
|
83
|
+
thread_id=str(thread_id),
|
|
84
|
+
engine=client_message.engine_name,
|
|
85
|
+
source_host=client_message.client_name,
|
|
86
|
+
source_ip=client_message.client_ip,
|
|
87
|
+
tls_type=tls_type,
|
|
88
|
+
tls_version=tls_version,
|
|
89
|
+
protocol=client_message.protocol,
|
|
90
|
+
protocol2=client_message.protocol2,
|
|
91
|
+
protocol3=client_message.protocol3,
|
|
92
|
+
dest_port=client_message.destination_port,
|
|
93
|
+
host=client_message.server_name,
|
|
94
|
+
path=http_path,
|
|
95
|
+
status_code=http_status_code,
|
|
96
|
+
command=http_command,
|
|
97
|
+
timestamp=client_message.timestamp,
|
|
98
|
+
request_size_bytes=request_size_bytes,
|
|
99
|
+
response_size_bytes=response_size_bytes,
|
|
100
|
+
recorded_file_path=client_message.recorded_file_path,
|
|
101
|
+
process_cmd=process_commandline,
|
|
102
|
+
action=client_message.action,
|
|
103
|
+
error=error_string
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
def record_and_statistics_write(client_message: ClientMessage):
|
|
107
|
+
# If recorder wasn't executed before, then execute it now
|
|
108
|
+
if config_static.LogRec.enable_request_response_recordings_in_logs:
|
|
109
|
+
recorded_file = recorder.record(class_client_message=client_message)
|
|
110
|
+
client_message.recorded_file_path = recorded_file
|
|
111
|
+
|
|
112
|
+
# Save statistics file.
|
|
113
|
+
output_statistics_csv_row(client_message)
|
|
114
|
+
|
|
115
|
+
def parse_http(
|
|
116
|
+
raw_bytes: bytes,
|
|
117
|
+
client_message: ClientMessage):
|
|
118
|
+
nonlocal protocol
|
|
119
|
+
|
|
120
|
+
# Parsing the raw bytes as HTTP.
|
|
121
|
+
request_http_parsed, is_http_request, request_parsing_error = (
|
|
122
|
+
HTTPRequestParse(raw_bytes).parse())
|
|
123
|
+
|
|
124
|
+
response_http_parsed, is_http_response, response_parsing_error = (
|
|
125
|
+
HTTPResponseParse(raw_bytes).parse())
|
|
126
|
+
|
|
127
|
+
if is_http_request:
|
|
128
|
+
if protocol == '':
|
|
129
|
+
protocol = 'HTTP'
|
|
130
|
+
|
|
131
|
+
auto_parsed = request_http_parsed
|
|
132
|
+
network_logger.info(
|
|
133
|
+
f"HTTP Request Parsed: Method: {request_http_parsed.command} | Path: {request_http_parsed.path}")
|
|
134
|
+
http_path_queue.put(request_http_parsed.path)
|
|
135
|
+
network_logger.info(f"HTTP Request Parsed: Putting PATH to queue.")
|
|
136
|
+
|
|
137
|
+
is_http_request_a_websocket(auto_parsed, client_message)
|
|
138
|
+
elif is_http_response:
|
|
139
|
+
auto_parsed = response_http_parsed
|
|
140
|
+
network_logger.info(
|
|
141
|
+
f"HTTP Response Parsed: Status: {response_http_parsed.code}")
|
|
142
|
+
|
|
143
|
+
auto_parsed.path = http_path_queue.get()
|
|
144
|
+
network_logger.info(f"HTTP Response Parsed: Got PATH from queue: [{auto_parsed.path}]")
|
|
145
|
+
elif protocol == 'Websocket':
|
|
146
|
+
client_message.protocol2 = 'Frame'
|
|
147
|
+
auto_parsed = parse_websocket(raw_bytes)
|
|
148
|
+
if protocol3:
|
|
149
|
+
client_message.protocol3 = protocol3
|
|
150
|
+
else:
|
|
151
|
+
auto_parsed = None
|
|
152
|
+
|
|
153
|
+
return auto_parsed
|
|
154
|
+
|
|
155
|
+
def is_http_request_a_websocket(
|
|
156
|
+
auto_parsed,
|
|
157
|
+
client_message: ClientMessage):
|
|
158
|
+
nonlocal protocol
|
|
159
|
+
nonlocal protocol3
|
|
160
|
+
|
|
161
|
+
if protocol == 'HTTP':
|
|
162
|
+
if auto_parsed and hasattr(auto_parsed, 'headers') and 'Upgrade' in auto_parsed.headers:
|
|
163
|
+
if auto_parsed.headers['Upgrade'] == 'websocket':
|
|
164
|
+
protocol = 'Websocket'
|
|
165
|
+
client_message.protocol2 = 'Handshake'
|
|
166
|
+
protocol3 = auto_parsed.headers.get('Sec-WebSocket-Protocol', None)
|
|
167
|
+
if protocol3:
|
|
168
|
+
client_message.protocol3 = protocol3
|
|
169
|
+
|
|
170
|
+
network_logger.info(f'Protocol upgraded to Websocket')
|
|
171
|
+
|
|
172
|
+
def parse_websocket(raw_bytes):
|
|
173
|
+
return websocket_frame_parser.parse_frame_bytes(raw_bytes)
|
|
174
|
+
|
|
175
|
+
def finish_thread(send_connection_reset: bool = False):
|
|
176
|
+
"""
|
|
177
|
+
Finishing the thread, closing sockets.
|
|
178
|
+
|
|
179
|
+
:param send_connection_reset: Whether to send TCP RST flag to the client when closing the socket.
|
|
180
|
+
Basically what happens is that the server socket is closed abruptly sending us any Connection*Error exception.
|
|
181
|
+
We can simulate the ConnectionResetError on the client side by sending TCP RST flag when closing the socket.
|
|
182
|
+
But not the ConnectionAbortedError, since it is caused by other reasons (local TCP stack and has nothing
|
|
183
|
+
to do with the remote server).
|
|
184
|
+
"""
|
|
185
|
+
# At this stage there could be several times that the same socket was used to the service server - we need to
|
|
186
|
+
# close this socket as well if it still opened.
|
|
187
|
+
# The first part of the condition is to check if the service socket was connected at all.
|
|
188
|
+
# If the service socket couldn't connect, then the instance will be None.
|
|
189
|
+
if service_socket_instance and service_socket_instance.fileno() != -1:
|
|
190
|
+
if origin_service_client_instance.socket_instance:
|
|
191
|
+
origin_service_client_instance.close_socket()
|
|
192
|
+
|
|
193
|
+
# If client socket is still opened - close
|
|
194
|
+
if client_socket.fileno() != -1:
|
|
195
|
+
if send_connection_reset:
|
|
196
|
+
# Abort the connection
|
|
197
|
+
linger = struct.pack('ii', 1, 0)
|
|
198
|
+
client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, linger)
|
|
199
|
+
client_socket.close()
|
|
200
|
+
|
|
201
|
+
if send_connection_reset:
|
|
202
|
+
network_logger.info(f"Closed client socket [{client_ip}:{source_port}] with TCP RST flag (Sent ConnectionResetError)...")
|
|
203
|
+
else:
|
|
204
|
+
network_logger.info(f"Closed client socket [{client_ip}:{source_port}]...")
|
|
205
|
+
|
|
206
|
+
network_logger.info("Thread Finished. Will continue listening on the Main thread")
|
|
207
|
+
|
|
208
|
+
def create_requester_request(
|
|
209
|
+
client_message: ClientMessage,
|
|
210
|
+
sending_socket: socket.socket
|
|
211
|
+
) -> tuple[bytes, bool]:
|
|
212
|
+
request_received_raw: bytes = client_message.request_raw_bytes
|
|
213
|
+
request_custom_raw: bytes = requester.create_request(client_message, sending_socket=sending_socket)
|
|
100
214
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
# Get current destination domain
|
|
104
|
-
client_message.server_name = function_client_socket_object.server_hostname
|
|
105
|
-
# client_message.server_name = domain_from_dns
|
|
106
|
-
# If the protocol is not TLS, then we'll use the domain from the DNS.
|
|
215
|
+
if request_custom_raw is None or request_received_raw == request_custom_raw:
|
|
216
|
+
is_requester_worked: bool = False
|
|
107
217
|
else:
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
statistics_dict: dict = dict()
|
|
132
|
-
statistics_dict['host'] = client_message.server_name
|
|
133
|
-
statistics_dict['path'] = str()
|
|
134
|
-
statistics_dict['status_code'] = str()
|
|
135
|
-
statistics_dict['command'] = str()
|
|
136
|
-
statistics_dict['request_time_sent'] = str()
|
|
137
|
-
statistics_dict['request_size_bytes'] = str()
|
|
138
|
-
# statistics_dict['response_time_sent'] = str()
|
|
139
|
-
statistics_dict['response_size_bytes'] = str()
|
|
140
|
-
statistics_dict['file_path'] = str()
|
|
141
|
-
statistics_dict['process_cmd'] = str()
|
|
142
|
-
statistics_dict['error'] = str()
|
|
143
|
-
|
|
144
|
-
network_logger.info("Initializing Receiver")
|
|
145
|
-
# Getting message from the client over the socket using specific class.
|
|
146
|
-
client_received_raw_data = Receiver(function_client_socket_object).receive()
|
|
147
|
-
|
|
148
|
-
# If the message is empty, then the connection was closed already by the other side,
|
|
149
|
-
# so we can close the socket as well.
|
|
150
|
-
# If the received message from the client is not empty, then continue.
|
|
151
|
-
if client_received_raw_data:
|
|
152
|
-
# Putting the received message to the aggregating message class.
|
|
153
|
-
client_message.request_raw_bytes = client_received_raw_data
|
|
154
|
-
# Getting current time of message received from client.
|
|
155
|
-
client_message.request_time_received = datetime.now()
|
|
156
|
-
|
|
157
|
-
# HTTP Parsing section =================================================================================
|
|
158
|
-
# Parsing the raw bytes as HTTP.
|
|
159
|
-
try:
|
|
160
|
-
request_decoded = HTTPRequestParse(client_message.request_raw_bytes)
|
|
161
|
-
except Exception:
|
|
162
|
-
message = "There was an exception in HTTP Parsing module!"
|
|
163
|
-
print_api(
|
|
164
|
-
message, error_type=True, logger=network_logger, logger_method='critical',
|
|
165
|
-
traceback_string=True, oneline=True)
|
|
166
|
-
# Socket connection can be closed since we have a problem in current thread and break the loop
|
|
167
|
-
client_connection_boolean = False
|
|
168
|
-
break
|
|
169
|
-
|
|
170
|
-
# Getting the status of http parsing
|
|
171
|
-
request_is_http, http_parsing_reason, http_parsing_error = request_decoded.check_if_http()
|
|
172
|
-
|
|
173
|
-
# Currently, we don't care if it's HTTP. If there was no error we can continue. Just log the reason.
|
|
174
|
-
if not http_parsing_error:
|
|
175
|
-
network_logger.info(http_parsing_reason)
|
|
176
|
-
# If there was error - the request is really HTTP, but there's a problem with its structure.
|
|
177
|
-
# So, we'll stop the loop.
|
|
218
|
+
is_requester_worked: bool = True
|
|
219
|
+
|
|
220
|
+
# Output first 100 characters of the request.
|
|
221
|
+
requester.logger.info(f"{request_custom_raw[0: 100]}...")
|
|
222
|
+
|
|
223
|
+
return request_custom_raw, is_requester_worked
|
|
224
|
+
|
|
225
|
+
def create_responder_response(client_message: ClientMessage) -> list[bytes]:
|
|
226
|
+
if client_message.action == 'service_connect':
|
|
227
|
+
return responder.create_connect_response(client_message)
|
|
228
|
+
else:
|
|
229
|
+
# If we're in offline mode, and it's the first cycle and the protocol is Websocket, then we'll create the HTTP Handshake
|
|
230
|
+
# response automatically.
|
|
231
|
+
if config_static.MainConfig.is_offline and protocol == 'Websocket' and client_receive_count == 1:
|
|
232
|
+
responses: list = list()
|
|
233
|
+
responses.append(
|
|
234
|
+
websocket_parse.create_byte_http_response(client_message.request_raw_bytes))
|
|
235
|
+
responder.logger.info(f"Generated automatic WebSocket response in Offline Mode.")
|
|
236
|
+
else:
|
|
237
|
+
# Creating response for parsed message and printing
|
|
238
|
+
responder_responses: list = responder.create_response(client_message)
|
|
239
|
+
if responder_responses is None:
|
|
240
|
+
responses: list = [client_message.response_raw_bytes]
|
|
178
241
|
else:
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
242
|
+
responses: list = responder_responses
|
|
243
|
+
responder.logger.info(f"Generated {len(responses)} responses from responder.")
|
|
244
|
+
|
|
245
|
+
# Output first 100 characters of all the responses in the list.
|
|
246
|
+
for response_raw_bytes_single in responses:
|
|
247
|
+
responder.logger.info(f"{response_raw_bytes_single[0: 100]}...")
|
|
248
|
+
|
|
249
|
+
return responses
|
|
250
|
+
|
|
251
|
+
def create_client_socket(client_message: ClientMessage):
|
|
252
|
+
# If there is a custom certificate for the client for this domain, then we'll use it.
|
|
253
|
+
# noinspection PyTypeChecker
|
|
254
|
+
custom_client_pem_certificate_path: str = None
|
|
255
|
+
for subdomain, pem_file_path in found_domain_module.mtls.items():
|
|
256
|
+
if subdomain == client_message.server_name:
|
|
257
|
+
custom_client_pem_certificate_path = pem_file_path
|
|
258
|
+
break
|
|
259
|
+
|
|
260
|
+
# Check if the destination service is an ip address or a domain name.
|
|
261
|
+
if ip_addresses.is_ip_address(client_message.server_name, ip_type='ipv4'):
|
|
262
|
+
# If it's an ip address, connect to the ip address directly.
|
|
263
|
+
service_client_instance = socket_client.SocketClient(
|
|
264
|
+
service_name=client_message.server_name,
|
|
265
|
+
connection_ip=client_message.server_name,
|
|
266
|
+
service_port=client_message.destination_port,
|
|
267
|
+
tls=is_tls,
|
|
268
|
+
logger=network_logger,
|
|
269
|
+
custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
|
|
270
|
+
enable_sslkeylogfile_env_to_client_ssl_context=(
|
|
271
|
+
config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
|
|
272
|
+
sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
|
|
273
|
+
)
|
|
274
|
+
# If it's a domain name, then we'll use the DNS to resolve it.
|
|
275
|
+
else:
|
|
276
|
+
# If we're on localhost, then use external services list in order to resolve the domain:
|
|
277
|
+
# config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost']
|
|
278
|
+
if client_message.client_ip in socket_base.THIS_DEVICE_IP_LIST:
|
|
279
|
+
service_client_instance = socket_client.SocketClient(
|
|
280
|
+
service_name=client_message.server_name,
|
|
281
|
+
service_port=client_message.destination_port,
|
|
282
|
+
tls=is_tls,
|
|
283
|
+
dns_servers_list=[config_static.DNSServer.forwarding_dns_service_ipv4],
|
|
284
|
+
logger=network_logger,
|
|
285
|
+
custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
|
|
286
|
+
enable_sslkeylogfile_env_to_client_ssl_context=(
|
|
287
|
+
config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
|
|
288
|
+
sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
|
|
289
|
+
)
|
|
290
|
+
# If we're not on localhost, then connect to domain directly.
|
|
291
|
+
else:
|
|
292
|
+
service_client_instance = socket_client.SocketClient(
|
|
293
|
+
service_name=client_message.server_name,
|
|
294
|
+
service_port=client_message.destination_port,
|
|
295
|
+
tls=is_tls,
|
|
296
|
+
logger=network_logger,
|
|
297
|
+
custom_pem_client_certificate_file_path=custom_client_pem_certificate_path,
|
|
298
|
+
enable_sslkeylogfile_env_to_client_ssl_context=(
|
|
299
|
+
config_static.Certificates.enable_sslkeylogfile_env_to_client_ssl_context),
|
|
300
|
+
sslkeylog_file_path=config_static.Certificates.sslkeylog_file_path
|
|
301
|
+
)
|
|
302
|
+
|
|
303
|
+
return service_client_instance
|
|
304
|
+
|
|
305
|
+
def process_client_raw_data(
|
|
306
|
+
client_received_raw_data: bytes,
|
|
307
|
+
error_string: str,
|
|
308
|
+
client_message: ClientMessage):
|
|
309
|
+
"""
|
|
310
|
+
Process the client raw data request.
|
|
311
|
+
"""
|
|
312
|
+
nonlocal protocol
|
|
313
|
+
|
|
314
|
+
client_message.request_raw_bytes = client_received_raw_data
|
|
315
|
+
|
|
316
|
+
if error_string:
|
|
317
|
+
client_message.errors.append(error_string)
|
|
318
|
+
|
|
319
|
+
if client_received_raw_data == b'' or client_received_raw_data is None:
|
|
320
|
+
return
|
|
321
|
+
|
|
322
|
+
client_message.request_auto_parsed = parse_http(client_message.request_raw_bytes, client_message)
|
|
323
|
+
# This is needed for each cycle that is not HTTP, but its protocol maybe set by HTTP, like websocket.
|
|
324
|
+
if protocol != '':
|
|
325
|
+
client_message.protocol = protocol
|
|
326
|
+
|
|
327
|
+
# Parse websocket frames only if it is not the first protocol upgrade request.
|
|
328
|
+
if protocol == 'Websocket' and client_receive_count != 1:
|
|
329
|
+
client_message.request_auto_parsed = parse_websocket(client_message.request_raw_bytes)
|
|
330
|
+
|
|
331
|
+
# Custom parser, should parse HTTP body or the whole message if not HTTP.
|
|
332
|
+
parser_instance = parser(client_message)
|
|
333
|
+
parser_instance.parse()
|
|
334
|
+
|
|
335
|
+
# Converting body parsed to string on logging, there is no strict rule for the parameter to be string.
|
|
336
|
+
parser_instance.logger.info(f"{str(client_message.request_custom_parsed)[0: 100]}...")
|
|
337
|
+
|
|
338
|
+
def process_server_raw_data(
|
|
339
|
+
service_received_raw_data: bytes,
|
|
340
|
+
error_string: str,
|
|
341
|
+
client_message: ClientMessage
|
|
342
|
+
):
|
|
343
|
+
nonlocal protocol
|
|
344
|
+
|
|
345
|
+
client_message.response_raw_bytes = service_received_raw_data
|
|
346
|
+
|
|
347
|
+
if error_string:
|
|
348
|
+
client_message.errors.append(error_string)
|
|
349
|
+
|
|
350
|
+
if service_received_raw_data == b'' or service_received_raw_data is None:
|
|
351
|
+
return
|
|
352
|
+
|
|
353
|
+
client_message.response_auto_parsed = parse_http(client_message.response_raw_bytes, client_message)
|
|
354
|
+
if protocol != '':
|
|
355
|
+
client_message.protocol = protocol
|
|
356
|
+
|
|
357
|
+
def client_message_first_start() -> ClientMessage:
|
|
358
|
+
client_message: ClientMessage = ClientMessage()
|
|
359
|
+
client_message.client_name = client_name
|
|
360
|
+
client_message.client_ip = client_ip
|
|
361
|
+
client_message.server_ip = server_ip
|
|
362
|
+
client_message.source_port = source_port
|
|
363
|
+
client_message.destination_port = destination_port
|
|
364
|
+
client_message.server_name = server_name
|
|
365
|
+
client_message.thread_id = thread_id
|
|
366
|
+
client_message.thread_process = thread_process_name
|
|
367
|
+
client_message.process_name = process_commandline
|
|
368
|
+
client_message.engine_name = engine_name
|
|
369
|
+
|
|
370
|
+
return client_message
|
|
371
|
+
|
|
372
|
+
def receive_send_service_connect(
|
|
373
|
+
client_connection_message: ClientMessage,
|
|
374
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
375
|
+
) -> Literal['continue', 'return'] | None:
|
|
376
|
+
|
|
377
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
378
|
+
|
|
379
|
+
client_message = client_connection_message
|
|
380
|
+
|
|
381
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
382
|
+
print_api(f"Got responses from connect responder, count: [{len(bytes_to_send_list)}]",
|
|
383
|
+
logger=network_logger, logger_method='info')
|
|
384
|
+
|
|
385
|
+
# If the client message is the connection message, then we'll skip to the next iteration.
|
|
386
|
+
if not bytes_to_send_list:
|
|
387
|
+
return 'continue'
|
|
388
|
+
|
|
389
|
+
# is_socket_closed: bool = False
|
|
390
|
+
error_on_send: str = str()
|
|
391
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
392
|
+
client_message.reinitialize_dynamic_vars()
|
|
393
|
+
client_message.timestamp = datetime.now()
|
|
394
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
395
|
+
client_message.action = 'service_responder'
|
|
396
|
+
record_and_statistics_write(client_message)
|
|
397
|
+
|
|
398
|
+
# Send the bytes back to the client socket.
|
|
399
|
+
error_on_send: str = sender.Sender(
|
|
400
|
+
ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
|
|
401
|
+
logger=network_logger).send()
|
|
402
|
+
|
|
403
|
+
if error_on_send:
|
|
404
|
+
client_message.reinitialize_dynamic_vars()
|
|
405
|
+
client_message.errors.append(error_on_send)
|
|
406
|
+
client_message.timestamp = datetime.now()
|
|
407
|
+
client_message.action = 'client_send'
|
|
408
|
+
record_and_statistics_write(client_message)
|
|
409
|
+
|
|
410
|
+
if error_on_send:
|
|
411
|
+
exception_or_close_in_receiving_thread = True
|
|
412
|
+
finish_thread()
|
|
413
|
+
return 'return'
|
|
414
|
+
|
|
415
|
+
return None
|
|
416
|
+
|
|
417
|
+
def receive_send_client_offline(
|
|
418
|
+
client_message: ClientMessage,
|
|
419
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
420
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
421
|
+
) -> Literal['return'] | None:
|
|
422
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
423
|
+
nonlocal client_receive_count
|
|
424
|
+
|
|
425
|
+
client_receive_count += 1
|
|
426
|
+
|
|
427
|
+
network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
|
|
428
|
+
client_message.timestamp = datetime.now()
|
|
429
|
+
|
|
430
|
+
received_raw_data, is_socket_closed, error_message = receiver.Receiver(
|
|
431
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
432
|
+
|
|
433
|
+
process_client_raw_data(received_raw_data, error_message, client_message)
|
|
434
|
+
client_message.action = 'client_receive'
|
|
435
|
+
record_and_statistics_write(client_message)
|
|
436
|
+
if error_message:
|
|
437
|
+
print_api(error_message, logger=network_logger, logger_method='critical')
|
|
438
|
+
|
|
439
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
440
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
441
|
+
if (received_raw_data == b'' or error_message) and exception_or_close_in_receiving_thread:
|
|
442
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
443
|
+
logger_method='info')
|
|
444
|
+
return 'return'
|
|
445
|
+
|
|
446
|
+
# If the socket was closed on receive, and we're in offline mode, then we'll finish the thread right away.
|
|
447
|
+
# Since nothing more can be done, like responding to service or using requester.
|
|
448
|
+
if is_socket_closed:
|
|
449
|
+
exception_or_close_in_receiving_thread = True
|
|
450
|
+
finish_thread()
|
|
451
|
+
return 'return'
|
|
452
|
+
|
|
453
|
+
# Send to requester.
|
|
454
|
+
# THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
|
|
455
|
+
request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
|
|
456
|
+
# We will not process the raw data if requester didn't change anything.
|
|
457
|
+
if is_requester_worked:
|
|
458
|
+
client_message.reinitialize_dynamic_vars()
|
|
459
|
+
client_message.timestamp = datetime.now()
|
|
460
|
+
client_message.request_raw_bytes = request_custom_raw
|
|
461
|
+
client_message.action = 'client_requester'
|
|
462
|
+
process_client_raw_data(request_custom_raw, error_message, client_message)
|
|
463
|
+
record_and_statistics_write(client_message)
|
|
464
|
+
|
|
465
|
+
print_api("Offline Mode, sending to responder directly.", logger=network_logger,
|
|
466
|
+
logger_method='info')
|
|
467
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
468
|
+
|
|
469
|
+
error_on_send: str = str()
|
|
470
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
471
|
+
client_message.reinitialize_dynamic_vars()
|
|
472
|
+
client_message.timestamp = datetime.now()
|
|
473
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
474
|
+
client_message.action = 'client_responder_offline'
|
|
475
|
+
process_server_raw_data(bytes_to_send_single, '', client_message)
|
|
476
|
+
record_and_statistics_write(client_message)
|
|
477
|
+
|
|
478
|
+
error_on_send: str = sender.Sender(
|
|
479
|
+
ssl_socket=receiving_socket, bytes_to_send=bytes_to_send_single,
|
|
480
|
+
logger=network_logger).send()
|
|
481
|
+
|
|
482
|
+
if error_on_send:
|
|
483
|
+
client_message.reinitialize_dynamic_vars()
|
|
484
|
+
client_message.errors.append(error_on_send)
|
|
485
|
+
client_message.timestamp = datetime.now()
|
|
486
|
+
client_message.action = 'service_send'
|
|
487
|
+
|
|
488
|
+
record_and_statistics_write(client_message)
|
|
489
|
+
|
|
490
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
491
|
+
if is_socket_closed or error_on_send:
|
|
492
|
+
exception_or_close_in_receiving_thread = True
|
|
493
|
+
finish_thread()
|
|
494
|
+
return 'return'
|
|
495
|
+
|
|
496
|
+
return None
|
|
497
|
+
|
|
498
|
+
def receive_send_client(
|
|
499
|
+
client_message: ClientMessage,
|
|
500
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
501
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
502
|
+
) -> Literal['return'] | None:
|
|
503
|
+
|
|
504
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
505
|
+
nonlocal client_receive_count
|
|
506
|
+
|
|
507
|
+
client_receive_count += 1
|
|
508
|
+
|
|
509
|
+
network_logger.info(f"Initializing Receiver for Client cycle: {str(client_receive_count)}")
|
|
510
|
+
|
|
511
|
+
# Getting message from the client over the socket using specific class.
|
|
512
|
+
client_message.timestamp = datetime.now()
|
|
513
|
+
received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
|
|
514
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
515
|
+
|
|
516
|
+
process_client_raw_data(received_raw_data, error_on_receive, client_message)
|
|
517
|
+
client_message.action = 'client_receive'
|
|
518
|
+
|
|
519
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
520
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
521
|
+
if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
|
|
522
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
523
|
+
logger_method='info')
|
|
524
|
+
return 'return'
|
|
525
|
+
|
|
526
|
+
# We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
|
|
527
|
+
# Meaning 'exception_or_close_in_receiving_thread=True'.
|
|
528
|
+
record_and_statistics_write(client_message)
|
|
529
|
+
if error_on_receive:
|
|
530
|
+
print_api(error_on_receive, logger=network_logger, logger_method='critical')
|
|
531
|
+
|
|
532
|
+
# At this point if the socket was closed on receive, then there's no point to send anything to the service.
|
|
533
|
+
# But if the data was received and then the socket was closed, we first send the data and then close the socket.
|
|
534
|
+
error_on_send: str = str()
|
|
535
|
+
if received_raw_data != b'' and received_raw_data is not None:
|
|
536
|
+
# Send to requester.
|
|
537
|
+
# THERE IS ALWAYS WILL BE ONLY ONE REQUEST FROM REQUESTER, SINCE THIS IS WHAT WE GOT FROM THE CLIENT.
|
|
538
|
+
request_custom_raw, is_requester_worked = create_requester_request(client_message, sending_socket=sending_socket)
|
|
539
|
+
# We will not process the raw data if requester didn't change anything.
|
|
540
|
+
if is_requester_worked:
|
|
541
|
+
client_message.reinitialize_dynamic_vars()
|
|
542
|
+
client_message.timestamp = datetime.now()
|
|
543
|
+
client_message.request_raw_bytes = request_custom_raw
|
|
544
|
+
client_message.action = 'client_requester'
|
|
545
|
+
process_client_raw_data(request_custom_raw, error_on_receive, client_message)
|
|
546
|
+
record_and_statistics_write(client_message)
|
|
547
|
+
|
|
548
|
+
error_on_send: str = sender.Sender(
|
|
549
|
+
ssl_socket=sending_socket, bytes_to_send=client_message.request_raw_bytes,
|
|
550
|
+
logger=network_logger).send()
|
|
551
|
+
|
|
552
|
+
if error_on_send:
|
|
553
|
+
client_message.reinitialize_dynamic_vars()
|
|
554
|
+
client_message.errors.append(error_on_send)
|
|
555
|
+
client_message.timestamp = datetime.now()
|
|
556
|
+
client_message.action = 'service_send'
|
|
557
|
+
record_and_statistics_write(client_message)
|
|
558
|
+
|
|
559
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
560
|
+
if is_socket_closed or error_on_send:
|
|
561
|
+
exception_or_close_in_receiving_thread = True
|
|
562
|
+
finish_thread()
|
|
563
|
+
return 'return'
|
|
564
|
+
|
|
565
|
+
return None
|
|
566
|
+
|
|
567
|
+
def receive_send_service(
|
|
568
|
+
client_message: ClientMessage,
|
|
569
|
+
receiving_socket: ssl.SSLSocket | socket.socket,
|
|
570
|
+
sending_socket: ssl.SSLSocket | socket.socket
|
|
571
|
+
) -> Literal['return'] | None:
|
|
572
|
+
|
|
573
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
574
|
+
nonlocal server_receive_count
|
|
575
|
+
|
|
576
|
+
server_receive_count += 1
|
|
577
|
+
|
|
578
|
+
network_logger.info(f"Initializing Receiver for Service cycle: {str(server_receive_count)}")
|
|
579
|
+
|
|
580
|
+
# Getting message from the client over the socket using specific class.
|
|
581
|
+
client_message.timestamp = datetime.now()
|
|
582
|
+
received_raw_data, is_socket_closed, error_on_receive = receiver.Receiver(
|
|
583
|
+
ssl_socket=receiving_socket, logger=network_logger).receive()
|
|
584
|
+
|
|
585
|
+
process_server_raw_data(received_raw_data, error_on_receive, client_message)
|
|
586
|
+
client_message.action = 'service_receive'
|
|
587
|
+
|
|
588
|
+
# If there was an exception in the service thread, then receiving empty bytes doesn't mean that
|
|
589
|
+
# the socket was closed by the other side, it means that the service thread closed the socket.
|
|
590
|
+
if (received_raw_data == b'' or error_on_receive) and exception_or_close_in_receiving_thread:
|
|
591
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger,
|
|
592
|
+
logger_method='info')
|
|
593
|
+
return 'return'
|
|
594
|
+
|
|
595
|
+
# We don't need to record aborted socket receives if the socket was closed on receive on the second socket.
|
|
596
|
+
# Meaning 'exception_or_close_in_receiving_thread=True'.
|
|
597
|
+
record_and_statistics_write(client_message)
|
|
598
|
+
if error_on_receive:
|
|
599
|
+
print_api(error_on_receive, logger=network_logger, logger_method='critical')
|
|
600
|
+
|
|
601
|
+
# At this stage, we have received the response from the service, but there was an exception/error or the socket was simply closed.
|
|
602
|
+
# Meaning the 'received_raw_data' is None for exception or b'' for just closed socket.
|
|
603
|
+
# So, there's no point to send anything back to the client.
|
|
604
|
+
# Close both sockets and finish the threads.
|
|
605
|
+
# But if the data was received and then the socket was closed, we first send the data and then close the socket.
|
|
606
|
+
error_on_send: str = str()
|
|
607
|
+
if received_raw_data != b'' and received_raw_data is not None:
|
|
608
|
+
# Now send it to requester/responder.
|
|
609
|
+
bytes_to_send_list: list[bytes] = create_responder_response(client_message)
|
|
610
|
+
|
|
611
|
+
# is_socket_closed: bool = False
|
|
612
|
+
for bytes_to_send_single in bytes_to_send_list:
|
|
613
|
+
client_message.reinitialize_dynamic_vars()
|
|
614
|
+
client_message.timestamp = datetime.now()
|
|
615
|
+
client_message.response_raw_bytes = bytes_to_send_single
|
|
616
|
+
|
|
617
|
+
# This records the requester or responder output, only if it is not the same as the original
|
|
618
|
+
# message.
|
|
619
|
+
if bytes_to_send_single != received_raw_data:
|
|
620
|
+
client_message.action = 'service_responder'
|
|
621
|
+
record_and_statistics_write(client_message)
|
|
622
|
+
|
|
623
|
+
error_on_send: str = sender.Sender(
|
|
624
|
+
ssl_socket=sending_socket, bytes_to_send=bytes_to_send_single,
|
|
625
|
+
logger=network_logger).send()
|
|
626
|
+
|
|
627
|
+
if error_on_send:
|
|
628
|
+
client_message.reinitialize_dynamic_vars()
|
|
629
|
+
client_message.errors.append(error_on_send)
|
|
630
|
+
client_message.timestamp = datetime.now()
|
|
631
|
+
client_message.action = 'client_send'
|
|
632
|
+
|
|
633
|
+
record_and_statistics_write(client_message)
|
|
634
|
+
|
|
635
|
+
# If the socket was closed on message receive, then we'll break the loop only after send.
|
|
636
|
+
if is_socket_closed or error_on_send:
|
|
637
|
+
exception_or_close_in_receiving_thread = True
|
|
638
|
+
|
|
639
|
+
if error_on_receive and 'Connection' in error_on_receive and 'Error' in error_on_receive:
|
|
640
|
+
# If there was a connection error on receive from client, and we're closing the socket to the client,
|
|
641
|
+
# then we'll send TCP RST flag to simulate ConnectionResetError on the client side.
|
|
642
|
+
finish_thread(send_connection_reset=True)
|
|
643
|
+
else:
|
|
644
|
+
finish_thread()
|
|
645
|
+
|
|
646
|
+
return 'return'
|
|
647
|
+
|
|
648
|
+
return None
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def receive_send_start(
|
|
652
|
+
receiving_socket,
|
|
653
|
+
sending_socket = None,
|
|
654
|
+
exception_queue: queue.Queue = None,
|
|
655
|
+
client_connection_message: ClientMessage = None
|
|
656
|
+
):
|
|
657
|
+
nonlocal client_receive_count
|
|
658
|
+
nonlocal server_receive_count
|
|
659
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
660
|
+
|
|
661
|
+
# Set the thread name to the custom name for logging
|
|
662
|
+
# threading.current_thread().name = thread_name
|
|
663
|
+
|
|
664
|
+
# Initialize the client message object with current thread's data.
|
|
665
|
+
client_message: ClientMessage = client_message_first_start()
|
|
666
|
+
|
|
667
|
+
try:
|
|
668
|
+
if receiving_socket is client_socket:
|
|
669
|
+
side: str = 'Client'
|
|
670
|
+
elif receiving_socket is service_socket_instance:
|
|
671
|
+
side: str = 'Service'
|
|
672
|
+
else:
|
|
673
|
+
raise ValueError(f"Unknown side of the socket: {receiving_socket}")
|
|
674
|
+
|
|
675
|
+
while True:
|
|
676
|
+
client_message.reinitialize_dynamic_vars()
|
|
677
|
+
|
|
678
|
+
if side == 'Service' and client_connection_message:
|
|
679
|
+
result: Literal['continue', 'return'] | None = (
|
|
680
|
+
receive_send_service_connect(client_connection_message, sending_socket))
|
|
681
|
+
client_connection_message = None
|
|
682
|
+
if result == 'continue':
|
|
683
|
+
continue
|
|
684
|
+
elif side == 'Client' and config_static.MainConfig.is_offline:
|
|
685
|
+
result: Literal['return'] | None = receive_send_client_offline(client_message, receiving_socket, sending_socket)
|
|
686
|
+
elif side == 'Client':
|
|
687
|
+
result: Literal['return'] | None = receive_send_client(client_message, receiving_socket, sending_socket)
|
|
688
|
+
elif side == 'Service':
|
|
689
|
+
result: Literal['return'] | None = receive_send_service(client_message, receiving_socket, sending_socket)
|
|
247
690
|
else:
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
dns_servers_list=config['tcp']['forwarding_dns_service_ipv4_list___only_for_localhost'])
|
|
259
|
-
# If we're not on localhost, then connect to domain directly.
|
|
260
|
-
else:
|
|
261
|
-
service_client = SocketClient(
|
|
262
|
-
service_name=client_message.server_name, service_port=client_message.destination_port,
|
|
263
|
-
tls=is_tls)
|
|
264
|
-
|
|
265
|
-
# Sending current client message and receiving a response.
|
|
266
|
-
# If there was an error it will be passed to "client_message" object class and if not, "None" will
|
|
267
|
-
# be passed.
|
|
268
|
-
# If there was connection error or socket close, then "ssl_socket" of the "service_client"
|
|
269
|
-
# will be empty.
|
|
270
|
-
response_raw_bytes, client_message.error, client_message.server_ip, service_ssl_socket =\
|
|
271
|
-
service_client.send_receive_to_service(client_message.request_raw_bytes)
|
|
272
|
-
|
|
273
|
-
# Since we need a list for raw bytes, we'll add the 'response_raw_bytes' to our list object.
|
|
274
|
-
# But we need to re-initiate it first.
|
|
275
|
-
client_message.response_list_of_raw_bytes = list()
|
|
276
|
-
client_message.response_list_of_raw_bytes.append(response_raw_bytes)
|
|
277
|
-
|
|
278
|
-
client_message.response_list_of_raw_decoded = list()
|
|
279
|
-
# Make HTTP Response parsing only if there was response at all.
|
|
280
|
-
if response_raw_bytes:
|
|
281
|
-
response_raw_decoded = HTTPResponseParse(response_raw_bytes).response_raw_decoded
|
|
282
|
-
client_message.response_list_of_raw_decoded.append(response_raw_decoded)
|
|
283
|
-
|
|
284
|
-
# So if the socket was closed and there was an error we can break the loop
|
|
285
|
-
if not service_ssl_socket:
|
|
286
|
-
break
|
|
287
|
-
|
|
288
|
-
# This is the point after the response mode check was finished.
|
|
289
|
-
# Recording the message, doesn't matter what type of mode this is.
|
|
290
|
-
try:
|
|
291
|
-
recorded_file = recorder(class_client_message=client_message,
|
|
292
|
-
record_path=config['recorder']['recordings_path']).record()
|
|
293
|
-
except Exception:
|
|
294
|
-
message = "Exception in Recorder"
|
|
295
|
-
print_api(
|
|
296
|
-
message, error_type=True, logger=recorder.logger, logger_method='critical',
|
|
297
|
-
traceback_string=True, oneline=True)
|
|
298
|
-
print_api(
|
|
299
|
-
message, error_type=True, logger=network_logger, logger_method='critical',
|
|
300
|
-
traceback_string=True, oneline=True)
|
|
301
|
-
pass
|
|
302
|
-
|
|
303
|
-
function_recorded = True
|
|
304
|
-
|
|
305
|
-
# Save statistics file.
|
|
306
|
-
output_statistics_csv_row()
|
|
307
|
-
|
|
308
|
-
try:
|
|
309
|
-
# If there is a response, then send it.
|
|
310
|
-
if response_raw_bytes:
|
|
311
|
-
# Sending response/s to client no matter if in record mode or not.
|
|
312
|
-
network_logger.info(
|
|
313
|
-
f"Sending messages to client: {len(client_message.response_list_of_raw_bytes)}")
|
|
314
|
-
function_data_sent = None
|
|
315
|
-
|
|
316
|
-
# Iterate through the list of byte responses.
|
|
317
|
-
for response_raw_bytes in client_message.response_list_of_raw_bytes:
|
|
318
|
-
function_data_sent = Sender(function_client_socket_object, response_raw_bytes).send()
|
|
319
|
-
|
|
320
|
-
# If there was problem with sending data, we'll break current loop.
|
|
321
|
-
if not function_data_sent:
|
|
322
|
-
break
|
|
323
|
-
# If there is no response, close the socket.
|
|
324
|
-
else:
|
|
325
|
-
function_data_sent = None
|
|
326
|
-
network_logger.info(f"Response empty, nothing to send to client.")
|
|
327
|
-
except Exception:
|
|
328
|
-
message = "Not sending anything to the client, since there is no response available"
|
|
329
|
-
print_api(
|
|
330
|
-
message, error_type=True, logger=network_logger, logger_method='critical',
|
|
331
|
-
traceback_string=True, oneline=True)
|
|
332
|
-
# Pass the exception
|
|
333
|
-
pass
|
|
334
|
-
# Break the while loop
|
|
335
|
-
break
|
|
336
|
-
|
|
337
|
-
# If there was problem with sending data, we'll break the while loop
|
|
338
|
-
if not function_data_sent:
|
|
339
|
-
break
|
|
691
|
+
raise ValueError(f"Unknown side [{side}] of the socket: {receiving_socket}")
|
|
692
|
+
|
|
693
|
+
if result == 'return':
|
|
694
|
+
return
|
|
695
|
+
except Exception as exc:
|
|
696
|
+
# If the sockets were already closed, then there is nothing to do here besides log.
|
|
697
|
+
# if (isinstance(exc, OSError) and exc.errno == 10038 and
|
|
698
|
+
# client_socket.fileno() == -1 and service_socket_instance.fileno() == -1):
|
|
699
|
+
if isinstance(exc, OSError) and exc.errno == 10038:
|
|
700
|
+
print_api("Both sockets are closed, breaking the loop", logger=network_logger, logger_method='info')
|
|
340
701
|
else:
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
702
|
+
handle_exceptions_on_sub_connection_thread(client_message, exception_queue, exc)
|
|
703
|
+
|
|
704
|
+
def handle_exceptions_on_sub_connection_thread(
|
|
705
|
+
client_message: ClientMessage,
|
|
706
|
+
exception_queue: queue.Queue,
|
|
707
|
+
exc: Exception
|
|
708
|
+
):
|
|
709
|
+
nonlocal exception_or_close_in_receiving_thread
|
|
710
|
+
|
|
711
|
+
exception_or_close_in_receiving_thread = True
|
|
712
|
+
# handle_exceptions(exc, client_message, recorded)
|
|
713
|
+
exception_message = tracebacks.get_as_string(one_line=True)
|
|
714
|
+
|
|
715
|
+
error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
|
|
716
|
+
print_api("Exception in a thread, forwarding to parent thread.", logger_method='info', logger=network_logger)
|
|
717
|
+
client_message.errors.append(error_message)
|
|
718
|
+
|
|
719
|
+
# if not recorded:
|
|
720
|
+
# record_and_statistics_write(client_message)
|
|
721
|
+
|
|
722
|
+
finish_thread()
|
|
723
|
+
exception_queue.put(exc)
|
|
724
|
+
|
|
725
|
+
def handle_exceptions_on_main_connection_thread(
|
|
726
|
+
exc: Exception,
|
|
727
|
+
client_message: ClientMessage
|
|
728
|
+
):
|
|
729
|
+
exception_message = tracebacks.get_as_string(one_line=True)
|
|
730
|
+
error_message = f'Socket Thread [{str(thread_id)}] Exception: {exception_message}'
|
|
731
|
+
print_api(error_message, logger_method='critical', logger=network_logger)
|
|
732
|
+
client_message.errors.append(error_message)
|
|
345
733
|
|
|
346
734
|
# === At this point while loop of 'client_connection_boolean' was broken =======================================
|
|
347
735
|
# If recorder wasn't executed before, then execute it now
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
736
|
+
record_and_statistics_write(client_message)
|
|
737
|
+
|
|
738
|
+
finish_thread()
|
|
739
|
+
|
|
740
|
+
# Add custom attribute to the exception.
|
|
741
|
+
exc.engine_name = client_message.engine_name
|
|
742
|
+
|
|
743
|
+
# After the socket clean up, we will still raise the exception to the main thread.
|
|
744
|
+
raise exc
|
|
745
|
+
|
|
746
|
+
# ================================================================================================================
|
|
747
|
+
# This is the start of the thread_worker_main function
|
|
748
|
+
network_logger = loggingw.get_logger_with_level(config_static.MainConfig.LOGGER_NAME)
|
|
749
|
+
|
|
750
|
+
# Only protocols that are encrypted with TLS have the server name attribute.
|
|
751
|
+
if is_tls:
|
|
752
|
+
# Get current destination domain
|
|
753
|
+
server_name = client_socket.server_hostname
|
|
754
|
+
# If there is no server name from the TLS handshake, then we'll use the domain from the DNS.
|
|
755
|
+
if not server_name:
|
|
756
|
+
server_name = domain_from_dns
|
|
757
|
+
# If the protocol is not TLS, then we'll use the domain from the DNS.
|
|
758
|
+
else:
|
|
759
|
+
server_name = domain_from_dns
|
|
760
|
+
|
|
761
|
+
thread_id = threads.current_thread_id()
|
|
762
|
+
|
|
763
|
+
process_name: str = multiprocessing.current_process().name
|
|
764
|
+
current_thread = threading.current_thread()
|
|
765
|
+
thread_process_name: str = f"{process_name} | {current_thread.name}"
|
|
766
|
+
current_thread.name = thread_process_name
|
|
767
|
+
|
|
768
|
+
# This is the main protocols.
|
|
769
|
+
protocol: str = str()
|
|
770
|
+
# This is the secondary protocol in the websocket.
|
|
771
|
+
protocol3: str = str()
|
|
772
|
+
# # This is Client Masked Frame Parser.
|
|
773
|
+
# websocket_masked_frame_parser = websocket_parse.WebsocketFrameParser()
|
|
774
|
+
# # This is Server UnMasked Frame Parser.
|
|
775
|
+
# websocket_unmasked_frame_parser = websocket_parse.WebsocketFrameParser()
|
|
776
|
+
websocket_frame_parser = websocket_parse.WebsocketFrameParser()
|
|
777
|
+
|
|
778
|
+
# Loading parser by domain, if there is no parser for current domain - general reference parser is loaded.
|
|
779
|
+
# These should be outside any loop and initialized only once entering the thread.
|
|
780
|
+
found_domain_module = initialize_engines.assign_class_by_domain(
|
|
781
|
+
engines_list=engines_list,
|
|
782
|
+
message_domain_name=server_name,
|
|
783
|
+
reference_module=config_static.REFERENCE_MODULE
|
|
784
|
+
)
|
|
785
|
+
parser = found_domain_module.parser_class_object
|
|
786
|
+
requester = found_domain_module.requester_class_object()
|
|
787
|
+
responder = found_domain_module.responder_class_object()
|
|
788
|
+
recorder = found_domain_module.recorder_class_object(record_path=config_static.LogRec.recordings_path)
|
|
789
|
+
|
|
790
|
+
engine_name: str = recorder.engine_name
|
|
791
|
+
|
|
792
|
+
for engine in engines_list:
|
|
793
|
+
if engine.engine_name == engine_name:
|
|
794
|
+
responder.add_args(engine=engine)
|
|
795
|
+
break
|
|
796
|
+
|
|
797
|
+
network_logger.info(f"Assigned Modules for [{server_name}]: "
|
|
798
|
+
f"{parser.__name__}, "
|
|
799
|
+
f"{requester.__class__.__name__}, "
|
|
800
|
+
f"{responder.__class__.__name__}, "
|
|
801
|
+
f"{recorder.__class__.__name__}")
|
|
802
|
+
|
|
803
|
+
# Initializing the client message object with current thread's data.
|
|
804
|
+
# This is needed only to skip error alerts after 'try'.
|
|
805
|
+
client_message_connection: ClientMessage = ClientMessage()
|
|
806
|
+
# This is needed to indicate if there was an exception or socket was closed in any of the receiving thread.
|
|
807
|
+
exception_or_close_in_receiving_thread: bool = False
|
|
808
|
+
# Queue for http request URI paths.
|
|
809
|
+
http_path_queue: queue.Queue = queue.Queue()
|
|
364
810
|
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
if service_client:
|
|
368
|
-
if service_client.socket_instance:
|
|
369
|
-
service_client.close_socket()
|
|
811
|
+
try:
|
|
812
|
+
client_ip, source_port = client_socket.getpeername()
|
|
370
813
|
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
814
|
+
try:
|
|
815
|
+
client_name: str = socket.gethostbyaddr(client_ip)[0]
|
|
816
|
+
# This can happen if the host changed IP address, but it wasn't propagated over DHCP.
|
|
817
|
+
except socket.herror:
|
|
818
|
+
client_name = ""
|
|
819
|
+
|
|
820
|
+
client_name = client_name.lower()
|
|
821
|
+
destination_port: int = client_socket.getsockname()[1]
|
|
822
|
+
destination_port_str: str = str(destination_port)
|
|
823
|
+
|
|
824
|
+
# If the destination port is in the on_port_connect dictionary, then we'll get the port from there.
|
|
825
|
+
if destination_port_str in found_domain_module.on_port_connect:
|
|
826
|
+
on_port_connect_value = found_domain_module.on_port_connect[destination_port_str]
|
|
827
|
+
_, destination_port_str = initialize_engines.get_ipv4_from_engine_on_connect_port(on_port_connect_value)
|
|
828
|
+
destination_port: int = int(destination_port_str)
|
|
829
|
+
|
|
830
|
+
if config_static.MainConfig.is_offline:
|
|
831
|
+
# If in offline mode, then we'll get the TCP server's input address.
|
|
832
|
+
server_ip = client_socket.getsockname()[0]
|
|
833
|
+
else:
|
|
834
|
+
# If not in offline mode, we will get the ip from the socket that will connect later to the service.
|
|
835
|
+
server_ip = ""
|
|
836
|
+
|
|
837
|
+
network_logger.info(f"Thread Created - Client [{client_ip}:{source_port}] | "
|
|
838
|
+
f"Destination service: [{server_name}:{destination_port}]")
|
|
839
|
+
|
|
840
|
+
origin_service_client_instance = None
|
|
841
|
+
client_receive_count: int = 0
|
|
842
|
+
server_receive_count: int = 0
|
|
843
|
+
client_message_connection = client_message_first_start()
|
|
844
|
+
|
|
845
|
+
# If we're not in offline mode, then we'll create the client socket to the service.
|
|
846
|
+
# noinspection PyTypeChecker
|
|
847
|
+
connection_error: str = None
|
|
848
|
+
service_socket_instance = None
|
|
849
|
+
client_message_connection.action = 'service_connect'
|
|
850
|
+
client_message_connection.timestamp = datetime.now()
|
|
851
|
+
|
|
852
|
+
if config_static.MainConfig.is_offline:
|
|
853
|
+
client_message_connection.info = 'Offline Mode'
|
|
854
|
+
else:
|
|
855
|
+
origin_service_client_instance = create_client_socket(client_message_connection)
|
|
856
|
+
service_socket_instance, connection_error = origin_service_client_instance.service_connection()
|
|
375
857
|
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
858
|
+
if connection_error:
|
|
859
|
+
client_message_connection.errors.append(connection_error)
|
|
860
|
+
record_and_statistics_write(client_message_connection)
|
|
861
|
+
else:
|
|
862
|
+
# Now we'll update the server IP with the IP of the service.
|
|
863
|
+
server_ip = service_socket_instance.getpeername()[0]
|
|
864
|
+
client_message_connection.server_ip = server_ip
|
|
865
|
+
|
|
866
|
+
if not connection_error:
|
|
867
|
+
client_exception_queue: queue.Queue = queue.Queue()
|
|
868
|
+
client_thread = threading.Thread(
|
|
869
|
+
target=receive_send_start,
|
|
870
|
+
args=(client_socket, service_socket_instance, client_exception_queue, None),
|
|
871
|
+
name=f"{process_name} | Thread-{thread_id}-Client",
|
|
872
|
+
daemon=True)
|
|
873
|
+
client_thread.start()
|
|
874
|
+
|
|
875
|
+
service_exception_queue: queue.Queue = queue.Queue()
|
|
876
|
+
if not config_static.MainConfig.is_offline:
|
|
877
|
+
service_thread = threading.Thread(
|
|
878
|
+
target=receive_send_start,
|
|
879
|
+
args=(service_socket_instance, client_socket, service_exception_queue, client_message_connection),
|
|
880
|
+
name=f"{process_name} | Thread-{thread_id}-Service",
|
|
881
|
+
daemon=True)
|
|
882
|
+
service_thread.start()
|
|
883
|
+
|
|
884
|
+
client_thread.join()
|
|
885
|
+
# If we're in offline mode, then there is no service thread.
|
|
886
|
+
if not config_static.MainConfig.is_offline:
|
|
887
|
+
# If we're not in offline mode, then we'll wait for the service thread to finish.
|
|
888
|
+
# noinspection PyUnboundLocalVariable
|
|
889
|
+
service_thread.join()
|
|
890
|
+
|
|
891
|
+
# If there was an exception in any of the threads, then we'll raise it here.
|
|
892
|
+
if not client_exception_queue.empty():
|
|
893
|
+
raise client_exception_queue.get()
|
|
894
|
+
if not service_exception_queue.empty():
|
|
895
|
+
raise service_exception_queue.get()
|
|
896
|
+
|
|
897
|
+
finish_thread()
|
|
898
|
+
except Exception as e:
|
|
899
|
+
handle_exceptions_on_main_connection_thread(e, client_message_connection)
|