atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. atomicshop/__init__.py +1 -1
  2. atomicshop/{addons/mains → a_mains}/FACT/update_extract.py +3 -2
  3. atomicshop/a_mains/addons/process_list/compile.cmd +7 -0
  4. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
  5. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
  6. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
  7. atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +8 -1
  8. atomicshop/a_mains/dns_gateway_setting.py +11 -0
  9. atomicshop/a_mains/get_local_tcp_ports.py +85 -0
  10. atomicshop/a_mains/github_wrapper.py +11 -0
  11. atomicshop/a_mains/install_ca_certificate.py +172 -0
  12. atomicshop/{addons/mains → a_mains}/msi_unpacker.py +3 -1
  13. atomicshop/a_mains/process_from_port.py +119 -0
  14. atomicshop/a_mains/set_default_dns_gateway.py +90 -0
  15. atomicshop/a_mains/update_config_toml.py +38 -0
  16. atomicshop/appointment_management.py +5 -3
  17. atomicshop/basics/ansi_escape_codes.py +3 -1
  18. atomicshop/basics/argparse_template.py +2 -0
  19. atomicshop/basics/booleans.py +27 -30
  20. atomicshop/basics/bytes_arrays.py +43 -0
  21. atomicshop/basics/classes.py +149 -1
  22. atomicshop/basics/dicts.py +12 -0
  23. atomicshop/basics/enums.py +2 -2
  24. atomicshop/basics/exceptions.py +5 -1
  25. atomicshop/basics/list_of_classes.py +29 -0
  26. atomicshop/basics/list_of_dicts.py +69 -5
  27. atomicshop/basics/lists.py +14 -0
  28. atomicshop/basics/multiprocesses.py +374 -50
  29. atomicshop/basics/package_module.py +10 -0
  30. atomicshop/basics/strings.py +160 -7
  31. atomicshop/basics/threads.py +14 -0
  32. atomicshop/basics/tracebacks.py +13 -4
  33. atomicshop/certificates.py +153 -52
  34. atomicshop/config_init.py +12 -7
  35. atomicshop/console_user_response.py +7 -14
  36. atomicshop/consoles.py +9 -0
  37. atomicshop/datetimes.py +98 -0
  38. atomicshop/diff_check.py +340 -40
  39. atomicshop/dns.py +128 -12
  40. atomicshop/etws/_pywintrace_fix.py +17 -0
  41. atomicshop/etws/const.py +38 -0
  42. atomicshop/etws/providers.py +21 -0
  43. atomicshop/etws/sessions.py +43 -0
  44. atomicshop/etws/trace.py +168 -0
  45. atomicshop/etws/traces/trace_dns.py +162 -0
  46. atomicshop/etws/traces/trace_sysmon_process_creation.py +126 -0
  47. atomicshop/etws/traces/trace_tcp.py +130 -0
  48. atomicshop/file_io/csvs.py +222 -24
  49. atomicshop/file_io/docxs.py +35 -18
  50. atomicshop/file_io/file_io.py +35 -19
  51. atomicshop/file_io/jsons.py +49 -0
  52. atomicshop/file_io/tomls.py +139 -0
  53. atomicshop/filesystem.py +864 -293
  54. atomicshop/get_process_list.py +133 -0
  55. atomicshop/{process_name_cmd.py → get_process_name_cmd_dll.py} +52 -19
  56. atomicshop/http_parse.py +149 -93
  57. atomicshop/ip_addresses.py +6 -1
  58. atomicshop/mitm/centered_settings.py +132 -0
  59. atomicshop/mitm/config_static.py +207 -0
  60. atomicshop/mitm/config_toml_editor.py +55 -0
  61. atomicshop/mitm/connection_thread_worker.py +875 -357
  62. atomicshop/mitm/engines/__parent/parser___parent.py +4 -17
  63. atomicshop/mitm/engines/__parent/recorder___parent.py +108 -51
  64. atomicshop/mitm/engines/__parent/requester___parent.py +116 -0
  65. atomicshop/mitm/engines/__parent/responder___parent.py +75 -114
  66. atomicshop/mitm/engines/__reference_general/parser___reference_general.py +10 -7
  67. atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +5 -5
  68. atomicshop/mitm/engines/__reference_general/requester___reference_general.py +47 -0
  69. atomicshop/mitm/engines/__reference_general/responder___reference_general.py +95 -13
  70. atomicshop/mitm/engines/create_module_template.py +58 -14
  71. atomicshop/mitm/import_config.py +359 -139
  72. atomicshop/mitm/initialize_engines.py +160 -74
  73. atomicshop/mitm/message.py +64 -23
  74. atomicshop/mitm/mitm_main.py +892 -0
  75. atomicshop/mitm/recs_files.py +183 -0
  76. atomicshop/mitm/shared_functions.py +4 -10
  77. atomicshop/mitm/ssh_tester.py +82 -0
  78. atomicshop/mitm/statistic_analyzer.py +257 -166
  79. atomicshop/mitm/statistic_analyzer_helper/analyzer_helper.py +136 -0
  80. atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +525 -0
  81. atomicshop/monitor/change_monitor.py +96 -120
  82. atomicshop/monitor/checks/dns.py +139 -70
  83. atomicshop/monitor/checks/file.py +77 -0
  84. atomicshop/monitor/checks/network.py +81 -77
  85. atomicshop/monitor/checks/process_running.py +33 -34
  86. atomicshop/monitor/checks/url.py +94 -0
  87. atomicshop/networks.py +671 -0
  88. atomicshop/on_exit.py +205 -0
  89. atomicshop/package_mains_processor.py +84 -0
  90. atomicshop/permissions/permissions.py +22 -0
  91. atomicshop/permissions/ubuntu_permissions.py +239 -0
  92. atomicshop/permissions/win_permissions.py +33 -0
  93. atomicshop/print_api.py +24 -41
  94. atomicshop/process.py +63 -17
  95. atomicshop/process_poller/__init__.py +0 -0
  96. atomicshop/process_poller/pollers/__init__.py +0 -0
  97. atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py +95 -0
  98. atomicshop/process_poller/process_pool.py +207 -0
  99. atomicshop/process_poller/simple_process_pool.py +311 -0
  100. atomicshop/process_poller/tracer_base.py +45 -0
  101. atomicshop/process_poller/tracers/__init__.py +0 -0
  102. atomicshop/process_poller/tracers/event_log.py +46 -0
  103. atomicshop/process_poller/tracers/sysmon_etw.py +68 -0
  104. atomicshop/python_file_patcher.py +1 -1
  105. atomicshop/python_functions.py +27 -75
  106. atomicshop/question_answer_engine.py +2 -2
  107. atomicshop/scheduling.py +24 -5
  108. atomicshop/sound.py +4 -2
  109. atomicshop/speech_recognize.py +8 -0
  110. atomicshop/ssh_remote.py +158 -172
  111. atomicshop/startup/__init__.py +0 -0
  112. atomicshop/startup/win/__init__.py +0 -0
  113. atomicshop/startup/win/startup_folder.py +53 -0
  114. atomicshop/startup/win/task_scheduler.py +119 -0
  115. atomicshop/system_resource_monitor.py +61 -46
  116. atomicshop/system_resources.py +8 -8
  117. atomicshop/tempfiles.py +1 -2
  118. atomicshop/timer.py +30 -11
  119. atomicshop/urls.py +41 -0
  120. atomicshop/venvs.py +28 -0
  121. atomicshop/versioning.py +27 -0
  122. atomicshop/web.py +110 -25
  123. atomicshop/web_apis/__init__.py +0 -0
  124. atomicshop/web_apis/google_custom_search.py +44 -0
  125. atomicshop/web_apis/google_llm.py +188 -0
  126. atomicshop/websocket_parse.py +450 -0
  127. atomicshop/wrappers/certauthw/certauth.py +1 -0
  128. atomicshop/wrappers/cryptographyw.py +29 -8
  129. atomicshop/wrappers/ctyping/etw_winapi/__init__.py +0 -0
  130. atomicshop/wrappers/ctyping/etw_winapi/const.py +335 -0
  131. atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +393 -0
  132. atomicshop/wrappers/ctyping/file_details_winapi.py +67 -0
  133. atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
  134. atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +13 -9
  135. atomicshop/wrappers/ctyping/msi_windows_installer/tables.py +35 -0
  136. atomicshop/wrappers/ctyping/setup_device.py +466 -0
  137. atomicshop/wrappers/ctyping/win_console.py +39 -0
  138. atomicshop/wrappers/dockerw/dockerw.py +113 -2
  139. atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
  140. atomicshop/wrappers/elasticsearchw/elastic_infra.py +75 -0
  141. atomicshop/wrappers/elasticsearchw/elasticsearchw.py +2 -20
  142. atomicshop/wrappers/factw/get_file_data.py +12 -5
  143. atomicshop/wrappers/factw/install/install_after_restart.py +89 -5
  144. atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +20 -14
  145. atomicshop/wrappers/factw/postgresql/firmware.py +4 -6
  146. atomicshop/wrappers/githubw.py +583 -51
  147. atomicshop/wrappers/loggingw/consts.py +49 -0
  148. atomicshop/wrappers/loggingw/filters.py +102 -0
  149. atomicshop/wrappers/loggingw/formatters.py +58 -71
  150. atomicshop/wrappers/loggingw/handlers.py +459 -40
  151. atomicshop/wrappers/loggingw/loggers.py +19 -0
  152. atomicshop/wrappers/loggingw/loggingw.py +1010 -178
  153. atomicshop/wrappers/loggingw/reading.py +344 -19
  154. atomicshop/wrappers/mongodbw/__init__.py +0 -0
  155. atomicshop/wrappers/mongodbw/mongo_infra.py +31 -0
  156. atomicshop/wrappers/mongodbw/mongodbw.py +1432 -0
  157. atomicshop/wrappers/netshw.py +271 -0
  158. atomicshop/wrappers/playwrightw/engine.py +34 -19
  159. atomicshop/wrappers/playwrightw/infra.py +5 -0
  160. atomicshop/wrappers/playwrightw/javascript.py +7 -3
  161. atomicshop/wrappers/playwrightw/keyboard.py +14 -0
  162. atomicshop/wrappers/playwrightw/scenarios.py +172 -5
  163. atomicshop/wrappers/playwrightw/waits.py +9 -7
  164. atomicshop/wrappers/powershell_networking.py +80 -0
  165. atomicshop/wrappers/psutilw/processes.py +81 -0
  166. atomicshop/wrappers/psutilw/psutil_networks.py +85 -0
  167. atomicshop/wrappers/psutilw/psutilw.py +9 -0
  168. atomicshop/wrappers/pyopensslw.py +9 -2
  169. atomicshop/wrappers/pywin32w/__init__.py +0 -0
  170. atomicshop/wrappers/pywin32w/cert_store.py +116 -0
  171. atomicshop/wrappers/pywin32w/console.py +34 -0
  172. atomicshop/wrappers/pywin32w/win_event_log/__init__.py +0 -0
  173. atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
  174. atomicshop/wrappers/pywin32w/win_event_log/subscribe.py +212 -0
  175. atomicshop/wrappers/pywin32w/win_event_log/subscribes/__init__.py +0 -0
  176. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +57 -0
  177. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +49 -0
  178. atomicshop/wrappers/pywin32w/win_event_log/subscribes/schannel_logging.py +97 -0
  179. atomicshop/wrappers/pywin32w/winshell.py +19 -0
  180. atomicshop/wrappers/pywin32w/wmis/__init__.py +0 -0
  181. atomicshop/wrappers/pywin32w/wmis/msft_netipaddress.py +113 -0
  182. atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +259 -0
  183. atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +112 -0
  184. atomicshop/wrappers/pywin32w/wmis/wmi_helpers.py +236 -0
  185. atomicshop/wrappers/socketw/accepter.py +21 -7
  186. atomicshop/wrappers/socketw/certificator.py +216 -150
  187. atomicshop/wrappers/socketw/creator.py +190 -50
  188. atomicshop/wrappers/socketw/dns_server.py +500 -173
  189. atomicshop/wrappers/socketw/exception_wrapper.py +45 -52
  190. atomicshop/wrappers/socketw/process_getter.py +86 -0
  191. atomicshop/wrappers/socketw/receiver.py +144 -102
  192. atomicshop/wrappers/socketw/sender.py +65 -35
  193. atomicshop/wrappers/socketw/sni.py +334 -165
  194. atomicshop/wrappers/socketw/socket_base.py +134 -0
  195. atomicshop/wrappers/socketw/socket_client.py +137 -95
  196. atomicshop/wrappers/socketw/socket_server_tester.py +14 -9
  197. atomicshop/wrappers/socketw/socket_wrapper.py +717 -116
  198. atomicshop/wrappers/socketw/ssl_base.py +15 -14
  199. atomicshop/wrappers/socketw/statistics_csv.py +148 -17
  200. atomicshop/wrappers/sysmonw.py +157 -0
  201. atomicshop/wrappers/ubuntu_terminal.py +65 -26
  202. atomicshop/wrappers/win_auditw.py +189 -0
  203. atomicshop/wrappers/winregw/__init__.py +0 -0
  204. atomicshop/wrappers/winregw/winreg_installed_software.py +58 -0
  205. atomicshop/wrappers/winregw/winreg_network.py +232 -0
  206. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/METADATA +31 -49
  207. atomicshop-3.10.5.dist-info/RECORD +306 -0
  208. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/WHEEL +1 -1
  209. atomicshop/_basics_temp.py +0 -101
  210. atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
  211. atomicshop/addons/a_setup_scripts/install_pywintrace_0.3.cmd +0 -2
  212. atomicshop/addons/mains/install_docker_rootless_ubuntu.py +0 -11
  213. atomicshop/addons/mains/install_docker_ubuntu_main_sudo.py +0 -11
  214. atomicshop/addons/mains/install_elastic_search_and_kibana_ubuntu.py +0 -10
  215. atomicshop/addons/mains/install_wsl_ubuntu_lts_admin.py +0 -9
  216. atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
  217. atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
  218. atomicshop/addons/package_setup/Setup.cmd +0 -7
  219. atomicshop/addons/process_list/compile.cmd +0 -2
  220. atomicshop/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
  221. atomicshop/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
  222. atomicshop/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
  223. atomicshop/archiver/_search_in_zip.py +0 -189
  224. atomicshop/archiver/archiver.py +0 -34
  225. atomicshop/archiver/search_in_archive.py +0 -250
  226. atomicshop/archiver/sevenz_app_w.py +0 -86
  227. atomicshop/archiver/sevenzs.py +0 -44
  228. atomicshop/archiver/zips.py +0 -293
  229. atomicshop/etw/dns_trace.py +0 -118
  230. atomicshop/etw/etw.py +0 -61
  231. atomicshop/file_types.py +0 -24
  232. atomicshop/mitm/engines/create_module_template_example.py +0 -13
  233. atomicshop/mitm/initialize_mitm_server.py +0 -240
  234. atomicshop/monitor/checks/hash.py +0 -44
  235. atomicshop/monitor/checks/hash_checks/file.py +0 -55
  236. atomicshop/monitor/checks/hash_checks/url.py +0 -62
  237. atomicshop/pbtkmultifile_argparse.py +0 -88
  238. atomicshop/permissions.py +0 -110
  239. atomicshop/process_poller.py +0 -237
  240. atomicshop/script_as_string_processor.py +0 -38
  241. atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
  242. atomicshop/ssh_scripts/process_from_port.py +0 -27
  243. atomicshop/wrappers/_process_wrapper_curl.py +0 -27
  244. atomicshop/wrappers/_process_wrapper_tar.py +0 -21
  245. atomicshop/wrappers/dockerw/install_docker.py +0 -209
  246. atomicshop/wrappers/elasticsearchw/infrastructure.py +0 -265
  247. atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -232
  248. atomicshop/wrappers/ffmpegw.py +0 -125
  249. atomicshop/wrappers/loggingw/checks.py +0 -20
  250. atomicshop/wrappers/nodejsw/install_nodejs.py +0 -139
  251. atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
  252. atomicshop/wrappers/socketw/base.py +0 -59
  253. atomicshop/wrappers/socketw/get_process.py +0 -107
  254. atomicshop/wrappers/wslw.py +0 -191
  255. atomicshop-2.11.47.dist-info/RECORD +0 -251
  256. /atomicshop/{addons/mains → a_mains}/FACT/factw_fact_extractor_docker_image_main_sudo.py +0 -0
  257. /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
  258. /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
  259. /atomicshop/{addons/mains → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
  260. /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
  261. /atomicshop/{addons/mains → a_mains}/search_for_hyperlinks_in_docx.py +0 -0
  262. /atomicshop/{archiver → etws}/__init__.py +0 -0
  263. /atomicshop/{etw → etws/traces}/__init__.py +0 -0
  264. /atomicshop/{monitor/checks/hash_checks → mitm/statistic_analyzer_helper}/__init__.py +0 -0
  265. /atomicshop/{wrappers/nodejsw → permissions}/__init__.py +0 -0
  266. /atomicshop/wrappers/pywin32w/{wmi_win32process.py → wmis/win32process.py} +0 -0
  267. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info/licenses}/LICENSE.txt +0 -0
  268. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/top_level.txt +0 -0
@@ -1,240 +0,0 @@
1
- import os
2
- import threading
3
-
4
- # Importing atomicshop package to get the version of the package.
5
- import atomicshop
6
-
7
- from .import_config import ImportConfig
8
- from .initialize_engines import ModuleCategory
9
- from .connection_thread_worker import thread_worker_main
10
- from .. import filesystem, queues
11
- from ..python_functions import get_current_python_version_string, check_python_version_compliance
12
- from ..wrappers.socketw.socket_wrapper import SocketWrapper
13
- from ..wrappers.socketw.dns_server import DnsServer
14
- from ..basics import dicts_nested
15
- from ..wrappers.loggingw import loggingw
16
- from ..print_api import print_api
17
-
18
-
19
- def initialize_mitm_server(config_static):
20
- # Main function should return integer with error code, 0 is successful.
21
- # Since listening server is infinite, this will not be reached.
22
- def output_statistics_csv_header():
23
- # Since there is no implementation of header in logging file handler modules, we'll do it manually each time.
24
- statistics_header: list = ['request_time_sent',
25
- 'host',
26
- 'path',
27
- 'command',
28
- 'status_code',
29
- 'request_size_bytes',
30
- 'response_size_bytes',
31
- # 'request_hex',
32
- # 'response_hex',
33
- 'file_path',
34
- 'process_cmd',
35
- 'error'
36
- ]
37
- statistics_logger.info(','.join(statistics_header))
38
-
39
- # After modules import - we check for python version.
40
- check_python_version_compliance(minimum_version='3.11')
41
-
42
- # Preparing everything for the logging module.
43
- # Log folder path is in the "config.ini" file, so we need to read it before setting loggers.
44
- config_importer = ImportConfig(
45
- file_name=config_static.CONFIG_INI_SERVER_FILE_NAME, directory_path=config_static.WORKING_DIRECTORY)
46
- config_importer.open()
47
- config = config_importer.config
48
-
49
- # Create folders.
50
- filesystem.create_directory(config['log']['logs_path'])
51
- filesystem.create_directory(config['recorder']['recordings_path'])
52
- if config['certificates']['sni_get_server_certificate_from_server_socket']:
53
- filesystem.create_directory(
54
- config['certificates']['sni_server_certificate_from_server_socket_download_directory'])
55
-
56
- # Create a logger that will log messages to file, Initiate System logger.
57
- system_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
58
- "system", config['log']['logs_path'], disable_duplicate_ms=True)
59
-
60
- # Writing first log.
61
- system_logger.info("======================================")
62
-
63
- if config_importer.admin_rights is not None:
64
- if not config_importer.admin_rights:
65
- system_logger.error("User continued with errors on Command Line harvesting for system processes.")
66
-
67
- system_logger.info("Server Started.")
68
- system_logger.info(f"Python Version: {get_current_python_version_string()}")
69
- system_logger.info(f"Script Version: {config_static.SCRIPT_VERSION}")
70
- system_logger.info(f"Atomic Workshop Version: {atomicshop.__version__}")
71
- system_logger.info(f"Loaded config.ini: {config_importer.config_parser.file_path}")
72
- system_logger.info(f"Log folder: {config['log']['logs_path']}")
73
- system_logger.info(f"Recordings folder for Requests/Responses: {config['recorder']['recordings_path']}")
74
- system_logger.info(f"Loaded system logger: {system_logger}")
75
-
76
- system_logger.info(f"TCP Server Target IP: {config['dns']['target_tcp_server_ipv4']}")
77
-
78
- # Some 'config.ini' settings logging ===========================================================================
79
- if config['certificates']['default_server_certificate_usage']:
80
- system_logger.info(
81
- f"Default server certificate usage enabled, if no SNI available: "
82
- f"{config_static.CONFIG_EXTENDED['certificates']['default_server_certificate_directory']}"
83
- f"{os.sep}{config_static.CONFIG_EXTENDED['certificates']['default_server_certificate_name']}.pem")
84
-
85
- if config['certificates']['sni_server_certificates_cache_directory']:
86
- system_logger.info(
87
- f"SNI function certificates creation enabled. Certificates cache: "
88
- f"{config['certificates']['sni_server_certificates_cache_directory']}")
89
- else:
90
- system_logger.info(f"SNI function certificates creation disabled.")
91
-
92
- if config['certificates']['custom_server_certificate_usage']:
93
- system_logger.info(f"Custom server certificate usage is enabled.")
94
- system_logger.info(f"Custom Certificate Path: {config['certificates']['custom_server_certificate_path']}")
95
-
96
- # If 'custom_private_key_path' field was populated.
97
- if config['certificates']['custom_private_key_path']:
98
- system_logger.info(
99
- f"Custom Certificate Private Key Path: {config['certificates']['custom_private_key_path']}")
100
- else:
101
- system_logger.info(f"Custom Certificate Private Key Path wasn't provided in [advanced] section. "
102
- f"Assuming the private key is inside the certificate file.")
103
-
104
- # === Importing engine modules =================================================================================
105
- system_logger.info("Importing engine modules.")
106
-
107
- # Get full paths of all the 'engine_config.ini' files.
108
- engine_config_path_list = filesystem.get_file_paths_from_directory(
109
- directory_path=config_static.ENGINES_DIRECTORY_PATH,
110
- file_name_check_pattern=config_static.ENGINE_CONFIG_FILE_NAME)
111
-
112
- # Iterate through all the 'engine_config.ini' file paths.
113
- domains_engine_list_full: list = list()
114
- engines_list: list = list()
115
- for engine_config_path in engine_config_path_list:
116
- # Initialize engine.
117
- current_module = ModuleCategory(config_static.WORKING_DIRECTORY)
118
- current_module.fill_engine_fields_from_config(engine_config_path)
119
- current_module.initialize_engine(logs_path=config['log']['logs_path'],
120
- logger=system_logger)
121
-
122
- # Extending the full engine domain list with this list.
123
- domains_engine_list_full.extend(current_module.domain_list)
124
- # Append the object to the engines list
125
- engines_list.append(current_module)
126
- # === EOF Importing engine modules =============================================================================
127
- # ==== Initialize Reference Module =============================================================================
128
- reference_module = ModuleCategory(config_static.WORKING_DIRECTORY)
129
- reference_module.fill_engine_fields_from_general_reference(config_static.ENGINES_DIRECTORY_PATH)
130
- reference_module.initialize_engine(logs_path=config['log']['logs_path'],
131
- logger=system_logger, stdout=False, reference_general=True)
132
- # === EOF Initialize Reference Module ==========================================================================
133
- # === Engine logging ===========================================================================================
134
- # Printing the parsers using "start=1" for index to start counting from "1" and not "0"
135
- print_api(f"[*] Found Engines:", logger=system_logger)
136
- for index, engine in enumerate(engines_list, start=1):
137
- message = f"[*] {index}: {engine.engine_name} | {engine.domain_list}"
138
- print_api(message, logger=system_logger)
139
-
140
- message = (f"[*] Modules: {engine.parser_class_object.__name__}, "
141
- f"{engine.responder_class_object.__name__}, "
142
- f"{engine.recorder_class_object.__name__}")
143
- print_api(message, logger=system_logger)
144
-
145
- if config['dns']['enable_dns_server']:
146
- print_api("DNS Server is enabled.", logger=system_logger)
147
-
148
- # If engines were found and dns is set to route by the engine domains.
149
- if engines_list and config['dns']['route_to_tcp_server_only_engine_domains']:
150
- print_api("Engine domains will be routed by the DNS server to Built-in TCP Server.", logger=system_logger)
151
- # If engines were found, but the dns isn't set to route to engines.
152
- elif engines_list and not config['dns']['route_to_tcp_server_only_engine_domains']:
153
- message = f"[*] Engine domains found, but the DNS routing is set not to use them for routing."
154
- print_api(message, color="yellow", logger=system_logger)
155
- elif not engines_list and config['dns']['route_to_tcp_server_only_engine_domains']:
156
- raise ValueError("No engines were found, but the DNS routing is set to use them for routing.\n"
157
- "Please check your DNS configuration in the 'config.ini' file.")
158
-
159
- if config['dns']['route_to_tcp_server_all_domains']:
160
- print_api("All domains will be routed by the DNS server to Built-in TCP Server.", logger=system_logger)
161
-
162
- if config['dns']['regular_resolving']:
163
- print_api(
164
- "Regular DNS resolving is enabled. Built-in TCP server will not be routed to",
165
- logger=system_logger, color="yellow")
166
- else:
167
- print_api("DNS Server is disabled.", logger=system_logger, color="yellow")
168
-
169
- if config['tcp']['enable_tcp_server']:
170
- print_api("TCP Server is enabled.", logger=system_logger)
171
-
172
- if engines_list and not config['tcp']['engines_usage']:
173
- message = \
174
- f"Engines found, but the TCP server is set not to use them for processing. General responses only."
175
- print_api(message, color="yellow", logger=system_logger)
176
- elif engines_list and config['tcp']['engines_usage']:
177
- message = f"Engines found, and the TCP server is set to use them for processing."
178
- print_api(message, logger=system_logger)
179
- elif not engines_list and config['tcp']['engines_usage']:
180
- raise ValueError("No engines were found, but the TCP server is set to use them for processing.\n"
181
- "Please check your TCP configuration in the 'config.ini' file.")
182
- else:
183
- print_api("TCP Server is disabled.", logger=system_logger, color="yellow")
184
-
185
- # === EOF Engine Logging =======================================================================================
186
-
187
- # Assigning all the engines domains to all time domains, that will be responsible for adding new domains.
188
- config_static.CONFIG_EXTENDED['certificates']['domains_all_times'] = list(domains_engine_list_full)
189
-
190
- # Creating Statistics logger.
191
- statistics_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
192
- logger_name="statistics", directory_path=config['log']['logs_path'],
193
- file_extension=config_static.CSV_EXTENSION, formatter_message_only=True
194
- )
195
- output_statistics_csv_header()
196
-
197
- network_logger_name = "network"
198
- network_logger = loggingw.get_logger_with_stream_handler_and_timedfilehandler(
199
- logger_name=network_logger_name, directory_path=config['log']['logs_path'], disable_duplicate_ms=True)
200
- system_logger.info(f"Loaded network logger: {network_logger}")
201
-
202
- # Initiate Listener logger, which is a child of network logger, so he uses the same settings and handlers
203
- listener_logger = loggingw.get_logger_with_level(f'{network_logger_name}.listener')
204
- system_logger.info(f"Loaded listener logger: {listener_logger}")
205
-
206
- # Create request domain queue.
207
- domain_queue = queues.NonBlockQueue()
208
-
209
- # === Initialize DNS module ====================================================================================
210
- if config['dns']['enable_dns_server']:
211
- # before executing TCP sockets and after executing 'network' logger.
212
- dns_server = DnsServer(config)
213
- # Passing the engine domain list to DNS server to work with.
214
- # 'list' function re-initializes the current list, or else it will be the same instance object.
215
- dns_server.domain_list = list(domains_engine_list_full)
216
-
217
- dns_server.request_domain_queue = domain_queue
218
- # Initiate the thread.
219
- threading.Thread(target=dns_server.start).start()
220
-
221
- # === EOF Initialize DNS module ================================================================================
222
- # === Initialize TCP Server ====================================================================================
223
- if config['tcp']['enable_tcp_server']:
224
- socket_wrapper = SocketWrapper(
225
- config=dicts_nested.merge(config, config_static.CONFIG_EXTENDED), logger=listener_logger,
226
- statistics_logger=statistics_logger)
227
-
228
- socket_wrapper.create_tcp_listening_socket_list()
229
-
230
- socket_wrapper.requested_domain_from_dns_server = domain_queue
231
-
232
- # General exception handler will catch all the exceptions that occurred in the threads and write it to the log.
233
- try:
234
- socket_wrapper.loop_for_incoming_sockets(function_reference=thread_worker_main, reference_args=(
235
- network_logger, statistics_logger, engines_list, reference_module, config,))
236
- except Exception:
237
- message = f"Unhandled Exception occurred in 'loop_for_incoming_sockets' function"
238
- print_api(message, error_type=True, color="red", logger=network_logger, traceback_string=True, oneline=True)
239
-
240
- # === EOF Initialize TCP Server ================================================================================
@@ -1,44 +0,0 @@
1
- from ...print_api import print_api
2
- from .hash_checks import file, url
3
-
4
-
5
- def _execute_cycle(change_monitor_instance, print_kwargs: dict = None):
6
- """
7
- This function executes the cycle of the change monitor: hash.
8
-
9
- :param change_monitor_instance: Instance of the ChangeMonitor class.
10
-
11
- :return: List of dictionaries with the results of the cycle.
12
- """
13
-
14
- if print_kwargs is None:
15
- print_kwargs = dict()
16
-
17
- return_list = list()
18
- # Loop through all the objects to check.
19
- for check_object_index, check_object in enumerate(change_monitor_instance.check_object_list):
20
- if change_monitor_instance.object_type == 'file':
21
- # Get the hash of the object.
22
- file.get_hash(change_monitor_instance, check_object_index, check_object, print_kwargs=print_kwargs)
23
- elif 'url_' in change_monitor_instance.object_type:
24
- # Get the hash of the object.
25
- url.get_hash(change_monitor_instance, check_object_index, check_object, print_kwargs=print_kwargs)
26
-
27
- if change_monitor_instance.first_cycle:
28
- # Set the input file path.
29
- change_monitor_instance._set_input_file_path(check_object_index=check_object_index)
30
-
31
- # Check if the object was updated.
32
- result, message = change_monitor_instance.diff_check_list[check_object_index].check_string(
33
- print_kwargs=print_kwargs)
34
-
35
- # If the object was updated, print the message in yellow color, otherwise print in green color.
36
- if result:
37
- print_api(message, color='yellow', **print_kwargs)
38
- # create_message_file(message, self.__class__.__name__, logger=self.logger)
39
-
40
- return_list.append(message)
41
- else:
42
- print_api(message, color='green', **print_kwargs)
43
-
44
- return return_list
@@ -1,55 +0,0 @@
1
- from .... import filesystem, hashing
2
-
3
-
4
- def get_hash(change_monitor_instance, check_object_index: int, check_object: str, print_kwargs: dict = None):
5
- """
6
- The function will get the hash of the URL content.
7
-
8
- :param change_monitor_instance: Instance of the ChangeMonitor class.
9
- :param check_object_index: integer, index of the object in the 'check_object_list' list.
10
- :param check_object: string, full URL to a web page.
11
- :param print_kwargs: dict, that contains all the arguments for 'print_api' function.
12
- """
13
-
14
- # If this is the first cycle, we need to set several things.
15
- if change_monitor_instance.first_cycle:
16
- original_name: str = str()
17
-
18
- # If 'generate_input_file_name' is True, or 'store_original_object' is True, we need to create a
19
- # filename without extension.
20
- if change_monitor_instance.store_original_object or change_monitor_instance.generate_input_file_name:
21
- # Get the last directory from the url.
22
- original_name = filesystem.get_file_name(check_object)
23
- # Make characters lower case.
24
- original_name = original_name.lower()
25
-
26
- # If 'store_original_object' is True, then we need to create a filepath to store.
27
- if change_monitor_instance.original_object_directory:
28
- # Add extension to the file name.
29
- original_file_name = original_name
30
-
31
- # Make path for original object.
32
- change_monitor_instance.original_object_file_path = filesystem.add_object_to_path(
33
- change_monitor_instance.original_object_directory, original_file_name)
34
-
35
- if change_monitor_instance.generate_input_file_name:
36
- # Remove dots from the file name.
37
- original_name_no_dots = original_name.replace('.', '-')
38
- # Make path for 'input_file_name'.
39
- change_monitor_instance.input_file_name = f'{original_name_no_dots}.txt'
40
-
41
- # Change settings for the DiffChecker object.
42
- change_monitor_instance.diff_check_list[check_object_index].return_first_cycle = False
43
-
44
- change_monitor_instance.diff_check_list[check_object_index].check_object_display_name = \
45
- f'{original_name}|{change_monitor_instance.object_type}'
46
-
47
- # Copy the file to the original object directory.
48
- if change_monitor_instance.original_object_file_path:
49
- filesystem.copy_file(check_object, change_monitor_instance.original_object_file_path)
50
-
51
- # Get hash of the file.
52
- hash_string = hashing.hash_file(check_object)
53
-
54
- # Set the hash string to the 'check_object' variable.
55
- change_monitor_instance.diff_check_list[check_object_index].check_object = hash_string
@@ -1,62 +0,0 @@
1
- from .... import filesystem, hashing, urls
2
-
3
-
4
- def get_hash(change_monitor_instance, check_object_index: int, check_object: str, print_kwargs: dict = None):
5
- """
6
- The function will get the hash of the URL content.
7
-
8
- :param change_monitor_instance: Instance of the ChangeMonitor class.
9
- :param check_object_index: integer, index of the object in the 'check_object_list' list.
10
- :param check_object: string, full URL to a web page.
11
- :param print_kwargs: dict, that contains all the arguments for 'print_api' function.
12
- """
13
- # Extract the method name from the object type.
14
- get_method = change_monitor_instance.object_type.split('_', 1)[1]
15
-
16
- # If this is the first cycle, we need to set several things.
17
- if change_monitor_instance.first_cycle:
18
- original_name: str = str()
19
-
20
- # If 'generate_input_file_name' is True, or 'store_original_object' is True, we need to create a
21
- # filename without extension.
22
- if change_monitor_instance.store_original_object or change_monitor_instance.generate_input_file_name:
23
- # Get the last directory from the url.
24
- original_name = urls.url_parser(check_object)['directories'][-1]
25
- # Make characters lower case.
26
- original_name = original_name.lower()
27
-
28
- # If 'store_original_object' is True, then we need to create a filepath to store.
29
- extension = None
30
- if change_monitor_instance.original_object_directory:
31
- # Add extension to the file name.
32
- if 'playwright' in get_method:
33
- extension = get_method.split('_')[1]
34
- elif get_method == 'urllib':
35
- extension = 'html'
36
- original_file_name = f'{original_name}.{extension}'
37
-
38
- # Make path for original object.
39
- change_monitor_instance.original_object_file_path = filesystem.add_object_to_path(
40
- change_monitor_instance.original_object_directory, original_file_name)
41
-
42
- if change_monitor_instance.generate_input_file_name:
43
- # Make path for 'input_file_name'.
44
- change_monitor_instance.input_file_name = f'{original_name}.txt'
45
-
46
- # Change settings for the DiffChecker object.
47
- change_monitor_instance.diff_check_list[check_object_index].return_first_cycle = False
48
-
49
- change_monitor_instance.diff_check_list[check_object_index].check_object_display_name = \
50
- f'{original_name}|{change_monitor_instance.object_type}'
51
-
52
- # Get hash of the url. The hash will be different between direct hash of the URL content and the
53
- # hash of the file that was downloaded from the URL. Since the file has headers and other information
54
- # that is not part of the URL content. The Original downloaded file is for reference only to see
55
- # what was the content of the URL at the time of the download.
56
- hash_string = hashing.hash_url(
57
- check_object, get_method=get_method, path=change_monitor_instance.original_object_file_path,
58
- print_kwargs=print_kwargs
59
- )
60
-
61
- # Set the hash string to the 'check_object' variable.
62
- change_monitor_instance.diff_check_list[check_object_index].check_object = hash_string
@@ -1,88 +0,0 @@
1
- # v1.0.1 - 26.03.2023 23:50
2
- import sys
3
- import argparse
4
- from argparse import RawTextHelpFormatter
5
-
6
-
7
- class ArgparseWrapper:
8
- """
9
- # Usage in the main:
10
- args = ArgparseWrapper().parser_arguments
11
- # Defining variables to each argument
12
- input_path: str = args.input
13
- output_path: str = args.output
14
- """
15
-
16
- def __init__(self):
17
- self.application_short: str = 'pbtkMultiFile'
18
- self.application_full: str = 'pbtk Multi File wrapper'
19
- self.version: str = '1.0.0'
20
- self.description: str = 'Find ".proto" files in directory of binaries.'
21
- self.description_full: str = f'{self.application_full} v{self.version}\n' \
22
- f'Description: {self.description}'
23
- self.usage_variable: str = "%(prog)s [-h] -in folder_with_binary_files -out full_path_to_output_files\n" \
24
- "Input or Output path shouldn't end with separator. Example: '\\'."
25
- self.parser_arguments = None
26
-
27
- # Execute argparse.
28
- self.define_argparse()
29
-
30
- # Function to define argument parser
31
- def define_argparse(self):
32
- # Create the parser
33
- # formatter_class=RawTextHelpFormatter: shows raw text and not the default argparse text parsing.
34
- parser = argparse.ArgumentParser(description=self.description_full,
35
- usage=self.usage_variable,
36
- formatter_class=RawTextHelpFormatter)
37
-
38
- # Add arguments
39
- parser.add_argument('-in', '--input',
40
- action='store', type=str, metavar='PATH_TO_FOLDER_WITH_BINARY_FILES',
41
- required=True,
42
- help='Provide full path to folder that contains binary files.')
43
- parser.add_argument('-out', '--output', action='store', type=str, metavar='PATH_TO_SAVE_EXPORTED_FILES',
44
- required=True,
45
- help='Provide full path where you want to store exported file.')
46
-
47
- # A problem before executing 'parse_args()'.
48
- # If we get directory path as argument, on windows we can get a path that ends with backslash:
49
- # C:\Users\user\documents\
50
- # This is the default behaviour of windows when copying path of only the directory.
51
- # When the path contains spaces, we need to pass it with double quotes:
52
- # "C:\Users\user\documents\some folder name\another\"
53
- # When python receives the arguments from CMD they get already parsed, meaning python can do nothing about it.
54
- # From input:
55
- # python_script.py -in "C:\some folder name\another\" -out "C:\some folder name\another1\"
56
- # You will get output:
57
- # ['python_script.py',
58
- # '-in',
59
- # 'C:\some folder name\another" -out C:\some',
60
- # 'folder',
61
- # 'name\another1"]
62
- # 'parse_args()' gets its input from 'sys.argv'. Meaning, you will need to do some manipulations on that
63
- # Before executing the argparse argument parsing.
64
- # Probably the fix should be individual for each case.
65
- # The simplest solution though is to tell the user not to use backslash in the end of directory in case
66
- # of exception.
67
-
68
- try:
69
- # Execute parse_args()
70
- parsed_arguments = parser.parse_args()
71
- # The only thing that you can catch on without modifying Argparse code is 'SystemExit' exception.
72
- # You can also provide just 'except' without anything, which isn't the best practice.
73
- # Another fix would be to use
74
- # argparse.ArgumentParser(exit_on_error=False)
75
- # But as of python 3.10.8 it is not working yet.
76
- except SystemExit as exception_object:
77
- print('======================================')
78
- print('[*] Info: Error in provided arguments.')
79
- print('[*] Tip: Check if you have backslash "\\" in the end of folder path, if so remove it.')
80
- print('======================================')
81
- sys.exit()
82
-
83
- # if the folder path argument in the middle will have backslash "\" it will cause an exception.
84
- # If the backslash will be in the end, it will not cause exception, but the string will end with double quotes.
85
- parsed_arguments.input = parsed_arguments.input.replace('"', '')
86
- parsed_arguments.output = parsed_arguments.output.replace('"', '')
87
-
88
- self.parser_arguments = parsed_arguments
atomicshop/permissions.py DELETED
@@ -1,110 +0,0 @@
1
- import os
2
- import stat
3
- import ctypes
4
- import contextlib
5
- import subprocess
6
-
7
- # Import pwd only on linux.
8
- if os.name == 'posix':
9
- import pwd
10
-
11
-
12
- def is_admin() -> bool:
13
- """
14
- Function checks on Windows or POSIX OSes if the script is executed under Administrative Privileges.
15
- :return: True / False.
16
- """
17
-
18
- if os.name == 'nt':
19
- if ctypes.windll.shell32.IsUserAnAdmin() == 0:
20
- result = False
21
- else:
22
- result = True
23
- else:
24
- if 'SUDO_USER' in os.environ and os.geteuid() == 0:
25
- result = True
26
- else:
27
- result = False
28
-
29
- return result
30
-
31
-
32
- def get_ubuntu_sudo_executer_username() -> str:
33
- """
34
- Function gets the username of the user who executed the script with sudo.
35
- :return: str, username.
36
- """
37
-
38
- if 'SUDO_USER' in os.environ:
39
- return os.environ['SUDO_USER']
40
- else:
41
- return ''
42
-
43
-
44
- def set_executable_permission(file_path: str):
45
- """
46
- Function sets the executable permission on a file.
47
- Equivalent to: chmod +x <file_path>
48
-
49
- :param file_path: str, path to the file.
50
- :return:
51
- """
52
-
53
- # os.chmod(file_path, os.stat(file_path).st_mode | 0o111)
54
- os.chmod(file_path, os.stat(file_path).st_mode | stat.S_IXUSR)
55
-
56
-
57
- def is_executable_permission(file_path: str) -> bool:
58
- """
59
- Function checks if the file has the executable permission.
60
- Equivalent to: stat -c "%a %n" <file_path>
61
-
62
- :param file_path: str, path to the file.
63
- :return: bool, True / False.
64
- """
65
-
66
- return bool(os.stat(file_path).st_mode & stat.S_IXUSR)
67
-
68
-
69
- def run_as_root(command):
70
- subprocess.check_call(['sudo'] + command)
71
-
72
-
73
- @contextlib.contextmanager
74
- def temporary_regular_permissions():
75
- """
76
- This function is used to temporarily change the effective user and group ID to the original user's.
77
- This is used to run commands with the original user's permissions.
78
- If you executed a script with 'sudo' and wanted certain action to execute as regular user and not root.
79
-
80
- Example:
81
- with temporary_regular_permissions():
82
- # Do something with regular permissions.
83
- pass
84
-
85
- :return:
86
- """
87
- # Save the current effective user and group ID
88
- original_euid, original_egid = os.geteuid(), os.getegid()
89
-
90
- try:
91
- # Get the original user's UID and GID
92
- orig_uid = int(os.environ.get('SUDO_UID', os.getuid()))
93
- orig_gid = int(os.environ.get('SUDO_GID', os.getgid()))
94
-
95
- # Set the effective user and group ID to the original user's
96
- os.setegid(orig_gid)
97
- os.seteuid(orig_uid)
98
-
99
- # Provide the context to do something with these permissions
100
- yield
101
- finally:
102
- # Revert to the original effective user and group ID
103
- os.seteuid(original_euid)
104
- os.setegid(original_egid)
105
-
106
-
107
- def expand_user_path(user_name, path):
108
- pwnam = pwd.getpwnam(user_name)
109
- home_dir = pwnam.pw_dir
110
- return path.replace("~", home_dir)