atomicshop 2.11.47__py3-none-any.whl → 3.10.5__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. atomicshop/__init__.py +1 -1
  2. atomicshop/{addons/mains → a_mains}/FACT/update_extract.py +3 -2
  3. atomicshop/a_mains/addons/process_list/compile.cmd +7 -0
  4. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
  5. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
  6. atomicshop/a_mains/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
  7. atomicshop/{addons → a_mains/addons}/process_list/process_list.cpp +8 -1
  8. atomicshop/a_mains/dns_gateway_setting.py +11 -0
  9. atomicshop/a_mains/get_local_tcp_ports.py +85 -0
  10. atomicshop/a_mains/github_wrapper.py +11 -0
  11. atomicshop/a_mains/install_ca_certificate.py +172 -0
  12. atomicshop/{addons/mains → a_mains}/msi_unpacker.py +3 -1
  13. atomicshop/a_mains/process_from_port.py +119 -0
  14. atomicshop/a_mains/set_default_dns_gateway.py +90 -0
  15. atomicshop/a_mains/update_config_toml.py +38 -0
  16. atomicshop/appointment_management.py +5 -3
  17. atomicshop/basics/ansi_escape_codes.py +3 -1
  18. atomicshop/basics/argparse_template.py +2 -0
  19. atomicshop/basics/booleans.py +27 -30
  20. atomicshop/basics/bytes_arrays.py +43 -0
  21. atomicshop/basics/classes.py +149 -1
  22. atomicshop/basics/dicts.py +12 -0
  23. atomicshop/basics/enums.py +2 -2
  24. atomicshop/basics/exceptions.py +5 -1
  25. atomicshop/basics/list_of_classes.py +29 -0
  26. atomicshop/basics/list_of_dicts.py +69 -5
  27. atomicshop/basics/lists.py +14 -0
  28. atomicshop/basics/multiprocesses.py +374 -50
  29. atomicshop/basics/package_module.py +10 -0
  30. atomicshop/basics/strings.py +160 -7
  31. atomicshop/basics/threads.py +14 -0
  32. atomicshop/basics/tracebacks.py +13 -4
  33. atomicshop/certificates.py +153 -52
  34. atomicshop/config_init.py +12 -7
  35. atomicshop/console_user_response.py +7 -14
  36. atomicshop/consoles.py +9 -0
  37. atomicshop/datetimes.py +98 -0
  38. atomicshop/diff_check.py +340 -40
  39. atomicshop/dns.py +128 -12
  40. atomicshop/etws/_pywintrace_fix.py +17 -0
  41. atomicshop/etws/const.py +38 -0
  42. atomicshop/etws/providers.py +21 -0
  43. atomicshop/etws/sessions.py +43 -0
  44. atomicshop/etws/trace.py +168 -0
  45. atomicshop/etws/traces/trace_dns.py +162 -0
  46. atomicshop/etws/traces/trace_sysmon_process_creation.py +126 -0
  47. atomicshop/etws/traces/trace_tcp.py +130 -0
  48. atomicshop/file_io/csvs.py +222 -24
  49. atomicshop/file_io/docxs.py +35 -18
  50. atomicshop/file_io/file_io.py +35 -19
  51. atomicshop/file_io/jsons.py +49 -0
  52. atomicshop/file_io/tomls.py +139 -0
  53. atomicshop/filesystem.py +864 -293
  54. atomicshop/get_process_list.py +133 -0
  55. atomicshop/{process_name_cmd.py → get_process_name_cmd_dll.py} +52 -19
  56. atomicshop/http_parse.py +149 -93
  57. atomicshop/ip_addresses.py +6 -1
  58. atomicshop/mitm/centered_settings.py +132 -0
  59. atomicshop/mitm/config_static.py +207 -0
  60. atomicshop/mitm/config_toml_editor.py +55 -0
  61. atomicshop/mitm/connection_thread_worker.py +875 -357
  62. atomicshop/mitm/engines/__parent/parser___parent.py +4 -17
  63. atomicshop/mitm/engines/__parent/recorder___parent.py +108 -51
  64. atomicshop/mitm/engines/__parent/requester___parent.py +116 -0
  65. atomicshop/mitm/engines/__parent/responder___parent.py +75 -114
  66. atomicshop/mitm/engines/__reference_general/parser___reference_general.py +10 -7
  67. atomicshop/mitm/engines/__reference_general/recorder___reference_general.py +5 -5
  68. atomicshop/mitm/engines/__reference_general/requester___reference_general.py +47 -0
  69. atomicshop/mitm/engines/__reference_general/responder___reference_general.py +95 -13
  70. atomicshop/mitm/engines/create_module_template.py +58 -14
  71. atomicshop/mitm/import_config.py +359 -139
  72. atomicshop/mitm/initialize_engines.py +160 -74
  73. atomicshop/mitm/message.py +64 -23
  74. atomicshop/mitm/mitm_main.py +892 -0
  75. atomicshop/mitm/recs_files.py +183 -0
  76. atomicshop/mitm/shared_functions.py +4 -10
  77. atomicshop/mitm/ssh_tester.py +82 -0
  78. atomicshop/mitm/statistic_analyzer.py +257 -166
  79. atomicshop/mitm/statistic_analyzer_helper/analyzer_helper.py +136 -0
  80. atomicshop/mitm/statistic_analyzer_helper/moving_average_helper.py +525 -0
  81. atomicshop/monitor/change_monitor.py +96 -120
  82. atomicshop/monitor/checks/dns.py +139 -70
  83. atomicshop/monitor/checks/file.py +77 -0
  84. atomicshop/monitor/checks/network.py +81 -77
  85. atomicshop/monitor/checks/process_running.py +33 -34
  86. atomicshop/monitor/checks/url.py +94 -0
  87. atomicshop/networks.py +671 -0
  88. atomicshop/on_exit.py +205 -0
  89. atomicshop/package_mains_processor.py +84 -0
  90. atomicshop/permissions/permissions.py +22 -0
  91. atomicshop/permissions/ubuntu_permissions.py +239 -0
  92. atomicshop/permissions/win_permissions.py +33 -0
  93. atomicshop/print_api.py +24 -41
  94. atomicshop/process.py +63 -17
  95. atomicshop/process_poller/__init__.py +0 -0
  96. atomicshop/process_poller/pollers/__init__.py +0 -0
  97. atomicshop/process_poller/pollers/psutil_pywin32wmi_dll.py +95 -0
  98. atomicshop/process_poller/process_pool.py +207 -0
  99. atomicshop/process_poller/simple_process_pool.py +311 -0
  100. atomicshop/process_poller/tracer_base.py +45 -0
  101. atomicshop/process_poller/tracers/__init__.py +0 -0
  102. atomicshop/process_poller/tracers/event_log.py +46 -0
  103. atomicshop/process_poller/tracers/sysmon_etw.py +68 -0
  104. atomicshop/python_file_patcher.py +1 -1
  105. atomicshop/python_functions.py +27 -75
  106. atomicshop/question_answer_engine.py +2 -2
  107. atomicshop/scheduling.py +24 -5
  108. atomicshop/sound.py +4 -2
  109. atomicshop/speech_recognize.py +8 -0
  110. atomicshop/ssh_remote.py +158 -172
  111. atomicshop/startup/__init__.py +0 -0
  112. atomicshop/startup/win/__init__.py +0 -0
  113. atomicshop/startup/win/startup_folder.py +53 -0
  114. atomicshop/startup/win/task_scheduler.py +119 -0
  115. atomicshop/system_resource_monitor.py +61 -46
  116. atomicshop/system_resources.py +8 -8
  117. atomicshop/tempfiles.py +1 -2
  118. atomicshop/timer.py +30 -11
  119. atomicshop/urls.py +41 -0
  120. atomicshop/venvs.py +28 -0
  121. atomicshop/versioning.py +27 -0
  122. atomicshop/web.py +110 -25
  123. atomicshop/web_apis/__init__.py +0 -0
  124. atomicshop/web_apis/google_custom_search.py +44 -0
  125. atomicshop/web_apis/google_llm.py +188 -0
  126. atomicshop/websocket_parse.py +450 -0
  127. atomicshop/wrappers/certauthw/certauth.py +1 -0
  128. atomicshop/wrappers/cryptographyw.py +29 -8
  129. atomicshop/wrappers/ctyping/etw_winapi/__init__.py +0 -0
  130. atomicshop/wrappers/ctyping/etw_winapi/const.py +335 -0
  131. atomicshop/wrappers/ctyping/etw_winapi/etw_functions.py +393 -0
  132. atomicshop/wrappers/ctyping/file_details_winapi.py +67 -0
  133. atomicshop/wrappers/ctyping/msi_windows_installer/cabs.py +2 -1
  134. atomicshop/wrappers/ctyping/msi_windows_installer/extract_msi_main.py +13 -9
  135. atomicshop/wrappers/ctyping/msi_windows_installer/tables.py +35 -0
  136. atomicshop/wrappers/ctyping/setup_device.py +466 -0
  137. atomicshop/wrappers/ctyping/win_console.py +39 -0
  138. atomicshop/wrappers/dockerw/dockerw.py +113 -2
  139. atomicshop/wrappers/elasticsearchw/config_basic.py +0 -12
  140. atomicshop/wrappers/elasticsearchw/elastic_infra.py +75 -0
  141. atomicshop/wrappers/elasticsearchw/elasticsearchw.py +2 -20
  142. atomicshop/wrappers/factw/get_file_data.py +12 -5
  143. atomicshop/wrappers/factw/install/install_after_restart.py +89 -5
  144. atomicshop/wrappers/factw/install/pre_install_and_install_before_restart.py +20 -14
  145. atomicshop/wrappers/factw/postgresql/firmware.py +4 -6
  146. atomicshop/wrappers/githubw.py +583 -51
  147. atomicshop/wrappers/loggingw/consts.py +49 -0
  148. atomicshop/wrappers/loggingw/filters.py +102 -0
  149. atomicshop/wrappers/loggingw/formatters.py +58 -71
  150. atomicshop/wrappers/loggingw/handlers.py +459 -40
  151. atomicshop/wrappers/loggingw/loggers.py +19 -0
  152. atomicshop/wrappers/loggingw/loggingw.py +1010 -178
  153. atomicshop/wrappers/loggingw/reading.py +344 -19
  154. atomicshop/wrappers/mongodbw/__init__.py +0 -0
  155. atomicshop/wrappers/mongodbw/mongo_infra.py +31 -0
  156. atomicshop/wrappers/mongodbw/mongodbw.py +1432 -0
  157. atomicshop/wrappers/netshw.py +271 -0
  158. atomicshop/wrappers/playwrightw/engine.py +34 -19
  159. atomicshop/wrappers/playwrightw/infra.py +5 -0
  160. atomicshop/wrappers/playwrightw/javascript.py +7 -3
  161. atomicshop/wrappers/playwrightw/keyboard.py +14 -0
  162. atomicshop/wrappers/playwrightw/scenarios.py +172 -5
  163. atomicshop/wrappers/playwrightw/waits.py +9 -7
  164. atomicshop/wrappers/powershell_networking.py +80 -0
  165. atomicshop/wrappers/psutilw/processes.py +81 -0
  166. atomicshop/wrappers/psutilw/psutil_networks.py +85 -0
  167. atomicshop/wrappers/psutilw/psutilw.py +9 -0
  168. atomicshop/wrappers/pyopensslw.py +9 -2
  169. atomicshop/wrappers/pywin32w/__init__.py +0 -0
  170. atomicshop/wrappers/pywin32w/cert_store.py +116 -0
  171. atomicshop/wrappers/pywin32w/console.py +34 -0
  172. atomicshop/wrappers/pywin32w/win_event_log/__init__.py +0 -0
  173. atomicshop/wrappers/pywin32w/win_event_log/fetch.py +174 -0
  174. atomicshop/wrappers/pywin32w/win_event_log/subscribe.py +212 -0
  175. atomicshop/wrappers/pywin32w/win_event_log/subscribes/__init__.py +0 -0
  176. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_create.py +57 -0
  177. atomicshop/wrappers/pywin32w/win_event_log/subscribes/process_terminate.py +49 -0
  178. atomicshop/wrappers/pywin32w/win_event_log/subscribes/schannel_logging.py +97 -0
  179. atomicshop/wrappers/pywin32w/winshell.py +19 -0
  180. atomicshop/wrappers/pywin32w/wmis/__init__.py +0 -0
  181. atomicshop/wrappers/pywin32w/wmis/msft_netipaddress.py +113 -0
  182. atomicshop/wrappers/pywin32w/wmis/win32_networkadapterconfiguration.py +259 -0
  183. atomicshop/wrappers/pywin32w/wmis/win32networkadapter.py +112 -0
  184. atomicshop/wrappers/pywin32w/wmis/wmi_helpers.py +236 -0
  185. atomicshop/wrappers/socketw/accepter.py +21 -7
  186. atomicshop/wrappers/socketw/certificator.py +216 -150
  187. atomicshop/wrappers/socketw/creator.py +190 -50
  188. atomicshop/wrappers/socketw/dns_server.py +500 -173
  189. atomicshop/wrappers/socketw/exception_wrapper.py +45 -52
  190. atomicshop/wrappers/socketw/process_getter.py +86 -0
  191. atomicshop/wrappers/socketw/receiver.py +144 -102
  192. atomicshop/wrappers/socketw/sender.py +65 -35
  193. atomicshop/wrappers/socketw/sni.py +334 -165
  194. atomicshop/wrappers/socketw/socket_base.py +134 -0
  195. atomicshop/wrappers/socketw/socket_client.py +137 -95
  196. atomicshop/wrappers/socketw/socket_server_tester.py +14 -9
  197. atomicshop/wrappers/socketw/socket_wrapper.py +717 -116
  198. atomicshop/wrappers/socketw/ssl_base.py +15 -14
  199. atomicshop/wrappers/socketw/statistics_csv.py +148 -17
  200. atomicshop/wrappers/sysmonw.py +157 -0
  201. atomicshop/wrappers/ubuntu_terminal.py +65 -26
  202. atomicshop/wrappers/win_auditw.py +189 -0
  203. atomicshop/wrappers/winregw/__init__.py +0 -0
  204. atomicshop/wrappers/winregw/winreg_installed_software.py +58 -0
  205. atomicshop/wrappers/winregw/winreg_network.py +232 -0
  206. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/METADATA +31 -49
  207. atomicshop-3.10.5.dist-info/RECORD +306 -0
  208. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/WHEEL +1 -1
  209. atomicshop/_basics_temp.py +0 -101
  210. atomicshop/addons/a_setup_scripts/install_psycopg2_ubuntu.sh +0 -3
  211. atomicshop/addons/a_setup_scripts/install_pywintrace_0.3.cmd +0 -2
  212. atomicshop/addons/mains/install_docker_rootless_ubuntu.py +0 -11
  213. atomicshop/addons/mains/install_docker_ubuntu_main_sudo.py +0 -11
  214. atomicshop/addons/mains/install_elastic_search_and_kibana_ubuntu.py +0 -10
  215. atomicshop/addons/mains/install_wsl_ubuntu_lts_admin.py +0 -9
  216. atomicshop/addons/package_setup/CreateWheel.cmd +0 -7
  217. atomicshop/addons/package_setup/Setup in Edit mode.cmd +0 -6
  218. atomicshop/addons/package_setup/Setup.cmd +0 -7
  219. atomicshop/addons/process_list/compile.cmd +0 -2
  220. atomicshop/addons/process_list/compiled/Win10x64/process_list.dll +0 -0
  221. atomicshop/addons/process_list/compiled/Win10x64/process_list.exp +0 -0
  222. atomicshop/addons/process_list/compiled/Win10x64/process_list.lib +0 -0
  223. atomicshop/archiver/_search_in_zip.py +0 -189
  224. atomicshop/archiver/archiver.py +0 -34
  225. atomicshop/archiver/search_in_archive.py +0 -250
  226. atomicshop/archiver/sevenz_app_w.py +0 -86
  227. atomicshop/archiver/sevenzs.py +0 -44
  228. atomicshop/archiver/zips.py +0 -293
  229. atomicshop/etw/dns_trace.py +0 -118
  230. atomicshop/etw/etw.py +0 -61
  231. atomicshop/file_types.py +0 -24
  232. atomicshop/mitm/engines/create_module_template_example.py +0 -13
  233. atomicshop/mitm/initialize_mitm_server.py +0 -240
  234. atomicshop/monitor/checks/hash.py +0 -44
  235. atomicshop/monitor/checks/hash_checks/file.py +0 -55
  236. atomicshop/monitor/checks/hash_checks/url.py +0 -62
  237. atomicshop/pbtkmultifile_argparse.py +0 -88
  238. atomicshop/permissions.py +0 -110
  239. atomicshop/process_poller.py +0 -237
  240. atomicshop/script_as_string_processor.py +0 -38
  241. atomicshop/ssh_scripts/process_from_ipv4.py +0 -37
  242. atomicshop/ssh_scripts/process_from_port.py +0 -27
  243. atomicshop/wrappers/_process_wrapper_curl.py +0 -27
  244. atomicshop/wrappers/_process_wrapper_tar.py +0 -21
  245. atomicshop/wrappers/dockerw/install_docker.py +0 -209
  246. atomicshop/wrappers/elasticsearchw/infrastructure.py +0 -265
  247. atomicshop/wrappers/elasticsearchw/install_elastic.py +0 -232
  248. atomicshop/wrappers/ffmpegw.py +0 -125
  249. atomicshop/wrappers/loggingw/checks.py +0 -20
  250. atomicshop/wrappers/nodejsw/install_nodejs.py +0 -139
  251. atomicshop/wrappers/process_wrapper_pbtk.py +0 -16
  252. atomicshop/wrappers/socketw/base.py +0 -59
  253. atomicshop/wrappers/socketw/get_process.py +0 -107
  254. atomicshop/wrappers/wslw.py +0 -191
  255. atomicshop-2.11.47.dist-info/RECORD +0 -251
  256. /atomicshop/{addons/mains → a_mains}/FACT/factw_fact_extractor_docker_image_main_sudo.py +0 -0
  257. /atomicshop/{addons → a_mains/addons}/PlayWrightCodegen.cmd +0 -0
  258. /atomicshop/{addons → a_mains/addons}/ScriptExecution.cmd +0 -0
  259. /atomicshop/{addons/mains → a_mains/addons}/inits/init_to_import_all_modules.py +0 -0
  260. /atomicshop/{addons → a_mains/addons}/process_list/ReadMe.txt +0 -0
  261. /atomicshop/{addons/mains → a_mains}/search_for_hyperlinks_in_docx.py +0 -0
  262. /atomicshop/{archiver → etws}/__init__.py +0 -0
  263. /atomicshop/{etw → etws/traces}/__init__.py +0 -0
  264. /atomicshop/{monitor/checks/hash_checks → mitm/statistic_analyzer_helper}/__init__.py +0 -0
  265. /atomicshop/{wrappers/nodejsw → permissions}/__init__.py +0 -0
  266. /atomicshop/wrappers/pywin32w/{wmi_win32process.py → wmis/win32process.py} +0 -0
  267. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info/licenses}/LICENSE.txt +0 -0
  268. {atomicshop-2.11.47.dist-info → atomicshop-3.10.5.dist-info}/top_level.txt +0 -0
@@ -1,265 +0,0 @@
1
- import sys
2
- import time
3
- import requests
4
-
5
- from ...print_api import print_api
6
- from ... import filesystem
7
- from .. import ubuntu_terminal
8
- from . import config_basic
9
-
10
-
11
- def is_elastic_service_running():
12
- return ubuntu_terminal.is_service_running(config_basic.UBUNTU_ELASTIC_SERVICE_NAME, return_false_on_error=False)
13
-
14
-
15
- def enable_elastic_service():
16
- ubuntu_terminal.enable_service(config_basic.UBUNTU_ELASTIC_SERVICE_NAME, sudo=True)
17
-
18
-
19
- def start_elastic_service():
20
- ubuntu_terminal.start_service(config_basic.UBUNTU_ELASTIC_SERVICE_NAME, sudo=True)
21
-
22
-
23
- def is_kibana_service_running():
24
- return ubuntu_terminal.is_service_running(config_basic.UBUNTU_KIBANA_SERVICE_NAME, return_false_on_error=False)
25
-
26
-
27
- def enable_kibana_service():
28
- ubuntu_terminal.enable_service(config_basic.UBUNTU_KIBANA_SERVICE_NAME, sudo=True)
29
-
30
-
31
- def start_kibana_service():
32
- ubuntu_terminal.start_service(config_basic.UBUNTU_KIBANA_SERVICE_NAME, sudo=True)
33
-
34
-
35
- def start_elastic_and_check_service_availability(wait_time_seconds: float = 30, exit_on_error: bool = True):
36
- """
37
- Function starts the Elasticsearch service and checks its availability.
38
- :param wait_time_seconds: float, the time to wait after starting the service before checking
39
- the service availability.
40
- :param exit_on_error: bool, if True, the function will exit the program if the service is not available.
41
- :return:
42
- """
43
-
44
- # Start, enable and check the Elasticsearch service.
45
- ubuntu_terminal.start_enable_service_check_availability(
46
- service_name=config_basic.UBUNTU_ELASTIC_SERVICE_NAME,
47
- wait_time_seconds=wait_time_seconds,
48
- exit_on_error=exit_on_error
49
- )
50
-
51
- # Check if Elasticsearch is running.
52
- if not is_server_available():
53
- if exit_on_error:
54
- sys.exit(1)
55
-
56
-
57
- def start_kibana_and_check_service_availability(wait_time_seconds: float = 30, exit_on_error: bool = True):
58
- """
59
- Function starts the Kibana service and checks its availability.
60
- :param wait_time_seconds: float, the time to wait after starting the service before checking
61
- the service availability.
62
- :param exit_on_error: bool, if True, the function will exit the program if the service is not available.
63
- :return:
64
- """
65
-
66
- # Start, enable and check the Elasticsearch service.
67
- ubuntu_terminal.start_enable_service_check_availability(
68
- service_name=config_basic.UBUNTU_KIBANA_SERVICE_NAME,
69
- wait_time_seconds=wait_time_seconds,
70
- exit_on_error=exit_on_error
71
- )
72
-
73
-
74
- def is_elastic_config_file_exists(
75
- config_file_path: str = None,
76
- exit_on_error: bool = False,
77
- output_message: bool = False
78
- ) -> bool:
79
- """
80
- The function checks if the Elasticsearch configuration file exists.
81
-
82
- :param config_file_path: str, the path to the configuration file.
83
- :param exit_on_error: bool, if True, the function will exit the program if the file does not exist.
84
- :param output_message: bool, if True, the function will print a message if the file does not exist.
85
- :return:
86
- """
87
-
88
- if not config_file_path:
89
- config_file_path = config_basic.ELASTIC_CONFIG_FILE
90
-
91
- # if not ubuntu_terminal.is_sudo_file_exists(config_file_path):
92
- if not filesystem.check_file_existence(config_file_path):
93
- if output_message:
94
- message = f"Configuration file does not exist at {config_file_path}."
95
- print_api(message, color='red', error_type=True)
96
- if exit_on_error:
97
- sys.exit(1)
98
- return False
99
- else:
100
- return True
101
-
102
-
103
- def check_xpack_security_setting(config_file_path: str = None):
104
- """
105
- The function checks if the 'xpack.security.enabled' setting is set to 'false' in the Elasticsearch
106
- configuration file.
107
-
108
- :param config_file_path:
109
- :return:
110
- """
111
-
112
- if not config_file_path:
113
- config_file_path = config_basic.ELASTIC_CONFIG_FILE
114
-
115
- with open(config_file_path, 'r') as file:
116
- # Read the file contents
117
- contents = file.read()
118
- # Check if the specific setting exists
119
- if f"{config_basic.XPACK_SECURITY_SETTING_NAME}: false" in contents:
120
- return False
121
- elif f"{config_basic.XPACK_SECURITY_SETTING_NAME}: true" in contents:
122
- return True
123
- # If the setting doesn't exist, return None.
124
- else:
125
- return None
126
-
127
-
128
- def modify_xpack_security_setting(
129
- config_file_path: str = None,
130
- setting: bool = False,
131
- output_message: bool = True
132
- ):
133
- """
134
- The function modifies the 'xpack.security.enabled' setting in the Elasticsearch configuration file.
135
- :param config_file_path: str, the path to the configuration file.
136
- :param setting: bool, the setting to change to. Will be added, if doesn't exist.
137
- :param output_message: bool, if True, the function will print a message.
138
- :return:
139
- """
140
-
141
- if not config_file_path:
142
- config_file_path = config_basic.ELASTIC_CONFIG_FILE
143
-
144
- # The setting to set in the configuration file.
145
- xpack_setting_to_set: str = f'{config_basic.XPACK_SECURITY_SETTING_NAME}: {str(setting).lower()}'
146
-
147
- # Check if the setting exists in the configuration file and get its value.
148
- current_xpack_security_setting = check_xpack_security_setting(config_file_path)
149
-
150
- # If the setting doesn't exist, add it to the configuration file.
151
- if current_xpack_security_setting is None:
152
- with open(config_file_path, 'a') as file:
153
- file.write(f'{xpack_setting_to_set}\n')
154
- if output_message:
155
- print_api(f"Added [{xpack_setting_to_set}] to the configuration.")
156
- # If the setting exists and is different from the desired setting, change it.
157
- elif current_xpack_security_setting != setting:
158
- with open(config_file_path, 'r') as file:
159
- lines = file.readlines()
160
- with open(config_file_path, 'w') as file:
161
- for line in lines:
162
- if f"{config_basic.XPACK_SECURITY_SETTING_NAME}:" in line:
163
- file.write(f'{xpack_setting_to_set}\n')
164
- else:
165
- file.write(line)
166
- if output_message:
167
- print_api(f"Changed [{config_basic.XPACK_SECURITY_SETTING_NAME}] to [{setting}].")
168
- # If the setting is already set to the desired value, print a message.
169
- elif current_xpack_security_setting == setting:
170
- if output_message:
171
- print_api(f"The setting is already set to [{setting}].")
172
-
173
-
174
- def create_jvm_options_custom_file(file_path: str, options: list):
175
- """
176
- The function creates a custom JVM options file for Elasticsearch.
177
- You can use the default directory path as 'config_basic.ELASTIC_JVM_OPTIONS_DIRECTORY'.
178
- :param file_path: str, the path to the custom JVM options file.
179
- :param options: list, the list of JVM options.
180
- :return:
181
- """
182
-
183
- # Write the options to the file.
184
- with open(file_path, 'w') as file:
185
- for option in options:
186
- file.write(f"{option}\n")
187
-
188
-
189
- def create_jvm_options_custom_4gb_memory_heap_file(file_path: str = None):
190
- """
191
- The function creates a custom JVM options file with 4GB memory heap usage.
192
- The 4GB memory usage options are needed for the Elasticsearch to work properly and not to crash.
193
- :param file_path: str, the path to the custom JVM options file.
194
- :return:
195
- """
196
-
197
- if not file_path:
198
- file_path = config_basic.ELASTIC_JVM_OPTIONS_4GB_CUSTOM_FILE
199
-
200
- create_jvm_options_custom_file(file_path, config_basic.ELASTIC_JVM_OPTIONS_4GB_MEMORY_USAGE)
201
-
202
-
203
- def is_server_available(
204
- max_attempts: int = 5,
205
- wait_between_attempts_seconds: float = 10,
206
- elastic_url: str = None,
207
- print_kwargs: dict = None
208
- ):
209
- """
210
- The function checks if Elasticsearch server is up and running by sending GET request to the Elasticsearch server.
211
- :param max_attempts: int, the maximum number of attempts to check if Elasticsearch is running.
212
- :param wait_between_attempts_seconds: float, the time to wait between attempts.
213
- :param elastic_url: str, the URL of the Elasticsearch server. If None, the default URL will be used.
214
- :param print_kwargs: dict, the keyword arguments for the print_api function.
215
- :return:
216
- """
217
-
218
- if not elastic_url:
219
- elastic_url = config_basic.DEFAULT_ELASTIC_URL
220
-
221
- if not print_kwargs:
222
- print_kwargs = dict()
223
-
224
- for attempt in range(1, max_attempts + 1):
225
- print_api(f"Checking if Elasticsearch is running (Attempt {attempt}/{max_attempts})...", **print_kwargs)
226
-
227
- try:
228
- response = requests.get(elastic_url)
229
- status_code = response.status_code
230
-
231
- if status_code == 200:
232
- print_api("Elasticsearch is up and running.", color='green', **print_kwargs)
233
- return True
234
- else:
235
- print_api(f"Elasticsearch is not running. Status code: {status_code}", color='yellow', **print_kwargs)
236
- except requests.exceptions.RequestException as e:
237
- print_api(f"Failed to connect to Elasticsearch: {e}", color='yellow', **print_kwargs)
238
-
239
- print_api("Waiting for Elasticsearch to start...", **print_kwargs)
240
- time.sleep(wait_between_attempts_seconds)
241
-
242
- print_api("Elasticsearch did not start within the expected time.", color='red', **print_kwargs)
243
- return False
244
-
245
-
246
- def is_4gb_memory_heap_options_applied_on_server() -> bool:
247
- """
248
- The function checks if the 4GB memory heap options are applied on the Elasticsearch server.
249
- :return: bool.
250
- """
251
-
252
- # Send a GET request
253
- response = requests.get(config_basic.DEFAULT_ELASTIC_URL_JVM_OPTIONS)
254
- response.raise_for_status() # Raise an exception for HTTP errors
255
-
256
- # Load JSON data from the response
257
- jvm_data = response.json()
258
-
259
- # Check if memory heap options are applied in 'input_arguments' key.
260
- for node in jvm_data['nodes'].values():
261
- # Get the JVM input arguments values.
262
- input_arguments = node['jvm']['input_arguments']
263
-
264
- # Check that the 4GB memory heap options are applied.
265
- return all(options in input_arguments for options in config_basic.ELASTIC_JVM_OPTIONS_4GB_MEMORY_USAGE)
@@ -1,232 +0,0 @@
1
- import sys
2
-
3
- from ...print_api import print_api
4
- from ... import process, permissions
5
- from .. import ubuntu_terminal
6
- from . import config_basic, infrastructure
7
-
8
-
9
- def install_elastic_kibana_ubuntu(install_elastic: bool = True, install_kibana: bool = True):
10
- """
11
- The function will install docker on ubuntu.
12
-
13
- :param install_elastic: bool, if True, install Elasticsearch.
14
- :param install_kibana: bool, if True, install Kibana.
15
-
16
- Usage in main.py (run with sudo):
17
- from atomicshop.wrappers.elasticw import install_elastic
18
-
19
-
20
- def main():
21
- install_elastic.install_elastic_ubuntu()
22
-
23
-
24
- if __name__ == '__main__':
25
- main()
26
- """
27
-
28
- # This is pure bash script.
29
- """
30
- #!/bin/bash
31
-
32
- # Color text in red.
33
- echo_red() {
34
- local color="\e[31m" # Red color
35
- local reset="\e[0m" # Reset formatting
36
- echo -e "${color}$1${reset}"
37
- }
38
-
39
- # Function to check if a service is running
40
- check_service_running() {
41
- local service_name=$1
42
- local status=$(systemctl is-active "$service_name")
43
-
44
- if [ "$status" == "active" ]; then
45
- echo "$service_name service is active and running."
46
- return 0
47
- else
48
- echo_red "$service_name service is not running or has failed. Status: $service_status, Failed: $service_failed"
49
- return 1
50
- fi
51
- }
52
-
53
- # Update and upgrade system packages
54
- sudo apt-get update && sudo apt-get upgrade -y
55
-
56
- # Install necessary dependencies
57
- sudo apt-get install apt-transport-https openjdk-11-jdk wget -y
58
-
59
- # Download and install the GPG signing key
60
- wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor | sudo tee /usr/share/keyrings/elasticsearch-keyring.gpg > /dev/null
61
-
62
- # Add the Elastic repository to the system
63
- echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
64
-
65
- # Update package index
66
- sudo apt-get update
67
-
68
- # Install Elasticsearch
69
- sudo apt-get install elasticsearch -y
70
-
71
- # Path to the Elasticsearch configuration file
72
- CONFIG_FILE="/etc/elasticsearch/elasticsearch.yml"
73
-
74
- # Check if the configuration file exists
75
- if [ ! -f "$CONFIG_FILE" ]; then
76
- echo "Configuration file does not exist at $CONFIG_FILE."
77
- exit 1
78
- fi
79
-
80
- # Function to check the setting in the configuration file
81
- check_setting() {
82
- if grep -q "^xpack.security.enabled: false" "$CONFIG_FILE"; then
83
- echo "The setting is confirmed to be 'xpack.security.enabled: false'."
84
- else
85
- echo "Failed to set 'xpack.security.enabled: false'."
86
- exit 1
87
- fi
88
- }
89
-
90
- # Check if 'xpack.security.enabled' is set to 'false'
91
- if grep -q "^xpack.security.enabled: false" "$CONFIG_FILE"; then
92
- echo "The setting is already set to false."
93
- elif grep -q "^xpack.security.enabled: true" "$CONFIG_FILE"; then
94
- # If the setting is true, change it to false
95
- sudo sed -i 's/^xpack.security.enabled: true/xpack.security.enabled: false/' "$CONFIG_FILE"
96
- echo "Changed xpack.security.enabled to false."
97
- check_setting
98
- else
99
- # If the setting doesn't exist, add it
100
- echo "xpack.security.enabled: false" | sudo tee -a "$CONFIG_FILE" > /dev/null
101
- echo "Added xpack.security.enabled: false to the configuration."
102
- check_setting
103
- fi
104
-
105
- # Start and enable Elasticsearch service
106
- sudo systemctl start elasticsearch
107
- sudo systemctl enable elasticsearch
108
-
109
- echo "Waiting 30 seconds for program to start before availability check..."
110
- sleep 30
111
-
112
- # Check if Elasticsearch service is running
113
- if ! check_service_running "elasticsearch"; then
114
- echo "Elasticsearch service failed to start. Exiting."
115
- exit 1
116
- fi
117
-
118
- # Function to check if Elasticsearch is up and running
119
- check_elasticsearch() {
120
- max_attempts=5
121
- wait_seconds=10
122
-
123
- for ((i=1; i<=max_attempts; i++)); do
124
- echo "Checking if Elasticsearch is running (Attempt $i/$max_attempts)..."
125
-
126
- # Using curl to get the HTTP status code
127
- status=$(curl --write-out %{http_code} --silent --output /dev/null http://localhost:9200)
128
-
129
- if [ "$status" -eq 200 ]; then
130
- echo "Elasticsearch is up and running."
131
- return 0
132
- else
133
- echo "Elasticsearch is not running. Status code: $status"
134
- fi
135
-
136
- echo "Waiting for Elasticsearch to start..."
137
- sleep $wait_seconds
138
- done
139
-
140
- echo "Elasticsearch did not start within the expected time."
141
- return 1
142
- }
143
-
144
- # Check if Elasticsearch is running
145
- if ! check_elasticsearch; then
146
- echo "Elasticsearch failed to start. Exiting."
147
- exit 1
148
- fi
149
-
150
- # Install Kibana
151
- sudo apt-get install kibana -y
152
-
153
- # Start and enable Kibana service
154
- sudo systemctl start kibana
155
- sudo systemctl enable kibana
156
-
157
- echo "Waiting 30 seconds for program to start before availability check..."
158
- sleep 30
159
-
160
- # Check if Kibana service is running
161
- if ! check_service_running "kibana"; then
162
- echo "Kibana service failed to start. Exiting."
163
- exit 1
164
- fi
165
-
166
- # Print status
167
- echo "Elasticsearch and Kibana installation completed."
168
- echo "Elasticsearch is running on http://localhost:9200"
169
- echo "Kibana is running on http://localhost:5601"
170
- """
171
-
172
- if not install_elastic and not install_kibana:
173
- raise ValueError("At least one of the services (Elasticsearch or Kibana) must be installed.")
174
-
175
- # Update and upgrade system packages.
176
- ubuntu_terminal.update_system_packages()
177
- ubuntu_terminal.upgrade_system_packages()
178
-
179
- # Install necessary dependencies.
180
- ubuntu_terminal.install_packages(config_basic.UBUNTU_DEPENDENCY_PACKAGES)
181
-
182
- # Install the GPG key and add elastic repository.
183
- script = f"""
184
- # Download and install the GPG signing key
185
- wget -qO - https://artifacts.elastic.co/GPG-KEY-elasticsearch | gpg --dearmor | sudo tee /usr/share/keyrings/elasticsearch-keyring.gpg > /dev/null
186
-
187
- # Add the Elastic repository to the system
188
- echo "deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/8.x/apt stable main" | sudo tee /etc/apt/sources.list.d/elastic-8.x.list
189
- """
190
- process.execute_script(script, shell=True)
191
-
192
- # Update system with elastic search packages.
193
- ubuntu_terminal.update_system_packages()
194
-
195
- if install_elastic:
196
- # Install Elasticsearch.
197
- ubuntu_terminal.install_packages([config_basic.UBUNTU_ELASTIC_PACKAGE_NAME])
198
-
199
- if not permissions.is_admin():
200
- print_api("This script requires root privileges...", color='red')
201
- sys.exit(1)
202
-
203
- # Check if the configuration file exists.
204
- infrastructure.is_elastic_config_file_exists(exit_on_error=True, output_message=True)
205
-
206
- # Check if the specific setting exists or not and set it to false.
207
- infrastructure.modify_xpack_security_setting(setting=False, output_message=True)
208
-
209
- # Check if the setting was really set to false.
210
- if infrastructure.check_xpack_security_setting() is False:
211
- print_api(f"The setting is confirmed to be [{config_basic.XPACK_SECURITY_SETTING_NAME}: false].")
212
- else:
213
- print_api(f"Failed to set [{config_basic.XPACK_SECURITY_SETTING_NAME}: false].")
214
- sys.exit(1)
215
-
216
- infrastructure.start_elastic_and_check_service_availability()
217
-
218
- print_api("Creating custom JVM options file with 4GB memory usage.")
219
- infrastructure.create_jvm_options_custom_4gb_memory_heap_file()
220
-
221
- if install_kibana:
222
- # Install Kibana.
223
- ubuntu_terminal.install_packages([config_basic.UBUNTU_KIBANA_PACKAGE_NAME])
224
-
225
- # Start and enable Kibana service.
226
- infrastructure.start_kibana_and_check_service_availability()
227
-
228
- print_api("Installation completed.", color='green')
229
- if install_elastic:
230
- print_api(f"Default Elasticsearch on {config_basic.DEFAULT_ELASTIC_URL}")
231
- if install_kibana:
232
- print_api(f"Default Kibana on {config_basic.DEFAULT_KIBANA_URL}")
@@ -1,125 +0,0 @@
1
- import fnmatch
2
- import os
3
- import shlex
4
-
5
- from ..print_api import print_api
6
- from ..tempfiles import TempFile
7
- from .githubw import GitHubWrapper
8
- from ..process import execute_with_live_output
9
- from ..filesystem import create_directory
10
-
11
-
12
- class FFmpegWrapper:
13
- def __init__(self, working_directory: str = str(), ffmpeg_exe_path: str = str()):
14
- self.working_directory: str = working_directory
15
- self.ffmpeg_exe_path: str = ffmpeg_exe_path
16
- self.relative_path: str = 'bin'
17
- self.ffmpeg_exe_name: str = 'ffmpeg.exe'
18
-
19
- # Variables to download the latest release from GitHub in case execution fails.
20
- self.git_user: str = 'GyanD'
21
- self.git_repo: str = 'codexffmpeg'
22
- self.git_latest_release_pattern: str = '*full_build.zip'
23
-
24
- # ffmpeg Release directory name.
25
- self.ffmpeg_release_directory_name: str = \
26
- f"ffmpeg_{self.git_latest_release_pattern.replace('*', '').split('.')[0]}"
27
- self.ffmpeg_release_directory_path: str = str()
28
-
29
- # Execute functions.
30
- self.build_ffmpeg_exe_and_directory_path()
31
-
32
- def build_ffmpeg_exe_and_directory_path(self):
33
- # If working directory was specified, but not the full path to exe.
34
- if self.working_directory and not self.ffmpeg_exe_path:
35
- self.ffmpeg_release_directory_path = self.working_directory + os.sep + self.ffmpeg_release_directory_name
36
- self.ffmpeg_exe_path = \
37
- self.ffmpeg_release_directory_path + os.sep + self.relative_path + os.sep + self.ffmpeg_exe_name
38
- # If no working directory and no path to exe was specified.
39
- elif not self.working_directory and not self.ffmpeg_exe_path:
40
- # Use just 'ffmpeg.exe' as path to exe. Maybe it is already installed and is in environment PATH var.
41
- self.ffmpeg_exe_path = self.ffmpeg_exe_name
42
-
43
- def change_to_temp_directory(self):
44
- temp_file = TempFile()
45
- self.working_directory = temp_file.directory
46
- self.ffmpeg_exe_path = str()
47
- self.build_ffmpeg_exe_and_directory_path()
48
-
49
- def download_ffmpeg_and_extract(self):
50
- github_wrapper = GitHubWrapper(user_name=self.git_user, repo_name=self.git_repo)
51
- github_wrapper.build_links_from_user_and_repo()
52
- github_wrapper.download_and_extract_latest_release(
53
- target_directory=self.ffmpeg_release_directory_path, string_pattern=self.git_latest_release_pattern,
54
- archive_remove_first_directory=True)
55
-
56
- def execute_ffmpeg(self, cmd_list: list):
57
- continue_loop: bool = True
58
- while continue_loop:
59
- # If first entry contains 'ffmpeg.exe' and it is not 'self.ffmpeg_exe_path' already.
60
- if self.ffmpeg_exe_name in cmd_list[0] and cmd_list[0] != self.ffmpeg_exe_path:
61
- # We'll change it to the updated one.
62
- cmd_list[0] = self.ffmpeg_exe_path
63
- # If first entry doesn't contain 'ffmpeg.exe'.
64
- elif self.ffmpeg_exe_name not in cmd_list[0]:
65
- # We'll insert the current path into first entry.
66
- cmd_list.insert(0, self.ffmpeg_exe_path)
67
-
68
- output_strings: list = [
69
- 'Input',
70
- 'Output',
71
- 'video:'
72
- ]
73
-
74
- try:
75
- print(f'FFmpeg processing: {shlex.join(cmd_list)}')
76
- result_lines = execute_with_live_output(
77
- cmd=cmd_list, output_strings=output_strings, raise_exception=True, exit_on_error=False)
78
- # If 'ffmpeg.exe' is non-existent.
79
- except FileNotFoundError:
80
- # Check if full path to 'ffmpeg.exe' is just 'ffmpeg.exe'.
81
- if self.ffmpeg_exe_path == self.ffmpeg_exe_name:
82
- print_api('Will try temp folder...', raise_exception=False)
83
- # Change to temp folder and try executing again.
84
- self.change_to_temp_directory()
85
- continue
86
-
87
- print_api('Trying to download...', raise_exception=False)
88
- create_directory(self.ffmpeg_release_directory_path)
89
- self.download_ffmpeg_and_extract()
90
- continue
91
-
92
- # === At this point python exceptions are finished. ==========================
93
- # If 'Invalid argument' was returned by 'ffmpeg' in the last line.
94
- if 'Invalid argument' in result_lines[-1]:
95
- print_api(result_lines[-1], message_type_error=True, color="red", exit_on_error=True)
96
-
97
- # === Successful execution section ==========================================
98
- string_pattern = 'video:*audio:*subtitle*'
99
- # if 'video:' in result_lines[-1] and 'audio:' in result_lines[-1] and 'subtitle:' in result_lines[-1]:
100
- if fnmatch.fnmatch(result_lines[-1], string_pattern):
101
- print_api(f'FFmpeg finished successfully.', color="green")
102
-
103
- # Since exceptions are finished, we can stop the while loop.
104
- continue_loop = False
105
-
106
- def convert_file(self, source_file_path: str, dest_file_path: str, overwrite: bool = False) -> None:
107
- """
108
- The function converts source file to destination file. The source format is defined by the file extension
109
- as well as destination file format defined by the destination file extension.
110
-
111
- Example convert MP3 file to WAV:
112
- convert_file(source_file_path=some_music.mp3, dest_file_path=converted_file.wav)
113
-
114
- :param source_file_path: string, full file path to source file.
115
- :param dest_file_path: string, full file path to destination file.
116
- :param overwrite: boolean, set if destination file should be overwritten if it exists.
117
- :return: None.
118
- """
119
-
120
- cmd_list = ['-i', source_file_path, dest_file_path]
121
-
122
- if overwrite:
123
- cmd_list.append('-y')
124
-
125
- self.execute_ffmpeg(cmd_list)
@@ -1,20 +0,0 @@
1
- import logging
2
-
3
-
4
- def has_handlers(logger: logging.Logger) -> bool:
5
- """
6
- Function to check if the logger has handlers.
7
- :param logger: Logger to check
8
- :return: True if logger has handlers, False otherwise
9
- """
10
-
11
- # Omitted the usage of "hasHandlers()" method, since sometimes returned "True" even when there were no handlers
12
- # Didn't research the issue much, just used the "len(logger.handlers)" to check how many handlers there are
13
- # in the logger.
14
- # if not logging.getLogger(function_module_name).hasHandlers():
15
- # if len(logging.getLogger(function_module_name).handlers) == 0:
16
-
17
- if len(logger.handlers) == 0:
18
- return False
19
- else:
20
- return True