LinuxMonitor 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,2151 @@
|
|
|
1
|
+
|
|
2
|
+
"""
|
|
3
|
+
Linux Monitor Library: A library to monitor a Linux server and send the results to 'anything' (or to console if using only the library).
|
|
4
|
+
|
|
5
|
+
Non exhaustive list of features (available by using it in shell or in python script):
|
|
6
|
+
- Do all checks bellow in a scheduled tasks and display the results only if there is an issue (only in console if using only the library)
|
|
7
|
+
- Do all checks bellow in a scheduled tasks and display the results every time (only in console if using only the library)
|
|
8
|
+
|
|
9
|
+
- Check CPU, RAM, SWAP, Temperature
|
|
10
|
+
- Check disk usage
|
|
11
|
+
- Check folder usage
|
|
12
|
+
- Check websites basic availability (ping)
|
|
13
|
+
- Check services status and restart them if needed
|
|
14
|
+
- Check certificates expiration and validity
|
|
15
|
+
- Check last user connections IPs
|
|
16
|
+
|
|
17
|
+
- Get hostname, OS details, kernel version, server datetime, uptime
|
|
18
|
+
- Get connected users
|
|
19
|
+
|
|
20
|
+
- Get processes list (PID and name)
|
|
21
|
+
- Kill a process by PID
|
|
22
|
+
|
|
23
|
+
- Reboot server
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
__author__ = 'Quentin Comte-Gaz'
|
|
27
|
+
__email__ = "quentin@comte-gaz.com"
|
|
28
|
+
__license__ = "MIT License"
|
|
29
|
+
__copyright__ = "Copyright Quentin Comte-Gaz (2024)"
|
|
30
|
+
__python_version__ = "3.+"
|
|
31
|
+
__version__ = "1.0 (2024/09/12)"
|
|
32
|
+
__status__ = "Usable for any Linux project"
|
|
33
|
+
|
|
34
|
+
import json
|
|
35
|
+
import subprocess
|
|
36
|
+
import time
|
|
37
|
+
import shutil
|
|
38
|
+
from typing import Callable, Optional, Union, Dict, Set, List, Tuple, Awaitable, Any
|
|
39
|
+
import psutil
|
|
40
|
+
import os
|
|
41
|
+
import ssl
|
|
42
|
+
import socket
|
|
43
|
+
from datetime import datetime, timedelta
|
|
44
|
+
import platform
|
|
45
|
+
import re
|
|
46
|
+
import logging
|
|
47
|
+
import asyncio
|
|
48
|
+
import argparse
|
|
49
|
+
import sys
|
|
50
|
+
|
|
51
|
+
class LinuxMonitor:
|
|
52
|
+
def __init__(self, config_file: str, allow_scheduled_tasks_check_for_issues: bool, allow_scheduled_task_show_info: bool) -> None:
|
|
53
|
+
"""
|
|
54
|
+
Linux Monitor class to monitor a Linux server.
|
|
55
|
+
|
|
56
|
+
:param config_file: The path to the JSON configuration file.
|
|
57
|
+
:param allow_scheduled_tasks_check_for_issues: Allow scheduled tasks to check for issues (because it is the python script that will start the scheduled tasks).
|
|
58
|
+
:param allow_scheduled_task_show_info: Allow scheduled tasks to show info (because it is the python script that will start the scheduled tasks).
|
|
59
|
+
|
|
60
|
+
:raises ValueError: If the configuration file is incorrect.
|
|
61
|
+
"""
|
|
62
|
+
logging.debug(msg=f"Loading configuration file {config_file}...")
|
|
63
|
+
with open(file=config_file, mode='r') as file:
|
|
64
|
+
self.config = json.load(file)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
self.allow_scheduled_tasks_check_for_issues: bool = allow_scheduled_tasks_check_for_issues
|
|
68
|
+
self.allow_scheduled_task_show_info: bool = allow_scheduled_task_show_info
|
|
69
|
+
|
|
70
|
+
# Check if the configuration is correct
|
|
71
|
+
self._init_and_check_configuration()
|
|
72
|
+
|
|
73
|
+
def _init_and_check_configuration(self) -> None:
|
|
74
|
+
"""
|
|
75
|
+
Check if the JSON configuration file is correct.
|
|
76
|
+
"""
|
|
77
|
+
# Check if the configuration is a dictionary
|
|
78
|
+
if not isinstance(self.config, dict):
|
|
79
|
+
raise ValueError("The configuration must be a dictionary")
|
|
80
|
+
|
|
81
|
+
# Check if the configuration contains the necessary keys
|
|
82
|
+
if 'basic_config' not in self.config: # type: ignore
|
|
83
|
+
raise ValueError("The configuration must contain the 'basic_config' key")
|
|
84
|
+
if 'scheduled_tasks_check_for_issues' not in self.config: # type: ignore
|
|
85
|
+
raise ValueError("The configuration must contain the 'scheduled_tasks_check_for_issues' key")
|
|
86
|
+
if 'disks' not in self.config: # type: ignore
|
|
87
|
+
raise ValueError("The configuration must contain the 'disks' key")
|
|
88
|
+
if 'folders' not in self.config: # type: ignore
|
|
89
|
+
raise ValueError("The configuration must contain the 'folders' key")
|
|
90
|
+
if 'pings' not in self.config: # type: ignore
|
|
91
|
+
raise ValueError("The configuration must contain the 'pings' key")
|
|
92
|
+
if 'services' not in self.config: # type: ignore
|
|
93
|
+
raise ValueError("The configuration must contain the 'services' key")
|
|
94
|
+
if 'certificates' not in self.config: # type: ignore
|
|
95
|
+
raise ValueError("The configuration must contain the 'certificates' key")
|
|
96
|
+
|
|
97
|
+
# Get the basic configuration (check if it is a dictionary)
|
|
98
|
+
basic_config: Dict[str, Any] = self.config.get('basic_config', {}) # type: ignore
|
|
99
|
+
if not isinstance(basic_config, dict):
|
|
100
|
+
raise ValueError("The basic configuration must be a dictionary")
|
|
101
|
+
|
|
102
|
+
if 'warning_cpu_percent' not in basic_config:
|
|
103
|
+
raise ValueError("The basic configuration must contain the 'warning_cpu_percent' key")
|
|
104
|
+
self.warning_cpu_percent: float = basic_config.get('warning_cpu_percent') # type: ignore
|
|
105
|
+
|
|
106
|
+
if 'critical_cpu_percent' not in basic_config:
|
|
107
|
+
raise ValueError("The basic configuration must contain the 'critical_cpu_percent' key")
|
|
108
|
+
self.critical_cpu_percent: float = basic_config.get('critical_cpu_percent') # type: ignore
|
|
109
|
+
|
|
110
|
+
if 'warning_ram_percent' not in basic_config:
|
|
111
|
+
raise ValueError("The basic configuration must contain the 'warning_ram_percent' key")
|
|
112
|
+
self.warning_ram_percent: float = basic_config.get('warning_ram_percent') # type: ignore
|
|
113
|
+
|
|
114
|
+
if 'critical_ram_percent' not in basic_config:
|
|
115
|
+
raise ValueError("The basic configuration must contain the 'critical_ram_percent' key")
|
|
116
|
+
self.critical_ram_percent: float = basic_config.get('critical_ram_percent') # type: ignore
|
|
117
|
+
|
|
118
|
+
if 'warning_swap_percent' not in basic_config:
|
|
119
|
+
raise ValueError("The basic configuration must contain the 'warning_swap_percent' key")
|
|
120
|
+
self.warning_swap_percent: float = basic_config.get('warning_swap_percent') # type: ignore
|
|
121
|
+
|
|
122
|
+
if 'critical_swap_percent' not in basic_config:
|
|
123
|
+
raise ValueError("The basic configuration must contain the 'critical_swap_percent' key")
|
|
124
|
+
self.critical_swap_percent: float = basic_config.get('critical_swap_percent') # type: ignore
|
|
125
|
+
|
|
126
|
+
if 'warning_temperature_celsius' not in basic_config:
|
|
127
|
+
raise ValueError("The basic configuration must contain the 'warning_temperature_celsius' key")
|
|
128
|
+
self.warning_temperature_celsius: float = basic_config.get('warning_temperature_celsius') # type: ignore
|
|
129
|
+
|
|
130
|
+
if 'critical_temperature_celsius' not in basic_config:
|
|
131
|
+
raise ValueError("The basic configuration must contain the 'critical_temperature_celsius' key")
|
|
132
|
+
self.critical_temperature_celsius: float = basic_config.get('critical_temperature_celsius') # type: ignore
|
|
133
|
+
|
|
134
|
+
# Get the scheduled tasks for issues configuration
|
|
135
|
+
if self.allow_scheduled_tasks_check_for_issues:
|
|
136
|
+
schedule_check_for_issues_config: Dict[str, Any] = self.config.get('scheduled_tasks_check_for_issues', {}) # type: ignore
|
|
137
|
+
if not isinstance(schedule_check_for_issues_config, dict):
|
|
138
|
+
raise ValueError("The scheduled tasks checking for issues configuration (schedule_check_for_issues_config) must be a dictionary")
|
|
139
|
+
|
|
140
|
+
if 'max_duration_seconds_showing_same_error' not in schedule_check_for_issues_config:
|
|
141
|
+
raise ValueError("The scheduled tasks configuration must contain the 'max_duration_seconds_showing_same_error' key")
|
|
142
|
+
self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks: int = schedule_check_for_issues_config.get('max_duration_seconds_showing_same_error') # type: ignore
|
|
143
|
+
|
|
144
|
+
if 'start_immediately' not in schedule_check_for_issues_config:
|
|
145
|
+
raise ValueError("The scheduled tasks configuration must contain the 'start_immediately' key")
|
|
146
|
+
self.start_scheduled_tasks_immediately: bool = schedule_check_for_issues_config.get('start_immediately') # type: ignore
|
|
147
|
+
|
|
148
|
+
if 'duration_in_sec_wait_between_each_execution' not in schedule_check_for_issues_config:
|
|
149
|
+
raise ValueError("The scheduled tasks configuration must contain the 'duration_in_sec_wait_between_each_execution' key")
|
|
150
|
+
self.duration_in_sec_wait_between_each_schedule_task_execution: int = schedule_check_for_issues_config.get('duration_in_sec_wait_between_each_execution') # type: ignore
|
|
151
|
+
|
|
152
|
+
# Get the scheduled tasks show info configuration
|
|
153
|
+
if self.allow_scheduled_task_show_info:
|
|
154
|
+
schedule_show_info_config: Dict[str, Any] = self.config.get('scheduled_tasks_show_infos', {}) # type: ignore
|
|
155
|
+
if not isinstance(schedule_show_info_config, dict):
|
|
156
|
+
raise ValueError("The scheduled show info tasks configuration (scheduled_tasks_show_infos) must be a dictionary")
|
|
157
|
+
|
|
158
|
+
if 'start_immediately' not in schedule_show_info_config:
|
|
159
|
+
raise ValueError("The scheduled tasks configuration must contain the 'start_immediately' key")
|
|
160
|
+
self.start_scheduled_task_show_info_immediately: bool = schedule_show_info_config.get('start_immediately') # type: ignore
|
|
161
|
+
|
|
162
|
+
if 'duration_in_sec_wait_between_each_execution' not in schedule_show_info_config:
|
|
163
|
+
raise ValueError("The scheduled tasks configuration must contain the 'duration_in_sec_wait_between_each_execution' key")
|
|
164
|
+
self.duration_in_sec_wait_between_each_schedule_task_show_info_execution: int = schedule_show_info_config.get('duration_in_sec_wait_between_each_execution') # type: ignore
|
|
165
|
+
|
|
166
|
+
logging.debug(msg="Configuration loaded successfully")
|
|
167
|
+
|
|
168
|
+
#region Execute Command
|
|
169
|
+
|
|
170
|
+
def execute_and_verify(self, command: List[str], display_name: str, timeout: Optional[int] = None, display_only_if_critical: bool = False, check_also_stdout_not_containing: Optional[str] = None) -> Tuple[bool, str]:
|
|
171
|
+
"""
|
|
172
|
+
Execute a shell command and verify its correct execution.
|
|
173
|
+
|
|
174
|
+
:param command: A list of strings representing the command and its arguments.
|
|
175
|
+
:param display_name: The name of the command to display in the output message.
|
|
176
|
+
:param timeout: Timeout in seconds for the command execution. None means no timeout.
|
|
177
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
178
|
+
:param check_also_stdout_not_containing: If not None, we check that the stdout does not contain this string, if found, we consider it as an execution error.
|
|
179
|
+
|
|
180
|
+
:return: A tuple containing a boolean indicating if the command was executed successfully and a string containing the result message.
|
|
181
|
+
"""
|
|
182
|
+
returncode: int = -1
|
|
183
|
+
|
|
184
|
+
try:
|
|
185
|
+
logging.debug(msg=f"Executing command {display_name} (command: {command})...")
|
|
186
|
+
pipe = subprocess.Popen(args=command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False)
|
|
187
|
+
try:
|
|
188
|
+
stdout, stderr = pipe.communicate(timeout=timeout)
|
|
189
|
+
returncode = pipe.returncode
|
|
190
|
+
except subprocess.TimeoutExpired:
|
|
191
|
+
logging.warning(msg=f"Timeout expired for {display_name} (command: {command})")
|
|
192
|
+
pipe.kill()
|
|
193
|
+
stdout, stderr = pipe.communicate()
|
|
194
|
+
except Exception as e:
|
|
195
|
+
out_msg = f"⚠️ **Error {display_name}**:\n```sh\n{e}\n```"
|
|
196
|
+
logging.exception(msg=out_msg)
|
|
197
|
+
return False, out_msg
|
|
198
|
+
|
|
199
|
+
# Decode and replace backticks with double quotes
|
|
200
|
+
stdout_str: str = stdout.decode(encoding='utf-8').replace('`', '"').strip()
|
|
201
|
+
stderr_str: str = stderr.decode(encoding='utf-8').replace('`', '"').strip()
|
|
202
|
+
|
|
203
|
+
# Check if the command executed successfully
|
|
204
|
+
res: bool = (returncode == 0)
|
|
205
|
+
if res and check_also_stdout_not_containing:
|
|
206
|
+
res = check_also_stdout_not_containing not in stdout_str
|
|
207
|
+
|
|
208
|
+
out_msg: str = ""
|
|
209
|
+
if not res:
|
|
210
|
+
out_msg = f"❌ **Error {display_name}**\n"
|
|
211
|
+
if stderr_str != "":
|
|
212
|
+
out_msg += f"- Error log:\n```sh\n{stderr_str}\n```"
|
|
213
|
+
|
|
214
|
+
if stdout_str != "":
|
|
215
|
+
out_msg += f"- Info:\n```sh\n{stdout_str}\n```"
|
|
216
|
+
|
|
217
|
+
out_msg += f"- Error code: {returncode}"
|
|
218
|
+
else:
|
|
219
|
+
if not display_only_if_critical:
|
|
220
|
+
out_msg = f"✅ **{display_name} executed successfully**"
|
|
221
|
+
|
|
222
|
+
if res:
|
|
223
|
+
logging.info(msg=f"Command {display_name} (command {command}) executed successfully")
|
|
224
|
+
else:
|
|
225
|
+
logging.error(msg=f"Error executing command {display_name} (command {command}):\n{out_msg}")
|
|
226
|
+
|
|
227
|
+
return res, out_msg
|
|
228
|
+
|
|
229
|
+
#endregion
|
|
230
|
+
|
|
231
|
+
#region Reboot
|
|
232
|
+
|
|
233
|
+
def reboot_server(self) -> str:
|
|
234
|
+
"""
|
|
235
|
+
Reboot the server.
|
|
236
|
+
|
|
237
|
+
:return: A string containing the output message.
|
|
238
|
+
|
|
239
|
+
IMPORTANT: To be usable, the user must have the right to execute `sudo /sbin/reboot` command without password.
|
|
240
|
+
"""
|
|
241
|
+
logging.info(msg="Rebooting the server...")
|
|
242
|
+
_, out_msg = self.execute_and_verify(command=["sudo", "/sbin/reboot"], display_name="Server reboot", timeout=None, display_only_if_critical=False)
|
|
243
|
+
return out_msg
|
|
244
|
+
|
|
245
|
+
#endregion
|
|
246
|
+
|
|
247
|
+
#region Disk Usage
|
|
248
|
+
|
|
249
|
+
def _check_disk_usage_per_disk(self, disk_path: str, display_name: str, warning_disk_percent: float, critical_disk_percent: float, display_only_if_critical: bool=False) -> str:
|
|
250
|
+
"""
|
|
251
|
+
Check the disk usage for a specific disk.
|
|
252
|
+
|
|
253
|
+
:param disk_path: The path to the disk.
|
|
254
|
+
:param display_name: The name of the disk to display in the output message.
|
|
255
|
+
:param warning_disk_percent: The warning disk usage percentage.
|
|
256
|
+
:param critical_disk_percent: The critical disk usage percentage.
|
|
257
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
258
|
+
|
|
259
|
+
:return: A string containing the result message.
|
|
260
|
+
"""
|
|
261
|
+
out_msg: str = ""
|
|
262
|
+
try:
|
|
263
|
+
total, used, free = shutil.disk_usage(path=disk_path)
|
|
264
|
+
total_gb: float = total / (2**30)
|
|
265
|
+
used_gb: float = used / (2**30)
|
|
266
|
+
free_gb: float = free / (2**30)
|
|
267
|
+
|
|
268
|
+
percent_used: float = (used / total) * 100
|
|
269
|
+
if critical_disk_percent != -1 and percent_used > critical_disk_percent:
|
|
270
|
+
out_msg = f"- 🚨 Critical disk {display_name} (`{disk_path}`) space:\n" \
|
|
271
|
+
f" - Total: {total_gb:.2f}GB\n" \
|
|
272
|
+
f" - Used: {used_gb:.2f}GB ({percent_used:.2f}%)\n" \
|
|
273
|
+
f" - Free: {free_gb:.2f}GB\n" \
|
|
274
|
+
f"⚠️ **Free up disk space** ⚠️"
|
|
275
|
+
logging.warning(msg=out_msg)
|
|
276
|
+
elif not display_only_if_critical:
|
|
277
|
+
if warning_disk_percent != -1 and percent_used > warning_disk_percent:
|
|
278
|
+
icon: str = "⚠️ "
|
|
279
|
+
elif critical_disk_percent == -1 and warning_disk_percent == -1:
|
|
280
|
+
icon = ""
|
|
281
|
+
else:
|
|
282
|
+
icon = "✅ "
|
|
283
|
+
out_msg = f"{icon} {display_name} (`{disk_path}`): {free_gb:.2f}GB free, {used_gb:.2f}GB used (**{percent_used:.2f}%** used on a total of {total_gb:.2f}GB)"
|
|
284
|
+
logging.info(msg=out_msg)
|
|
285
|
+
except Exception as e:
|
|
286
|
+
out_msg = f"⚠️ **Error getting disk {display_name} (`{disk_path}`) space**:\n```sh\n{e}\n```"
|
|
287
|
+
logging.exception(msg=out_msg)
|
|
288
|
+
|
|
289
|
+
return out_msg
|
|
290
|
+
|
|
291
|
+
def check_all_disk_usage(self, is_private: bool, display_only_if_critical: bool=False) -> str:
|
|
292
|
+
"""
|
|
293
|
+
Check the disk usage for all disks configured in the JSON configuration file.
|
|
294
|
+
|
|
295
|
+
:param is_private: User permission to check private or public disks (True for private, False for public).
|
|
296
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
297
|
+
|
|
298
|
+
:return: A string containing the result message.
|
|
299
|
+
"""
|
|
300
|
+
try:
|
|
301
|
+
out_msg: str = ""
|
|
302
|
+
for disk_config in self.config['disks']:
|
|
303
|
+
if is_private or is_private == disk_config['is_private']:
|
|
304
|
+
device = disk_config['device']
|
|
305
|
+
display_name = disk_config.get('display_name', device)
|
|
306
|
+
# Verify that warning_percent & critical_percent exist, otherwise use -1
|
|
307
|
+
warning_percent = disk_config.get('warning_percent', -1)
|
|
308
|
+
critical_percent = disk_config.get('critical_percent', -1)
|
|
309
|
+
|
|
310
|
+
# We only check what is needed
|
|
311
|
+
if not display_only_if_critical or (display_only_if_critical and critical_percent != -1):
|
|
312
|
+
result: str = self._check_disk_usage_per_disk(disk_path=device, display_name=display_name, warning_disk_percent=warning_percent, critical_disk_percent=critical_percent, display_only_if_critical=display_only_if_critical)
|
|
313
|
+
if result and result != "":
|
|
314
|
+
if out_msg != "":
|
|
315
|
+
out_msg += "\n"
|
|
316
|
+
out_msg += f"- {result}"
|
|
317
|
+
|
|
318
|
+
if out_msg != "":
|
|
319
|
+
out_msg = f"# 🖥️ Disk space 🖥️\n{out_msg}"
|
|
320
|
+
|
|
321
|
+
return out_msg
|
|
322
|
+
except Exception as e:
|
|
323
|
+
out_msg = f"⚠️ **Error retrieving disk space**:\n```sh\n{e}\n```"
|
|
324
|
+
logging.exception(msg=out_msg)
|
|
325
|
+
return out_msg
|
|
326
|
+
|
|
327
|
+
#endregion
|
|
328
|
+
|
|
329
|
+
#region Folder Usage
|
|
330
|
+
|
|
331
|
+
def _get_folder_size_in_bytes(self, start_path: str = '.') -> float:
|
|
332
|
+
"""
|
|
333
|
+
Get the size of a folder in bytes.
|
|
334
|
+
|
|
335
|
+
:param start_path: The path to the folder.
|
|
336
|
+
|
|
337
|
+
:return: The size of the folder in bytes.
|
|
338
|
+
"""
|
|
339
|
+
try:
|
|
340
|
+
total_size = 0
|
|
341
|
+
for dirpath, _, filenames in os.walk(start_path):
|
|
342
|
+
for f in filenames:
|
|
343
|
+
fp = os.path.join(dirpath, f)
|
|
344
|
+
|
|
345
|
+
# Skip if it is symbolic link
|
|
346
|
+
if not os.path.islink(fp):
|
|
347
|
+
total_size += os.path.getsize(fp)
|
|
348
|
+
logging.info(msg=f"Total size of folder {start_path}: {total_size} bytes")
|
|
349
|
+
return total_size
|
|
350
|
+
except Exception as e:
|
|
351
|
+
logging.exception(msg=f"Error getting folder size in bytes for {start_path}:\n{e}")
|
|
352
|
+
return -1
|
|
353
|
+
|
|
354
|
+
def _check_folder_usage(self, folder_path: str, display_name: str, warning_usage_giga: float, critical_usage_giga: float, total_disk_giga: float, display_only_if_critical: bool=False) -> str:
|
|
355
|
+
"""
|
|
356
|
+
Check the folder usage for a specific folder.
|
|
357
|
+
|
|
358
|
+
:param folder_path: The path to the folder.
|
|
359
|
+
:param display_name: The name of the folder to display in the output message.
|
|
360
|
+
:param warning_usage_giga: The warning folder usage in gigabytes.
|
|
361
|
+
:param critical_usage_giga: The critical folder usage in gigabytes.
|
|
362
|
+
:param total_disk_giga: The total disk space in gigabytes.
|
|
363
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
364
|
+
|
|
365
|
+
:return: A string containing the result message.
|
|
366
|
+
"""
|
|
367
|
+
out_msg: str = ""
|
|
368
|
+
try:
|
|
369
|
+
total_used_bytes = self._get_folder_size_in_bytes(start_path=folder_path)
|
|
370
|
+
total_used_giga = total_used_bytes / (2**30)
|
|
371
|
+
|
|
372
|
+
percent_used: float = 0
|
|
373
|
+
if critical_usage_giga > 0:
|
|
374
|
+
percent_used = (total_used_bytes / (critical_usage_giga * (2**30)) ) * 100
|
|
375
|
+
|
|
376
|
+
if critical_usage_giga > 0 and total_used_bytes >= critical_usage_giga * (2**30):
|
|
377
|
+
out_msg = f"- 🚨 Critical folder {display_name} space (`{folder_path}`):\n" \
|
|
378
|
+
f" - Total allowed: {critical_usage_giga:.2f}GB\n" \
|
|
379
|
+
f" - Used: {total_used_giga:.2f}GB ({percent_used:.2f}%)\n" \
|
|
380
|
+
f"⚠️ **Please free up space in this folder** ⚠️"
|
|
381
|
+
logging.warning(msg=out_msg)
|
|
382
|
+
elif not display_only_if_critical:
|
|
383
|
+
if warning_usage_giga > 0 and total_used_bytes >= warning_usage_giga * (2**30):
|
|
384
|
+
icon: str = "⚠️ "
|
|
385
|
+
elif critical_usage_giga == -1 and warning_usage_giga == -1:
|
|
386
|
+
icon = ""
|
|
387
|
+
else:
|
|
388
|
+
icon = "✅ "
|
|
389
|
+
|
|
390
|
+
if critical_usage_giga > 0:
|
|
391
|
+
out_msg = f"{icon}{display_name}: {total_used_giga:.2f}GB used (**{((total_used_bytes / (total_disk_giga * (2**30))) * 100):.2f}%** of total disk space, {percent_used:.2f}% used of a total allowed of {critical_usage_giga:.2f}GB)"
|
|
392
|
+
else:
|
|
393
|
+
out_msg = f"{icon}{display_name}: {total_used_giga:.2f}GB used (**{((total_used_bytes / (total_disk_giga * (2**30))) * 100):.2f}%** of total disk space)"
|
|
394
|
+
|
|
395
|
+
logging.info(msg=out_msg)
|
|
396
|
+
except Exception as e:
|
|
397
|
+
out_msg = f"⚠️ **Error getting folder {display_name} space**:\n```sh\n{e}\n```"
|
|
398
|
+
logging.exception(msg=out_msg)
|
|
399
|
+
|
|
400
|
+
return out_msg
|
|
401
|
+
|
|
402
|
+
def check_all_folder_usage(self, is_private: bool, display_only_if_critical: bool=False) -> str:
|
|
403
|
+
"""
|
|
404
|
+
Check the folder usage for all folders configured in the JSON configuration file.
|
|
405
|
+
|
|
406
|
+
:param is_private: User permission to check private or public folders (True for private, False for public).
|
|
407
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
408
|
+
|
|
409
|
+
:return: A string containing the result message.
|
|
410
|
+
"""
|
|
411
|
+
try:
|
|
412
|
+
# Get total disk space of the system to display a usage percentage
|
|
413
|
+
try:
|
|
414
|
+
total_disk, _, _ = shutil.disk_usage(path="/")
|
|
415
|
+
total_disk_giga: float = total_disk / (2**30)
|
|
416
|
+
except Exception as e:
|
|
417
|
+
total_disk_giga = 0
|
|
418
|
+
logging.exception(msg=f"Error getting total disk space:\n{e}")
|
|
419
|
+
|
|
420
|
+
out_msg: str = ""
|
|
421
|
+
for folder_config in self.config['folders']:
|
|
422
|
+
if is_private or is_private == folder_config['is_private']:
|
|
423
|
+
folder_path = folder_config['folder_path']
|
|
424
|
+
display_name = folder_config.get('display_name', folder_path)
|
|
425
|
+
# Verify that warning_usage_giga & critical_usage_giga exist, otherwise use -1
|
|
426
|
+
warning_usage_giga = folder_config.get('warning_usage_giga', -1)
|
|
427
|
+
critical_usage_giga = folder_config.get('critical_usage_giga', -1)
|
|
428
|
+
|
|
429
|
+
# We only check what is needed
|
|
430
|
+
if not display_only_if_critical or (display_only_if_critical and critical_usage_giga != -1):
|
|
431
|
+
result: str = self._check_folder_usage(folder_path=folder_path, display_name=display_name, warning_usage_giga=warning_usage_giga, critical_usage_giga=critical_usage_giga, total_disk_giga=total_disk_giga, display_only_if_critical=display_only_if_critical)
|
|
432
|
+
if result and result != "":
|
|
433
|
+
if out_msg != "":
|
|
434
|
+
out_msg += "\n"
|
|
435
|
+
out_msg += f"- {result}"
|
|
436
|
+
|
|
437
|
+
if out_msg != "":
|
|
438
|
+
out_msg = f"# 📂 Folder space 📂\n{out_msg}"
|
|
439
|
+
|
|
440
|
+
return out_msg
|
|
441
|
+
except Exception as e:
|
|
442
|
+
out_msg = f"⚠️ **Error getting folder space**:\n```sh\n{e}\n```"
|
|
443
|
+
logging.exception(msg=out_msg)
|
|
444
|
+
return out_msg
|
|
445
|
+
|
|
446
|
+
#endregion
|
|
447
|
+
|
|
448
|
+
#region CPU & RAM & Swap & Temperature & Uptime
|
|
449
|
+
|
|
450
|
+
def _get_cpu_name(self) -> str:
|
|
451
|
+
"""
|
|
452
|
+
Get the CPU name.
|
|
453
|
+
|
|
454
|
+
:return: The CPU name.
|
|
455
|
+
"""
|
|
456
|
+
cpu_name: str = ""
|
|
457
|
+
try:
|
|
458
|
+
cpu_name = platform.processor()
|
|
459
|
+
if not cpu_name or cpu_name == "":
|
|
460
|
+
with open(file="/proc/cpuinfo", mode="r") as file:
|
|
461
|
+
cpu_info_lines: List[str] = file.readlines()
|
|
462
|
+
for line in cpu_info_lines:
|
|
463
|
+
if "model name" in line:
|
|
464
|
+
cpu_name = re.sub(pattern=r"model name\s+:\s+", repl="", string=line)
|
|
465
|
+
# remove extra spaces
|
|
466
|
+
cpu_name = re.sub(pattern=r"\s+", repl=" ", string=cpu_name).strip()
|
|
467
|
+
break
|
|
468
|
+
|
|
469
|
+
logging.info(msg=f"CPU name: {cpu_name}")
|
|
470
|
+
return cpu_name
|
|
471
|
+
except Exception as e:
|
|
472
|
+
logging.exception(msg=f"Error getting CPU name:\n{e}")
|
|
473
|
+
return ""
|
|
474
|
+
|
|
475
|
+
def check_cpu_usage(self, display_only_if_critical: bool=False) -> str:
|
|
476
|
+
"""
|
|
477
|
+
Check the CPU usage.
|
|
478
|
+
|
|
479
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
480
|
+
|
|
481
|
+
:return: A string containing the result message.
|
|
482
|
+
"""
|
|
483
|
+
out_msg: str = ""
|
|
484
|
+
try:
|
|
485
|
+
# Getting average CPU usage for the last second
|
|
486
|
+
cpu_percent: float = psutil.cpu_percent(percpu=False)
|
|
487
|
+
|
|
488
|
+
# Getting number of cores and Ghz
|
|
489
|
+
cpu_info: float = psutil.cpu_freq().max / 1000
|
|
490
|
+
cpu_cores: int = psutil.cpu_count(logical=False)
|
|
491
|
+
cpu_name: str = self._get_cpu_name()
|
|
492
|
+
|
|
493
|
+
if cpu_percent > self.critical_cpu_percent:
|
|
494
|
+
out_msg = f"- 🚨 **Critical CPU usage**:\n- **{cpu_percent:.2f}%** used on {cpu_cores} core of {cpu_info:.2f}GHz ({cpu_name})\n⚠️ **Check what is using so much CPU power** ⚠️"
|
|
495
|
+
logging.warning(msg=out_msg)
|
|
496
|
+
elif not display_only_if_critical:
|
|
497
|
+
if cpu_percent > self.warning_cpu_percent:
|
|
498
|
+
icon = "⚠️"
|
|
499
|
+
else:
|
|
500
|
+
icon = "✅"
|
|
501
|
+
out_msg = f"- {icon} **{cpu_percent:.2f}%** used on {cpu_cores} core of {cpu_info:.2f}GHz ({cpu_name})"
|
|
502
|
+
logging.info(msg=out_msg)
|
|
503
|
+
except Exception as e:
|
|
504
|
+
out_msg = f"- ⚠️ **Error getting CPU usage**:\n```sh\n{e}\n```"
|
|
505
|
+
logging.exception(msg=out_msg)
|
|
506
|
+
|
|
507
|
+
if out_msg != "":
|
|
508
|
+
out_msg = f"# 📈 CPU 📈\n{out_msg}"
|
|
509
|
+
|
|
510
|
+
return out_msg
|
|
511
|
+
|
|
512
|
+
def check_ram_usage(self, display_only_if_critical: bool=False) -> str:
|
|
513
|
+
"""
|
|
514
|
+
Check the RAM usage.
|
|
515
|
+
|
|
516
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
517
|
+
|
|
518
|
+
:return: A string containing the result message.
|
|
519
|
+
"""
|
|
520
|
+
out_msg: str = ""
|
|
521
|
+
try:
|
|
522
|
+
# Getting RAM usage
|
|
523
|
+
ram = psutil.virtual_memory()
|
|
524
|
+
total_ram: float = ram.total / (2**30)
|
|
525
|
+
used_ram: float = ram.used / (2**30)
|
|
526
|
+
free_ram: float = total_ram - used_ram
|
|
527
|
+
percent_ram: float = ram.percent
|
|
528
|
+
if percent_ram > self.critical_ram_percent:
|
|
529
|
+
out_msg = f"- 🚨 **Critical RAM usage**:\n- Total: {total_ram:.2f}GB\n- Used: {used_ram:.2f}GB ({percent_ram:.2f}%)\n- Free: {free_ram:.2f}GB\n⚠️ **Check what is using so much RAM** ⚠️"
|
|
530
|
+
logging.warning(msg=out_msg)
|
|
531
|
+
elif not display_only_if_critical:
|
|
532
|
+
if percent_ram > self.warning_ram_percent:
|
|
533
|
+
icon = "⚠️"
|
|
534
|
+
else:
|
|
535
|
+
icon = "✅"
|
|
536
|
+
out_msg = f"- {icon} **{percent_ram:.2f}%** used on a total of {total_ram:.2f}GB ({free_ram:.2f}GB free, {used_ram:.2f}GB used)"
|
|
537
|
+
logging.info(msg=out_msg)
|
|
538
|
+
except Exception as e:
|
|
539
|
+
out_msg = f"⚠️ **Error getting RAM usage**:\n```sh\n{e}\n```"
|
|
540
|
+
logging.exception(msg=out_msg)
|
|
541
|
+
|
|
542
|
+
if out_msg != "":
|
|
543
|
+
out_msg = f"# 📊 RAM 📊\n{out_msg}"
|
|
544
|
+
|
|
545
|
+
return out_msg
|
|
546
|
+
|
|
547
|
+
def check_swap_usage(self, display_only_if_critical: bool=False) -> str:
|
|
548
|
+
"""
|
|
549
|
+
Check the SWAP usage.
|
|
550
|
+
|
|
551
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
552
|
+
|
|
553
|
+
:return: A string containing the result message.
|
|
554
|
+
"""
|
|
555
|
+
out_msg: str = ""
|
|
556
|
+
try:
|
|
557
|
+
# Getting Swap usage
|
|
558
|
+
swap = psutil.swap_memory()
|
|
559
|
+
total_swap: float = swap.total / (2**30)
|
|
560
|
+
used_swap: float = swap.used / (2**30)
|
|
561
|
+
free_swap: float = swap.free / (2**30)
|
|
562
|
+
percent_swap: float = swap.percent
|
|
563
|
+
if percent_swap > self.critical_swap_percent:
|
|
564
|
+
out_msg = f"- 🚨 **Critical SWAP usage**\n- Total: {total_swap:.2f}GB\n- Used: {used_swap:.2f}GB ({percent_swap:.2f}%)\n- Free: {free_swap:.2f}GB\n⚠️ **Check what is using so much SWAP** ⚠️"
|
|
565
|
+
logging.warning(msg=out_msg)
|
|
566
|
+
elif not display_only_if_critical:
|
|
567
|
+
if percent_swap > self.warning_swap_percent:
|
|
568
|
+
icon = "⚠️"
|
|
569
|
+
else:
|
|
570
|
+
icon = "✅"
|
|
571
|
+
out_msg = f"- {icon} **{percent_swap:.2f}%** used on a total of {total_swap:.2f}GB ({free_swap:.2f}GB free, {used_swap:.2f}GB used)"
|
|
572
|
+
logging.info(msg=out_msg)
|
|
573
|
+
except Exception as e:
|
|
574
|
+
out_msg = f"- ⚠️ **Error getting SWAP usage**:\n```sh\n{e}\n```"
|
|
575
|
+
logging.exception(msg=out_msg)
|
|
576
|
+
|
|
577
|
+
if out_msg != "":
|
|
578
|
+
out_msg = f"# 🔄 SWAP 🔄\n{out_msg}"
|
|
579
|
+
|
|
580
|
+
return out_msg
|
|
581
|
+
|
|
582
|
+
def check_cpu_temperature(self, display_only_if_critical: bool=False) -> str:
|
|
583
|
+
"""
|
|
584
|
+
Check the CPU temperature.
|
|
585
|
+
|
|
586
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
587
|
+
|
|
588
|
+
:return: A string containing the result message.
|
|
589
|
+
"""
|
|
590
|
+
out_msg: str = ""
|
|
591
|
+
try:
|
|
592
|
+
if not hasattr(psutil, "sensors_temperatures"):
|
|
593
|
+
logging.error(msg="psutil does not support reading CPU temperatures")
|
|
594
|
+
if not display_only_if_critical:
|
|
595
|
+
return "- ⚠️ **Error reading CPU temperature**: `psutil` does not support reading CPU temperatures."
|
|
596
|
+
else:
|
|
597
|
+
return ""
|
|
598
|
+
|
|
599
|
+
temps = psutil.sensors_temperatures() # type: ignore
|
|
600
|
+
cpu_temps = temps.get('coretemp', []) # type: ignore # 'coretemp' est commun sur les systèmes Intel
|
|
601
|
+
|
|
602
|
+
if not cpu_temps:
|
|
603
|
+
logging.error(msg="No CPU temperature sensors found")
|
|
604
|
+
if not display_only_if_critical:
|
|
605
|
+
return "- ⚠️ **Error reading CPU temperature**: No CPU temperature sensor found."
|
|
606
|
+
else:
|
|
607
|
+
return ""
|
|
608
|
+
|
|
609
|
+
# Calculer la température maximale
|
|
610
|
+
max_temp = max(temp.current for temp in cpu_temps) # type: ignore
|
|
611
|
+
|
|
612
|
+
# Vérifier la température par rapport au seuil critique
|
|
613
|
+
if max_temp > self.critical_temperature_celsius:
|
|
614
|
+
out_msg = f"- 🚨 **Critical CPU temperature**:\n- {max_temp:.2f}°C\n- Critical threshold: {self.critical_temperature_celsius}°C\n⚠️ **Check the CPU cooling system** ⚠️"
|
|
615
|
+
logging.warning(msg=out_msg)
|
|
616
|
+
elif not display_only_if_critical:
|
|
617
|
+
if max_temp > self.warning_temperature_celsius:
|
|
618
|
+
icon = "⚠️"
|
|
619
|
+
else:
|
|
620
|
+
icon = "✅"
|
|
621
|
+
out_msg = f"- {icon} **{max_temp:.2f}°C**"
|
|
622
|
+
logging.info(msg=out_msg)
|
|
623
|
+
except Exception as e:
|
|
624
|
+
out_msg = f"- ⚠️ **Error getting CPU temperature**:\n```sh\n{e}\n```"
|
|
625
|
+
logging.exception(msg=out_msg)
|
|
626
|
+
|
|
627
|
+
if out_msg != "":
|
|
628
|
+
out_msg = f"# 🌡️ Temperature 🌡️\n{out_msg}"
|
|
629
|
+
|
|
630
|
+
return out_msg
|
|
631
|
+
|
|
632
|
+
def get_uptime(self) -> str:
|
|
633
|
+
"""
|
|
634
|
+
Get the system uptime.
|
|
635
|
+
|
|
636
|
+
:return: A string containing the result message.
|
|
637
|
+
"""
|
|
638
|
+
try:
|
|
639
|
+
# Obtenir l'uptime du système en secondes
|
|
640
|
+
uptime_seconds: float = time.time() - psutil.boot_time()
|
|
641
|
+
years, months = divmod(uptime_seconds, 60*60*24*30*12)
|
|
642
|
+
months, days = divmod(months, 60*60*24*30)
|
|
643
|
+
days, hours = divmod(days, 60*60*24)
|
|
644
|
+
hours, minutes = divmod(hours, 60*60)
|
|
645
|
+
minutes, seconds = divmod(minutes, 60)
|
|
646
|
+
|
|
647
|
+
# Date de démarrage du système
|
|
648
|
+
boot_time: str = time.strftime("%d/%m/%Y %H:%M:%S", time.localtime(psutil.boot_time()))
|
|
649
|
+
|
|
650
|
+
# Funny emoji depending on uptime
|
|
651
|
+
emoji: str = ""
|
|
652
|
+
if years >= 2:
|
|
653
|
+
emoji = "🎂"
|
|
654
|
+
if years >= 1:
|
|
655
|
+
emoji = "🎉"
|
|
656
|
+
elif months >= 6:
|
|
657
|
+
emoji = "🥳"
|
|
658
|
+
elif months >= 1:
|
|
659
|
+
emoji = "😀"
|
|
660
|
+
elif days >= 1:
|
|
661
|
+
emoji = "🎊"
|
|
662
|
+
elif hours >= 1:
|
|
663
|
+
emoji = "👶"
|
|
664
|
+
elif minutes >= 20:
|
|
665
|
+
emoji = "🤔"
|
|
666
|
+
else:
|
|
667
|
+
emoji = "☢️"
|
|
668
|
+
|
|
669
|
+
dispo: str = ""
|
|
670
|
+
if years >= 1:
|
|
671
|
+
dispo += f"{int(years)} year(s) "
|
|
672
|
+
if months >= 1:
|
|
673
|
+
dispo += f"{int(months)} month(s) "
|
|
674
|
+
if days >= 1:
|
|
675
|
+
dispo += f"{int(days)} day(s) "
|
|
676
|
+
if hours >= 1:
|
|
677
|
+
dispo += f"{int(hours)}h "
|
|
678
|
+
if minutes >= 1:
|
|
679
|
+
dispo += f"{int(minutes)}min "
|
|
680
|
+
if seconds >= 1:
|
|
681
|
+
dispo += f"{int(seconds)}sec "
|
|
682
|
+
|
|
683
|
+
out_msg: str = (f"# 🕒 System availability 🕒\n"
|
|
684
|
+
f"- {emoji} **{dispo}**(started on {boot_time})")
|
|
685
|
+
logging.info(msg=out_msg)
|
|
686
|
+
|
|
687
|
+
except Exception as e:
|
|
688
|
+
out_msg = f"- ⚠️ **Error getting system uptime**:\n```sh\n{e}\n```"
|
|
689
|
+
logging.exception(msg=out_msg)
|
|
690
|
+
|
|
691
|
+
return out_msg
|
|
692
|
+
|
|
693
|
+
#endregion
|
|
694
|
+
|
|
695
|
+
#region Ping Websites
|
|
696
|
+
|
|
697
|
+
def _ping_website(self, website: str, display_name: str, display_only_if_critical: bool=False) -> str:
|
|
698
|
+
"""
|
|
699
|
+
Ping a website.
|
|
700
|
+
|
|
701
|
+
:param website: The website to ping.
|
|
702
|
+
:param display_name: The name of the website to display in the output message.
|
|
703
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
704
|
+
|
|
705
|
+
:return: A string containing the result message.
|
|
706
|
+
"""
|
|
707
|
+
ping_command: list[str] = ["ping", "-c", "1", website]
|
|
708
|
+
display_name = f"[{display_name}](https://{website})"
|
|
709
|
+
|
|
710
|
+
start_time: float = time.time()
|
|
711
|
+
res, out_msg = self.execute_and_verify(command=ping_command, display_name=f"ping {display_name}", timeout=5, display_only_if_critical=display_only_if_critical)
|
|
712
|
+
end_time: float = time.time()
|
|
713
|
+
|
|
714
|
+
if res and not display_only_if_critical:
|
|
715
|
+
res_ping_sec: str = "{:.2f}sec".format(end_time - start_time)
|
|
716
|
+
out_msg: str = f"✅ **{display_name} answered in {res_ping_sec}**."
|
|
717
|
+
logging.info(msg=out_msg)
|
|
718
|
+
|
|
719
|
+
return out_msg
|
|
720
|
+
|
|
721
|
+
def ping_all_websites(self, is_private: bool, display_only_if_critical: bool=False) -> str:
|
|
722
|
+
"""
|
|
723
|
+
Ping all websites configured in the JSON configuration file.
|
|
724
|
+
|
|
725
|
+
:param is_private: User permission to check private or public websites (True for private, False for public).
|
|
726
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
727
|
+
|
|
728
|
+
:return: A string containing the result message.
|
|
729
|
+
"""
|
|
730
|
+
out_msg: str = ""
|
|
731
|
+
for ping_config in self.config['pings']:
|
|
732
|
+
if is_private or is_private == ping_config['is_private']:
|
|
733
|
+
result: str = self._ping_website(website=ping_config['website'], display_name=ping_config['display_name'], display_only_if_critical=display_only_if_critical)
|
|
734
|
+
if result:
|
|
735
|
+
if out_msg:
|
|
736
|
+
out_msg += "\n"
|
|
737
|
+
out_msg += f"- {result}"
|
|
738
|
+
|
|
739
|
+
if out_msg != "":
|
|
740
|
+
out_msg = f"# 🌐 Website state 🌐\n{out_msg}"
|
|
741
|
+
|
|
742
|
+
return out_msg
|
|
743
|
+
|
|
744
|
+
#endregion
|
|
745
|
+
|
|
746
|
+
#region Services
|
|
747
|
+
|
|
748
|
+
def get_all_services_allowed_to_restart(self, is_private: bool) -> str:
|
|
749
|
+
"""
|
|
750
|
+
Get all services allowed to restart which are configured in the JSON configuration file.
|
|
751
|
+
|
|
752
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
753
|
+
|
|
754
|
+
:return: A string containing the result message.
|
|
755
|
+
"""
|
|
756
|
+
out_msg: str = ""
|
|
757
|
+
for service_name in self.config['services'].keys():
|
|
758
|
+
service = self.config['services'][service_name]
|
|
759
|
+
if is_private or service['is_private'] == is_private:
|
|
760
|
+
is_allowed_to_restart: bool = 'restart_command' in service
|
|
761
|
+
|
|
762
|
+
if out_msg != "":
|
|
763
|
+
out_msg += "\n"
|
|
764
|
+
out_msg += f"- `{service_name}`: {service['display_name']}"
|
|
765
|
+
if not is_allowed_to_restart:
|
|
766
|
+
out_msg += " (❌ Not authorized to restart)"
|
|
767
|
+
|
|
768
|
+
if out_msg != "":
|
|
769
|
+
out_msg = f"# 🔄 Services list 🔄\n{out_msg}"
|
|
770
|
+
else:
|
|
771
|
+
out_msg = f"❌ **No service found**."
|
|
772
|
+
|
|
773
|
+
logging.info(msg=out_msg)
|
|
774
|
+
return out_msg
|
|
775
|
+
|
|
776
|
+
def restart_service(self, is_private: bool, service_name: str) -> str:
|
|
777
|
+
"""
|
|
778
|
+
Restart a specific service.
|
|
779
|
+
|
|
780
|
+
:param is_private: User permission to restart private or public services (True for private, False for public).
|
|
781
|
+
:param service_name: The name of the service name to restart (as configured in the JSON configuration file).
|
|
782
|
+
|
|
783
|
+
:return: A string containing the result message.
|
|
784
|
+
"""
|
|
785
|
+
if service_name not in self.config['services'].keys():
|
|
786
|
+
out_msg = f"❌ **Service {service_name} not found**."
|
|
787
|
+
logging.error(msg=out_msg)
|
|
788
|
+
return out_msg
|
|
789
|
+
|
|
790
|
+
service = self.config['services'][service_name]
|
|
791
|
+
if not is_private and is_private != service['is_private']:
|
|
792
|
+
out_msg = f"❌ **Service {service_name} not authorized to restart in public**."
|
|
793
|
+
logging.error(msg=out_msg)
|
|
794
|
+
return out_msg
|
|
795
|
+
|
|
796
|
+
display_name: str = service.get('display_name', service_name)
|
|
797
|
+
if 'restart_command' not in service:
|
|
798
|
+
out_msg = f"❌ **Restart command not found for {display_name}**."
|
|
799
|
+
logging.error(msg=out_msg)
|
|
800
|
+
return out_msg
|
|
801
|
+
|
|
802
|
+
timeout_in_sec: int = service.get('timeout_in_sec', 90)
|
|
803
|
+
|
|
804
|
+
service_call: List[str] = service['restart_command']
|
|
805
|
+
out_msg: str = ""
|
|
806
|
+
|
|
807
|
+
logging.info(f"Trying to restart {display_name} (command: {service_call}) in less than {timeout_in_sec}sec...")
|
|
808
|
+
|
|
809
|
+
start_time: float = time.time()
|
|
810
|
+
res, out_msg = self.execute_and_verify(command=service_call, display_name=f"restart {display_name}", timeout=timeout_in_sec, display_only_if_critical=False)
|
|
811
|
+
end_time: float = time.time()
|
|
812
|
+
|
|
813
|
+
if res:
|
|
814
|
+
readable_duration: str = "{:.2f}".format(end_time - start_time)
|
|
815
|
+
out_msg = f"✅ **{display_name} restarted with success** in {readable_duration}sec."
|
|
816
|
+
logging.info(msg=out_msg)
|
|
817
|
+
|
|
818
|
+
return out_msg
|
|
819
|
+
|
|
820
|
+
def restart_all_services(self, is_private: bool) -> str:
|
|
821
|
+
"""
|
|
822
|
+
Restart all services allowed to restart which are configured in the JSON configuration file.
|
|
823
|
+
|
|
824
|
+
:param is_private: User permission to restart private or public services (True for private, False for public).
|
|
825
|
+
|
|
826
|
+
:return: A string containing the result message.
|
|
827
|
+
"""
|
|
828
|
+
out_msg: str = ""
|
|
829
|
+
for service_name in self.config["services"].keys():
|
|
830
|
+
if is_private or self.config["services"][service_name]['is_private'] == is_private:
|
|
831
|
+
res: str = self.restart_service(is_private=is_private, service_name=service_name)
|
|
832
|
+
if res != "":
|
|
833
|
+
if out_msg:
|
|
834
|
+
out_msg += "\n"
|
|
835
|
+
out_msg += f"- {res}"
|
|
836
|
+
|
|
837
|
+
if out_msg:
|
|
838
|
+
out_msg = f"# 📱 Restart services 📱\n{out_msg}"
|
|
839
|
+
|
|
840
|
+
return out_msg
|
|
841
|
+
|
|
842
|
+
def _get_service_status(self, is_private: bool, service_name: str) -> Tuple[Union[None,bool], str]:
|
|
843
|
+
"""
|
|
844
|
+
Get the status of a specific service.
|
|
845
|
+
|
|
846
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
847
|
+
:param service_name: The name of the service name to check (as configured in the JSON configuration file).
|
|
848
|
+
|
|
849
|
+
:return: A tuple containing the status of the service (True for active, False for inactive, None for error) and a string containing the result message.
|
|
850
|
+
"""
|
|
851
|
+
if service_name not in self.config["services"].keys():
|
|
852
|
+
logging.error(msg=f"Service {service_name} not found")
|
|
853
|
+
return None, ""
|
|
854
|
+
|
|
855
|
+
service = self.config["services"][service_name]
|
|
856
|
+
display_name = service.get('display_name', service_name)
|
|
857
|
+
|
|
858
|
+
if 'status_command' not in service:
|
|
859
|
+
logging.error(msg=f"Status command (status_command) not found for service {service_name}")
|
|
860
|
+
return None, ""
|
|
861
|
+
|
|
862
|
+
status_command = service['status_command']
|
|
863
|
+
check_also_stdout_not_containing = service.get('check_also_stdout_not_containing', None)
|
|
864
|
+
|
|
865
|
+
res, out_msg = self.execute_and_verify(command=status_command, display_name=f"état de {display_name}", timeout=5, display_only_if_critical=False, check_also_stdout_not_containing=check_also_stdout_not_containing)
|
|
866
|
+
return res, out_msg
|
|
867
|
+
|
|
868
|
+
def check_all_services_status(self, is_private: bool) -> str:
|
|
869
|
+
"""
|
|
870
|
+
Check the status of all services configured in the JSON configuration file.
|
|
871
|
+
|
|
872
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
873
|
+
|
|
874
|
+
:return: A string containing the result message.
|
|
875
|
+
"""
|
|
876
|
+
out_msg: str = ""
|
|
877
|
+
try:
|
|
878
|
+
logging.info(msg="Checking the status of services in progress...")
|
|
879
|
+
|
|
880
|
+
# Define the type hint for the lambda function
|
|
881
|
+
status_to_string: Callable[[Optional[bool]], str] = lambda res: (
|
|
882
|
+
"Error retrieving status" if res is None else
|
|
883
|
+
"Active" if res else
|
|
884
|
+
"Inactive"
|
|
885
|
+
)
|
|
886
|
+
|
|
887
|
+
status_to_icon: Callable[[Optional[bool]], str] = lambda res: (
|
|
888
|
+
"⚠️" if res is None else
|
|
889
|
+
"✅" if res else
|
|
890
|
+
"❌"
|
|
891
|
+
)
|
|
892
|
+
|
|
893
|
+
out_msg = "# 📱 Services status 📱\n"
|
|
894
|
+
|
|
895
|
+
for service_name in self.config["services"].keys():
|
|
896
|
+
if is_private or self.config["services"][service_name]['is_private'] == is_private:
|
|
897
|
+
status, status_msg = self._get_service_status(is_private=is_private, service_name=service_name)
|
|
898
|
+
service_status: str = status_to_string(res=status)
|
|
899
|
+
service_icon: str = status_to_icon(res=status)
|
|
900
|
+
if out_msg != "":
|
|
901
|
+
out_msg += "\n"
|
|
902
|
+
out_msg += f"- {service_icon} {self.config['services'][service_name]['display_name']}: **{service_status}**"
|
|
903
|
+
if status == False and status_msg != "":
|
|
904
|
+
out_msg += f"\n{status_msg}"
|
|
905
|
+
except Exception as e:
|
|
906
|
+
out_msg = f"**Internal error checking services status**:\n```sh\n{e}\n```"
|
|
907
|
+
logging.exception(msg=out_msg)
|
|
908
|
+
|
|
909
|
+
return out_msg
|
|
910
|
+
|
|
911
|
+
def check_all_services_status_and_restart_if_down(self, is_private: bool) -> str:
|
|
912
|
+
"""
|
|
913
|
+
Check the status of all services configured in the JSON configuration file and restart them if they are inactive.
|
|
914
|
+
|
|
915
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
916
|
+
|
|
917
|
+
:return: A string containing the result message.
|
|
918
|
+
"""
|
|
919
|
+
out_msg_full: str = ""
|
|
920
|
+
try:
|
|
921
|
+
for service_name in self.config["services"].keys():
|
|
922
|
+
if is_private or self.config["services"][service_name]['is_private'] == is_private:
|
|
923
|
+
status, status_msg = self._get_service_status(is_private=is_private, service_name=service_name)
|
|
924
|
+
if status is False:
|
|
925
|
+
out_msg: str = f"❌ **{self.config['services'][service_name]['display_name']} inactive**. Restarting the service is necessary."
|
|
926
|
+
logging.warning(msg=out_msg)
|
|
927
|
+
if status_msg != "":
|
|
928
|
+
out_msg += f"\n{status_msg}"
|
|
929
|
+
|
|
930
|
+
out_msg_full += out_msg + "\n"
|
|
931
|
+
out_msg_full += self.restart_service(is_private=is_private, service_name=service_name) + "\n"
|
|
932
|
+
except Exception as e:
|
|
933
|
+
out_msg_full = f"**Internal error during service status check.**:\n```sh\n{e}\n```"
|
|
934
|
+
logging.exception(msg=out_msg_full)
|
|
935
|
+
|
|
936
|
+
return out_msg_full
|
|
937
|
+
|
|
938
|
+
#endregion
|
|
939
|
+
|
|
940
|
+
#region SSL Certificates
|
|
941
|
+
|
|
942
|
+
def _get_certificate_info(self, hostname: str): # type: ignore
|
|
943
|
+
"""
|
|
944
|
+
Get the certificate information for a specific hostname.
|
|
945
|
+
|
|
946
|
+
:param hostname: The hostname to get the certificate information.
|
|
947
|
+
|
|
948
|
+
:return: A dictionary containing the certificate information.
|
|
949
|
+
"""
|
|
950
|
+
context: ssl.SSLContext = ssl.create_default_context()
|
|
951
|
+
conn: ssl.SSLSocket = context.wrap_socket(sock=socket.socket(socket.AF_INET), server_hostname=hostname)
|
|
952
|
+
|
|
953
|
+
try:
|
|
954
|
+
conn.connect((hostname, 443))
|
|
955
|
+
cert = conn.getpeercert()
|
|
956
|
+
|
|
957
|
+
# Get the certificate expiration date
|
|
958
|
+
expiry_date: datetime = datetime.strptime(cert.get("notAfter"), '%b %d %H:%M:%S %Y GMT') # type: ignore
|
|
959
|
+
|
|
960
|
+
# Get today's date
|
|
961
|
+
today: datetime = datetime.today()
|
|
962
|
+
|
|
963
|
+
# Calculate the number of remaining days
|
|
964
|
+
remaining_days: int = (expiry_date - today).days
|
|
965
|
+
|
|
966
|
+
# Check if the certificate is still valid
|
|
967
|
+
is_valid: bool = remaining_days > 0
|
|
968
|
+
|
|
969
|
+
logging.info(msg=f"Certificate for {hostname} is valid: {is_valid}, remaining days: {remaining_days}")
|
|
970
|
+
return {
|
|
971
|
+
'is_valid': is_valid,
|
|
972
|
+
'remaining_days': remaining_days,
|
|
973
|
+
'expiry_date': expiry_date,
|
|
974
|
+
'error': None
|
|
975
|
+
} # type: ignore
|
|
976
|
+
|
|
977
|
+
except Exception as e:
|
|
978
|
+
logging.exception(msg=f"Error getting certificate info for {hostname}:\n{e}")
|
|
979
|
+
return {
|
|
980
|
+
'hostname': hostname,
|
|
981
|
+
'is_valid': False,
|
|
982
|
+
'remaining_days': 0,
|
|
983
|
+
'expiry_date': None,
|
|
984
|
+
'error': str(e)
|
|
985
|
+
} # type: ignore
|
|
986
|
+
|
|
987
|
+
finally:
|
|
988
|
+
conn.close()
|
|
989
|
+
|
|
990
|
+
def _check_certificate(self, hostname: str, display_name: str, warning_remaining_days: int, critical_remaining_days: int, display_only_if_critical: bool=False) -> str:
|
|
991
|
+
"""
|
|
992
|
+
Check the SSL certificate for a specific hostname.
|
|
993
|
+
|
|
994
|
+
:param hostname: The hostname to check the SSL certificate.
|
|
995
|
+
:param display_name: The name of the website to display in the output message.
|
|
996
|
+
:param warning_remaining_days: The warning remaining days for the SSL certificate.
|
|
997
|
+
:param critical_remaining_days: The critical remaining days for the SSL certificate.
|
|
998
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
999
|
+
|
|
1000
|
+
:return: A string containing the result message.
|
|
1001
|
+
"""
|
|
1002
|
+
out_msg: str = ""
|
|
1003
|
+
try:
|
|
1004
|
+
cert_infos = self._get_certificate_info(hostname=hostname) # type: ignore
|
|
1005
|
+
|
|
1006
|
+
remaining_days: int = 0
|
|
1007
|
+
if isinstance(cert_infos["remaining_days"], int):
|
|
1008
|
+
remaining_days = cert_infos["remaining_days"]
|
|
1009
|
+
|
|
1010
|
+
expiry_date: datetime = datetime.today()
|
|
1011
|
+
if isinstance(cert_infos["expiry_date"], datetime):
|
|
1012
|
+
expiry_date = cert_infos["expiry_date"]
|
|
1013
|
+
|
|
1014
|
+
if not cert_infos["is_valid"]:
|
|
1015
|
+
out_msg = f"- 🚨 **Invalid SSL certificate [{display_name}](https://{hostname})**\n" \
|
|
1016
|
+
f" - **Certificate expired on {expiry_date.strftime('%d/%m/%Y')}**\n" \
|
|
1017
|
+
f"⚠️ **Renew the SSL certificate immediately** ⚠️"
|
|
1018
|
+
logging.warning(msg=out_msg)
|
|
1019
|
+
elif critical_remaining_days > 0 and remaining_days < critical_remaining_days:
|
|
1020
|
+
out_msg = f"- 🚨 **Critical SSL certificate [{display_name}](https://{hostname})**:\n" \
|
|
1021
|
+
f" - Certificate expires on {expiry_date.strftime('%d/%m/%Y')}\n" \
|
|
1022
|
+
f" - **{remaining_days} remaining days**\n" \
|
|
1023
|
+
f"⚠️ **Renew the SSL certificate quickly** ⚠️"
|
|
1024
|
+
logging.warning(msg=out_msg)
|
|
1025
|
+
elif not display_only_if_critical:
|
|
1026
|
+
if warning_remaining_days > 0 and remaining_days < warning_remaining_days:
|
|
1027
|
+
icon: str = "⚠️ "
|
|
1028
|
+
elif critical_remaining_days <= 0 and warning_remaining_days <= 0:
|
|
1029
|
+
icon = ""
|
|
1030
|
+
else:
|
|
1031
|
+
icon = "✅ "
|
|
1032
|
+
|
|
1033
|
+
out_msg = f"{icon}[{display_name}](https://{hostname}): {remaining_days} remaining days (expires on {expiry_date.strftime('%d/%m/%Y')})"
|
|
1034
|
+
logging.info(msg=out_msg)
|
|
1035
|
+
|
|
1036
|
+
# Display the error of retrieving the certificate if there is one
|
|
1037
|
+
if cert_infos["error"]:
|
|
1038
|
+
out_msg += f":\n- Error: `{cert_infos['error']}`"
|
|
1039
|
+
except Exception as e:
|
|
1040
|
+
out_msg = f"⚠️ **Error checking SSL certificate of [{display_name}](https://{hostname})**:\n```sh\n{e}\n```"
|
|
1041
|
+
logging.exception(msg=out_msg)
|
|
1042
|
+
|
|
1043
|
+
return out_msg
|
|
1044
|
+
|
|
1045
|
+
def check_all_certificates(self, is_private: bool, display_only_if_critical: bool=False) -> str:
|
|
1046
|
+
"""
|
|
1047
|
+
Check all SSL certificates configured in the JSON configuration file.
|
|
1048
|
+
|
|
1049
|
+
:param is_private: User permission to check private or public certificates (True for private, False for public).
|
|
1050
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
1051
|
+
|
|
1052
|
+
:return: A string containing the result message.
|
|
1053
|
+
"""
|
|
1054
|
+
try:
|
|
1055
|
+
out_msg: str = ""
|
|
1056
|
+
for cert_config in self.config['certificates']:
|
|
1057
|
+
if is_private or is_private == cert_config['is_private']:
|
|
1058
|
+
hostname = cert_config['website']
|
|
1059
|
+
display_name = cert_config.get('display_name', hostname)
|
|
1060
|
+
# Verify that warning_remaining_days & critical_remaining_days exist, otherwise use -1
|
|
1061
|
+
warning_remaining_days = cert_config.get('warning_remaining_days', -1)
|
|
1062
|
+
critical_remaining_days = cert_config.get('critical_remaining_days', -1)
|
|
1063
|
+
|
|
1064
|
+
# We only check what is needed
|
|
1065
|
+
if not display_only_if_critical or (display_only_if_critical and critical_remaining_days != -1):
|
|
1066
|
+
result: str = self._check_certificate(hostname=hostname, display_name=display_name, warning_remaining_days=warning_remaining_days, critical_remaining_days=critical_remaining_days, display_only_if_critical=display_only_if_critical)
|
|
1067
|
+
if result and result != "":
|
|
1068
|
+
if out_msg != "":
|
|
1069
|
+
out_msg += "\n"
|
|
1070
|
+
out_msg += f"- {result}"
|
|
1071
|
+
|
|
1072
|
+
if out_msg != "":
|
|
1073
|
+
out_msg = f"# 🔐 Certificates 🔐\n{out_msg}"
|
|
1074
|
+
|
|
1075
|
+
return out_msg
|
|
1076
|
+
except Exception as e:
|
|
1077
|
+
return f"⚠️ **Error getting SSL certificates**:\n```sh\n{e}\n```"
|
|
1078
|
+
|
|
1079
|
+
#endregion
|
|
1080
|
+
|
|
1081
|
+
#region System Information
|
|
1082
|
+
|
|
1083
|
+
def get_hostname(self) -> str:
|
|
1084
|
+
"""
|
|
1085
|
+
Get the hostname of the system.
|
|
1086
|
+
|
|
1087
|
+
:return: A string containing the result message.
|
|
1088
|
+
"""
|
|
1089
|
+
out_msg: str = "# 🖥️ Hostname 🖥️\n"
|
|
1090
|
+
try:
|
|
1091
|
+
hostname = socket.gethostname()
|
|
1092
|
+
out_msg += f"- **{hostname}**"
|
|
1093
|
+
logging.info(msg=out_msg)
|
|
1094
|
+
except Exception as e:
|
|
1095
|
+
out_msg += f"⚠️ **Error getting hostname**:\n```sh\n{e}\n```"
|
|
1096
|
+
logging.exception(msg=out_msg)
|
|
1097
|
+
|
|
1098
|
+
return out_msg
|
|
1099
|
+
|
|
1100
|
+
def get_os_details(self) -> str:
|
|
1101
|
+
"""
|
|
1102
|
+
Get the OS details.
|
|
1103
|
+
|
|
1104
|
+
:return: A string containing the result message.
|
|
1105
|
+
"""
|
|
1106
|
+
out_msg: str = "# 🖥️ OS 🖥️\n"
|
|
1107
|
+
|
|
1108
|
+
# Get OS details
|
|
1109
|
+
try:
|
|
1110
|
+
if os.path.exists(path="/etc/os-release"):
|
|
1111
|
+
with open(file="/etc/os-release") as f:
|
|
1112
|
+
os_info = {}
|
|
1113
|
+
for line in f:
|
|
1114
|
+
key, value = line.rstrip().split(sep="=", maxsplit=1)
|
|
1115
|
+
os_info[key] = value.strip('"')
|
|
1116
|
+
os_version: str = f"{os_info.get('PRETTY_NAME', 'Unknown OS')}" # type: ignore
|
|
1117
|
+
out_msg += f"- **{os_version}**"
|
|
1118
|
+
else:
|
|
1119
|
+
# Fallback method if /etc/os-release is not available
|
|
1120
|
+
os_version = platform.platform()
|
|
1121
|
+
out_msg += f"- **{os_version}**"
|
|
1122
|
+
|
|
1123
|
+
logging.info(msg=out_msg)
|
|
1124
|
+
except Exception as e:
|
|
1125
|
+
out_msg += f"⚠️ **Error getting OS details**:\n```sh\n{e}\n```"
|
|
1126
|
+
logging.exception(msg=out_msg)
|
|
1127
|
+
|
|
1128
|
+
return out_msg
|
|
1129
|
+
|
|
1130
|
+
def get_kernel_version(self) -> str:
|
|
1131
|
+
"""
|
|
1132
|
+
Get the kernel version.
|
|
1133
|
+
|
|
1134
|
+
:return: A string containing the result message.
|
|
1135
|
+
"""
|
|
1136
|
+
out_msg: str = "# 🖥️ Kernel version 🖥️\n"
|
|
1137
|
+
try:
|
|
1138
|
+
# Get Kernel version
|
|
1139
|
+
kernel_version: str = subprocess.check_output(args="uname -r", shell=True).decode().strip()
|
|
1140
|
+
out_msg += f"- **{kernel_version}**"
|
|
1141
|
+
logging.info(msg=out_msg)
|
|
1142
|
+
except Exception as e:
|
|
1143
|
+
out_msg += f"⚠️ **Error getting kernel version**:\n```sh\n{e}\n```"
|
|
1144
|
+
logging.exception(msg=out_msg)
|
|
1145
|
+
|
|
1146
|
+
return out_msg
|
|
1147
|
+
|
|
1148
|
+
def get_server_datetime(self) -> str:
|
|
1149
|
+
"""
|
|
1150
|
+
Get the server date and time.
|
|
1151
|
+
|
|
1152
|
+
:return: A string containing the result message.
|
|
1153
|
+
"""
|
|
1154
|
+
out_msg: str = "# 🕒 Server datetime 🕒\n"
|
|
1155
|
+
try:
|
|
1156
|
+
# Get server date and time
|
|
1157
|
+
current_datetime: str = time.strftime('%d/%m/%Y %H:%M:%S', time.localtime())
|
|
1158
|
+
out_msg += f"- **{current_datetime}**"
|
|
1159
|
+
logging.info(msg=out_msg)
|
|
1160
|
+
except Exception as e:
|
|
1161
|
+
out_msg += f"⚠️ **Error getting server datetime**:\n```sh\n{e}\n```"
|
|
1162
|
+
logging.exception(msg=out_msg)
|
|
1163
|
+
|
|
1164
|
+
return out_msg
|
|
1165
|
+
|
|
1166
|
+
#endregion
|
|
1167
|
+
|
|
1168
|
+
#region Users
|
|
1169
|
+
|
|
1170
|
+
def get_connected_users(self) -> str:
|
|
1171
|
+
"""
|
|
1172
|
+
Get the connected users.
|
|
1173
|
+
|
|
1174
|
+
:return: A string containing the result message.
|
|
1175
|
+
"""
|
|
1176
|
+
out_msg: str = "# 👥 Connected users 👥\n"
|
|
1177
|
+
try:
|
|
1178
|
+
# Get connected users
|
|
1179
|
+
users = psutil.users()
|
|
1180
|
+
if users:
|
|
1181
|
+
for user in users:
|
|
1182
|
+
# Show username, IP address and login time
|
|
1183
|
+
out_msg += f"- **{user.name}** (since {time.strftime('%d/%m/%Y %H:%M:%S', time.localtime(user.started))})\n"
|
|
1184
|
+
else:
|
|
1185
|
+
out_msg += "- No user connected"
|
|
1186
|
+
|
|
1187
|
+
logging.info(msg=out_msg)
|
|
1188
|
+
except Exception as e:
|
|
1189
|
+
out_msg += f"⚠️ **Error getting connected users**:\n```sh\n{e}\n```"
|
|
1190
|
+
logging.exception(msg=out_msg)
|
|
1191
|
+
|
|
1192
|
+
return out_msg
|
|
1193
|
+
|
|
1194
|
+
def _get_recent_user_logins(self, days: int = 7) -> Optional[Dict[str, Set[str]]]:
|
|
1195
|
+
"""
|
|
1196
|
+
Get the recent user logins.
|
|
1197
|
+
|
|
1198
|
+
:param days: The number of days to look back for recent user logins.
|
|
1199
|
+
|
|
1200
|
+
:return: A dictionary containing the recent user logins.
|
|
1201
|
+
"""
|
|
1202
|
+
# Format the date without microseconds (milliseconds)
|
|
1203
|
+
past_date: str = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%dT%H:%M:%S')
|
|
1204
|
+
|
|
1205
|
+
try:
|
|
1206
|
+
# Fetch the logs since the specified date
|
|
1207
|
+
command: str = f"last --ip --hostlast --time-format iso --since '{past_date}' | grep 'pts/'"
|
|
1208
|
+
last_output: str = subprocess.check_output(
|
|
1209
|
+
args=command,
|
|
1210
|
+
shell=True, text=True
|
|
1211
|
+
).strip()
|
|
1212
|
+
except subprocess.CalledProcessError as e:
|
|
1213
|
+
logging.error(msg=f"Error getting recent user logins:\n{e}")
|
|
1214
|
+
return None
|
|
1215
|
+
|
|
1216
|
+
logins: List[str] = last_output.splitlines()
|
|
1217
|
+
user_ip_dict: Dict[str, Set[str]] = {}
|
|
1218
|
+
|
|
1219
|
+
# Regex pattern to match IP addresses
|
|
1220
|
+
ip_pattern: re.Pattern[str] = re.compile(pattern=r'\d+\.\d+\.\d+\.\d+')
|
|
1221
|
+
|
|
1222
|
+
for line in logins:
|
|
1223
|
+
parts: List[str] = line.split()
|
|
1224
|
+
if len(parts) > 4: # Ensure the line has enough parts
|
|
1225
|
+
username: str = parts[0]
|
|
1226
|
+
ip_address: str = parts[-1]
|
|
1227
|
+
|
|
1228
|
+
# Check if IP address is valid and not '0.0.0.0'
|
|
1229
|
+
if ip_pattern.match(string=ip_address) and ip_address != '0.0.0.0':
|
|
1230
|
+
if username in user_ip_dict:
|
|
1231
|
+
user_ip_dict[username].add(ip_address)
|
|
1232
|
+
else:
|
|
1233
|
+
user_ip_dict[username] = {ip_address}
|
|
1234
|
+
|
|
1235
|
+
logging.info(msg=f"Recent user logins: {user_ip_dict}")
|
|
1236
|
+
return user_ip_dict
|
|
1237
|
+
|
|
1238
|
+
def check_all_recent_user_logins(self, display_only_if_critical: bool=False) -> str:
|
|
1239
|
+
"""
|
|
1240
|
+
Check all recent user logins.
|
|
1241
|
+
|
|
1242
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
1243
|
+
|
|
1244
|
+
:return: A string containing the result message.
|
|
1245
|
+
"""
|
|
1246
|
+
try:
|
|
1247
|
+
out_msg: str = ""
|
|
1248
|
+
recent_user_connections_config = self.config.get('recent_user_connections', {})
|
|
1249
|
+
|
|
1250
|
+
max_days: int = 7
|
|
1251
|
+
allow_all: bool = True
|
|
1252
|
+
allowed_ips: List[str] = []
|
|
1253
|
+
|
|
1254
|
+
if recent_user_connections_config:
|
|
1255
|
+
max_days = recent_user_connections_config.get('max_days', 7)
|
|
1256
|
+
allow_all = recent_user_connections_config.get('allow_all', False)
|
|
1257
|
+
if not allow_all:
|
|
1258
|
+
allowed_ips = recent_user_connections_config.get('allowed_ips', [])
|
|
1259
|
+
else:
|
|
1260
|
+
logging.warning(msg="No recent_user_connections config found, using default values")
|
|
1261
|
+
|
|
1262
|
+
user_ip_dict: Optional[Dict[str, Set[str]]] = self._get_recent_user_logins(days=max_days)
|
|
1263
|
+
|
|
1264
|
+
if user_ip_dict is None:
|
|
1265
|
+
out_msg += "⚠️ **Error getting recent user logins**"
|
|
1266
|
+
logging.error(msg=out_msg)
|
|
1267
|
+
else:
|
|
1268
|
+
for user, ip_addresses in user_ip_dict.items():
|
|
1269
|
+
ip_str: str = ', '.join(ip_addresses)
|
|
1270
|
+
# if at least one ip not allowed, display as critical
|
|
1271
|
+
if not allow_all and any(ip not in allowed_ips for ip in ip_addresses):
|
|
1272
|
+
if out_msg != "":
|
|
1273
|
+
out_msg += "\n"
|
|
1274
|
+
|
|
1275
|
+
# Display in bold the invalid ips
|
|
1276
|
+
ip_str = ', '.join([f"**{ip} not allowed**" if ip not in allowed_ips else ip for ip in ip_addresses])
|
|
1277
|
+
|
|
1278
|
+
out_msg += f"- 🚨 **{user}** ({ip_str})"
|
|
1279
|
+
|
|
1280
|
+
logging.warning(msg=out_msg)
|
|
1281
|
+
elif not display_only_if_critical:
|
|
1282
|
+
if out_msg != "":
|
|
1283
|
+
out_msg += "\n"
|
|
1284
|
+
out_msg += f"- ✅ **{user}** ({ip_str})"
|
|
1285
|
+
logging.info(msg=out_msg)
|
|
1286
|
+
|
|
1287
|
+
if out_msg != "":
|
|
1288
|
+
out_msg = f"# 👥 User logins since {max_days} days 👥\n{out_msg}"
|
|
1289
|
+
|
|
1290
|
+
return out_msg
|
|
1291
|
+
except Exception as e:
|
|
1292
|
+
return f"⚠️ **Error getting recent user logins**:\n```sh\n{e}\n```"
|
|
1293
|
+
|
|
1294
|
+
#endregion
|
|
1295
|
+
|
|
1296
|
+
#region Ports
|
|
1297
|
+
|
|
1298
|
+
def _is_port_is_required_state(self, display_name: str, port: int, host: str = "localhost", timeout_in_sec: float = 2, want_port_to_be_open: bool = True) -> Tuple[bool, str]:
|
|
1299
|
+
"""
|
|
1300
|
+
Check if a specific port is in the required state (open or closed).
|
|
1301
|
+
|
|
1302
|
+
:param display_name: The name of the port to display in the output message.
|
|
1303
|
+
:param port: The port number to check.
|
|
1304
|
+
:param host: The host to check the port.
|
|
1305
|
+
:param timeout_in_sec: The timeout in seconds to check the port.
|
|
1306
|
+
:param want_port_to_be_open: If True, the port should be open, otherwise it should be closed.
|
|
1307
|
+
|
|
1308
|
+
:return: A tuple containing a boolean indicating if the port is in the required state and a string containing the result message.
|
|
1309
|
+
"""
|
|
1310
|
+
out_msg: str = ""
|
|
1311
|
+
|
|
1312
|
+
res: bool = False
|
|
1313
|
+
try:
|
|
1314
|
+
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
|
1315
|
+
sock.settimeout(timeout_in_sec)
|
|
1316
|
+
res_port_open: bool = sock.connect_ex((host, port)) == 0
|
|
1317
|
+
res = res_port_open == want_port_to_be_open
|
|
1318
|
+
|
|
1319
|
+
if res_port_open:
|
|
1320
|
+
if want_port_to_be_open:
|
|
1321
|
+
out_msg = f"✅🔓 {display_name} (Port {port}) **open**"
|
|
1322
|
+
logging.info(msg=out_msg)
|
|
1323
|
+
else:
|
|
1324
|
+
out_msg = f"❌🔓 {display_name} (Port {port}) **open (should be closed)**"
|
|
1325
|
+
logging.warning(msg=out_msg)
|
|
1326
|
+
else:
|
|
1327
|
+
if want_port_to_be_open:
|
|
1328
|
+
out_msg = f"❌🔒 {display_name} (Port {port}) **closed**"
|
|
1329
|
+
logging.warning(msg=out_msg)
|
|
1330
|
+
else:
|
|
1331
|
+
out_msg = f"✅🔒 {display_name} (Port {port}) **closed (and should be closed)**"
|
|
1332
|
+
logging.info(msg=out_msg)
|
|
1333
|
+
|
|
1334
|
+
sock.close()
|
|
1335
|
+
except Exception as e:
|
|
1336
|
+
out_msg = f"⚠️ **Error checking {display_name} port**:\n```sh\n{e}\n```"
|
|
1337
|
+
logging.exception(msg=out_msg)
|
|
1338
|
+
|
|
1339
|
+
return res, out_msg
|
|
1340
|
+
|
|
1341
|
+
def check_all_ports(self, is_private: bool, display_only_if_critical: bool=False, restart_if_down: bool=False) -> str:
|
|
1342
|
+
"""
|
|
1343
|
+
Check all ports configured in the JSON configuration file (and restart the service if the port is down and service_name_to_restart added in config file).
|
|
1344
|
+
|
|
1345
|
+
:param is_private: User permission to check private or public ports (True for private, False for public).
|
|
1346
|
+
:param display_only_if_critical: If True, the string result will only be returned if there is an error during execution.
|
|
1347
|
+
:param restart_if_down: If True, restart the service if the port is down.
|
|
1348
|
+
|
|
1349
|
+
:return: A string containing the result message.
|
|
1350
|
+
"""
|
|
1351
|
+
try:
|
|
1352
|
+
out_msg: str = ""
|
|
1353
|
+
for port_config in self.config['ports']:
|
|
1354
|
+
if is_private or is_private == port_config['is_private']:
|
|
1355
|
+
port: int = port_config['port']
|
|
1356
|
+
display_name: str = port_config.get('display_name', f"Port {port}")
|
|
1357
|
+
host: str = port_config.get('host', 'localhost')
|
|
1358
|
+
timeout_in_sec: float = port_config.get('timeout_in_sec', 2)
|
|
1359
|
+
want_port_to_be_open: bool = port_config.get('want_port_to_be_open', True)
|
|
1360
|
+
|
|
1361
|
+
service_name_to_restart: str = ""
|
|
1362
|
+
if want_port_to_be_open and restart_if_down:
|
|
1363
|
+
service_name_to_restart = port_config.get('service_name_to_restart', "")
|
|
1364
|
+
|
|
1365
|
+
result, result_msg = self._is_port_is_required_state(port=port, host=host, timeout_in_sec=timeout_in_sec, want_port_to_be_open=want_port_to_be_open, display_name=display_name)
|
|
1366
|
+
if result_msg != "" and (not display_only_if_critical or not result):
|
|
1367
|
+
if out_msg != "":
|
|
1368
|
+
out_msg += "\n"
|
|
1369
|
+
out_msg += f"- {result_msg}"
|
|
1370
|
+
|
|
1371
|
+
if (not result) and restart_if_down and service_name_to_restart != "":
|
|
1372
|
+
if out_msg != "":
|
|
1373
|
+
out_msg += "\n"
|
|
1374
|
+
|
|
1375
|
+
restart_res: str = self.restart_service(is_private=is_private, service_name=service_name_to_restart)
|
|
1376
|
+
out_msg += f" - {restart_res}"
|
|
1377
|
+
|
|
1378
|
+
if out_msg != "":
|
|
1379
|
+
out_msg = f"# 🛡️ Ports 🛡️\n{out_msg}"
|
|
1380
|
+
|
|
1381
|
+
return out_msg
|
|
1382
|
+
except Exception as e:
|
|
1383
|
+
return f"⚠️ **Error checking ports**:\n```sh\n{e}\n```"
|
|
1384
|
+
|
|
1385
|
+
#endregion
|
|
1386
|
+
|
|
1387
|
+
#region Processes
|
|
1388
|
+
|
|
1389
|
+
def get_ordered_processes(self, get_non_consuming_processes: bool = False) -> str:
|
|
1390
|
+
"""
|
|
1391
|
+
Get the ordered list of processes by memory and CPU usage.
|
|
1392
|
+
|
|
1393
|
+
:param get_non_consuming_processes: If True, get all processes, otherwise only get processes consuming resources.
|
|
1394
|
+
|
|
1395
|
+
:return: A string containing the result message.
|
|
1396
|
+
"""
|
|
1397
|
+
processes = []
|
|
1398
|
+
for process in psutil.process_iter(['pid', 'name', 'username', 'cpu_percent', 'memory_info', 'create_time', 'cmdline']):
|
|
1399
|
+
try:
|
|
1400
|
+
create_time = datetime.fromtimestamp(process.info['create_time']).strftime("%Y-%m-%d %H:%M:%S")
|
|
1401
|
+
|
|
1402
|
+
cmdline = ' '.join(process.info['cmdline']) # Join the command line arguments
|
|
1403
|
+
# Remove extra spaces
|
|
1404
|
+
cmdline = re.sub(r'\s+', ' ', cmdline).strip()
|
|
1405
|
+
if len(cmdline) > 60:
|
|
1406
|
+
cmdline = cmdline[:60] + '...' # Truncate and add ellipsis
|
|
1407
|
+
|
|
1408
|
+
processes.append({ # type: ignore
|
|
1409
|
+
'pid': process.info['pid'],
|
|
1410
|
+
'name': process.info['name'],
|
|
1411
|
+
'username': process.info['username'],
|
|
1412
|
+
'cpu_percent': process.info['cpu_percent'],
|
|
1413
|
+
'memory': process.info['memory_info'].rss, # Resident Set Size (RSS) memory
|
|
1414
|
+
'create_time': create_time,
|
|
1415
|
+
'cmdline': cmdline
|
|
1416
|
+
})
|
|
1417
|
+
except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess):
|
|
1418
|
+
pass
|
|
1419
|
+
|
|
1420
|
+
# Sort processes by memory usage, then by CPU usage, both in descending order
|
|
1421
|
+
processes.sort(key=lambda proc: (proc['memory'], proc['cpu_percent'], proc['create_time']), reverse=True) # type: ignore
|
|
1422
|
+
|
|
1423
|
+
full_res: str = ""
|
|
1424
|
+
for proc in processes: # type: ignore
|
|
1425
|
+
if not get_non_consuming_processes and proc['cpu_percent'] == 0 and proc['memory'] == 0:
|
|
1426
|
+
continue
|
|
1427
|
+
|
|
1428
|
+
res = f"- PID **{proc['pid']}**: {proc['name']} ("
|
|
1429
|
+
|
|
1430
|
+
if proc['cpu_percent'] > 0:
|
|
1431
|
+
res += f"CPU {proc['cpu_percent']}%, "
|
|
1432
|
+
|
|
1433
|
+
if proc['memory'] > 0:
|
|
1434
|
+
res += f"RAM {proc['memory'] // (1024 * 1024)} MB, "
|
|
1435
|
+
|
|
1436
|
+
res += f"👤 {proc['username']}, "
|
|
1437
|
+
res += f"⏰ {proc['create_time']}"
|
|
1438
|
+
|
|
1439
|
+
if proc['cmdline'] != "":
|
|
1440
|
+
res += f", 📄 `{proc['cmdline']}`"
|
|
1441
|
+
|
|
1442
|
+
res += ")"
|
|
1443
|
+
|
|
1444
|
+
if full_res != "":
|
|
1445
|
+
full_res += "\n"
|
|
1446
|
+
full_res += res
|
|
1447
|
+
|
|
1448
|
+
if full_res != "":
|
|
1449
|
+
full_res = f"# 🔄 Processes 🔄\n{full_res}"
|
|
1450
|
+
|
|
1451
|
+
logging.info(msg=full_res)
|
|
1452
|
+
return full_res
|
|
1453
|
+
|
|
1454
|
+
def kill_process(self, pid: int, timeout_in_sec: int = 10) -> str:
|
|
1455
|
+
"""
|
|
1456
|
+
Kills a process with the specified PID.
|
|
1457
|
+
1. Tries to terminate the process gracefully
|
|
1458
|
+
2. If the process is still running after the timeout, tries to kill it forcefully
|
|
1459
|
+
|
|
1460
|
+
:param pid: The process ID to kill
|
|
1461
|
+
:param timeout_in_sec: The timeout in seconds to wait for the process to terminate gracefully
|
|
1462
|
+
|
|
1463
|
+
:return: A message indicating the result of the operation
|
|
1464
|
+
|
|
1465
|
+
IMPORTANT:
|
|
1466
|
+
- The script must be allowed to use "sudo /bin/kill" without password prompt
|
|
1467
|
+
(e.g., by adding a sudoers file in /etc/sudoers.d/ with the following content: echo "USERNAME_HERE ALL=(ALL) NOPASSWD: /bin/kill" >> /etc/sudoers.d/USERNAME_HERE)
|
|
1468
|
+
"""
|
|
1469
|
+
try:
|
|
1470
|
+
out_msg: str = ""
|
|
1471
|
+
process = psutil.Process(pid=pid)
|
|
1472
|
+
|
|
1473
|
+
# Check if process exists
|
|
1474
|
+
if not process.is_running():
|
|
1475
|
+
out_msg = f"❌ **PID process {pid} not found**."
|
|
1476
|
+
logging.error(msg=out_msg)
|
|
1477
|
+
return out_msg
|
|
1478
|
+
|
|
1479
|
+
process_name: str = process.name()
|
|
1480
|
+
process_username: str = process.username()
|
|
1481
|
+
process_cpu_percent: float = process.cpu_percent()
|
|
1482
|
+
process_memory: int = process.memory_info().rss // (1024 * 1024)
|
|
1483
|
+
process_create_time: str = datetime.fromtimestamp(process.create_time()).strftime("%Y-%m-%d %H:%M:%S")
|
|
1484
|
+
|
|
1485
|
+
process_cmdline: str = ' '.join(process.cmdline())
|
|
1486
|
+
process_cmdline = re.sub(r'\s+', ' ', process_cmdline).strip()
|
|
1487
|
+
if len(process_cmdline) > 60:
|
|
1488
|
+
process_cmdline = process_cmdline[:60] + '...'
|
|
1489
|
+
|
|
1490
|
+
# Attempt to terminate the process
|
|
1491
|
+
terminate_command: List[str] = ['sudo', '/bin/kill', '-TERM', str(pid)]
|
|
1492
|
+
result_terminate, _ = self.execute_and_verify(command=terminate_command, display_name=f"stop process {pid} ({process_name})", timeout=timeout_in_sec, display_only_if_critical=False)
|
|
1493
|
+
|
|
1494
|
+
if result_terminate:
|
|
1495
|
+
out_msg = f"✅ **Process {pid} ({process_name}) stopped with success** (nicely)).\n"
|
|
1496
|
+
else:
|
|
1497
|
+
# Termination did not complete in time or failed
|
|
1498
|
+
out_msg = f"⚠️ **Stopping nicely {pid} ({process_name}) expired of failed**.\n"
|
|
1499
|
+
|
|
1500
|
+
# Attempt to kill the process
|
|
1501
|
+
kill_command = ['sudo', '/bin/kill', '-KILL', str(pid)]
|
|
1502
|
+
_, strerror_kill = self.execute_and_verify(command=kill_command, display_name=f"kill process {pid} ({process_name})", timeout=timeout_in_sec, display_only_if_critical=False)
|
|
1503
|
+
out_msg += strerror_kill
|
|
1504
|
+
|
|
1505
|
+
if process_cpu_percent > 0:
|
|
1506
|
+
out_msg += f"- Used CPU by this process: {process_cpu_percent}%\n"
|
|
1507
|
+
if process_memory > 0:
|
|
1508
|
+
out_msg += f"- Used RAM by this process: {process_memory} MB\n"
|
|
1509
|
+
out_msg += f"- 👤 User: {process_username}\n"
|
|
1510
|
+
out_msg += f"- ⏰ Launched at {process_create_time}\n"
|
|
1511
|
+
out_msg += f"- 📄 Command: `{process_cmdline}`"
|
|
1512
|
+
|
|
1513
|
+
logging.info(msg=out_msg)
|
|
1514
|
+
return out_msg
|
|
1515
|
+
except psutil.NoSuchProcess:
|
|
1516
|
+
out_msg = f"❌ **PID process {pid} doesn't exist anymore**.\n"
|
|
1517
|
+
logging.error(msg=out_msg)
|
|
1518
|
+
return out_msg
|
|
1519
|
+
except psutil.AccessDenied:
|
|
1520
|
+
out_msg = f"❌ **Access denied to stop PID process {pid}**. Execute this script with user allowed to access this process.\n"
|
|
1521
|
+
logging.error(msg=out_msg)
|
|
1522
|
+
return out_msg
|
|
1523
|
+
except Exception as e:
|
|
1524
|
+
out_msg = f"⚠️ **Erro while stopping PID {pid}**:\n```sh\n{e}\n```"
|
|
1525
|
+
logging.error(msg=out_msg)
|
|
1526
|
+
return out_msg
|
|
1527
|
+
|
|
1528
|
+
#endregion
|
|
1529
|
+
|
|
1530
|
+
# region Network Information
|
|
1531
|
+
|
|
1532
|
+
def get_network_info(self) -> str:
|
|
1533
|
+
"""
|
|
1534
|
+
Get the network information.
|
|
1535
|
+
|
|
1536
|
+
:return: A string containing the result message.
|
|
1537
|
+
"""
|
|
1538
|
+
interfaces = psutil.net_if_addrs()
|
|
1539
|
+
stats = psutil.net_if_stats()
|
|
1540
|
+
network_usage: str = ""
|
|
1541
|
+
|
|
1542
|
+
# List of interfaces to exclude (e.g., local interface)
|
|
1543
|
+
excluded_interfaces = {"lo"}
|
|
1544
|
+
|
|
1545
|
+
for interface in interfaces:
|
|
1546
|
+
if interface in excluded_interfaces or interface not in stats:
|
|
1547
|
+
continue
|
|
1548
|
+
|
|
1549
|
+
ip_addresses = []
|
|
1550
|
+
for snic in interfaces[interface]:
|
|
1551
|
+
if snic.family == socket.AF_INET:
|
|
1552
|
+
ip_addresses.append(snic.address) # type: ignore
|
|
1553
|
+
|
|
1554
|
+
# Get network stats
|
|
1555
|
+
net_stats = psutil.net_io_counters(pernic=True).get(interface, None)
|
|
1556
|
+
if net_stats is None:
|
|
1557
|
+
continue
|
|
1558
|
+
|
|
1559
|
+
# Convert bytes to GB for readability
|
|
1560
|
+
receive_bytes: float = net_stats.bytes_recv / (1024 ** 3)
|
|
1561
|
+
transmit_bytes: float = net_stats.bytes_sent / (1024 ** 3)
|
|
1562
|
+
|
|
1563
|
+
ip_str: str = ", ".join(ip_addresses) if ip_addresses else "N/A" # type: ignore
|
|
1564
|
+
|
|
1565
|
+
if network_usage != "":
|
|
1566
|
+
network_usage += "\n"
|
|
1567
|
+
network_usage += f"- {interface} ({ip_str}): ⬇️ {receive_bytes:,.2f} GB, ⬆️ {transmit_bytes:,.2f} GB"
|
|
1568
|
+
|
|
1569
|
+
if network_usage != "":
|
|
1570
|
+
network_usage = f"# 🌐 Network usage 🌐\n{network_usage}"
|
|
1571
|
+
|
|
1572
|
+
logging.info(msg=network_usage)
|
|
1573
|
+
return network_usage
|
|
1574
|
+
|
|
1575
|
+
#endregion
|
|
1576
|
+
|
|
1577
|
+
#region Scheduled Tasks
|
|
1578
|
+
|
|
1579
|
+
async def schedule_task(self, handle_error_message: Callable[[str], Awaitable[None]], is_private: bool) -> None:
|
|
1580
|
+
"""
|
|
1581
|
+
Schedule task to check for issues periodically.
|
|
1582
|
+
This function doesn't return anything, it will run indefinitely.
|
|
1583
|
+
|
|
1584
|
+
:param handle_error_message: The function to handle the error message.
|
|
1585
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
1586
|
+
"""
|
|
1587
|
+
logging.info(msg=f"Starting {'private' if is_private else 'public'} scheduled tasks every {self.duration_in_sec_wait_between_each_schedule_task_execution}sec for error handling purpose...")
|
|
1588
|
+
|
|
1589
|
+
if not self.allow_scheduled_tasks_check_for_issues:
|
|
1590
|
+
raise Exception("Scheduled tasks are not allowed")
|
|
1591
|
+
|
|
1592
|
+
await asyncio.sleep(delay=10) # Sleep for 10 sec before lauching the scheduled tasks (to allow the lib to be ready)
|
|
1593
|
+
|
|
1594
|
+
datetime_last_disk_usage_error_displayed: Union[datetime,None] = None
|
|
1595
|
+
datetime_last_folder_usage_error_displayed: Union[datetime,None] = None
|
|
1596
|
+
datetime_last_cpu_usage_error_displayed: Union[datetime,None] = None
|
|
1597
|
+
datetime_last_ram_usage_error_displayed: Union[datetime,None] = None
|
|
1598
|
+
datetime_last_swap_usage_error_displayed: Union[datetime,None] = None
|
|
1599
|
+
datetime_last_cpu_temperature_error_displayed: Union[datetime,None] = None
|
|
1600
|
+
datetime_last_ping_error_displayed: Union[datetime,None] = None
|
|
1601
|
+
datetime_last_certificates_error_displayed: Union[datetime,None] = None
|
|
1602
|
+
datetime_last_user_logins_error_displayed: Union[datetime,None] = None
|
|
1603
|
+
datetime_last_port_error_displayed: Union[datetime,None] = None
|
|
1604
|
+
datetime_last_services_error_displayed: Union[datetime,None] = None
|
|
1605
|
+
|
|
1606
|
+
if not self.start_scheduled_task_show_info_immediately:
|
|
1607
|
+
logging.info(msg=f"Waiting for {self.duration_in_sec_wait_between_each_schedule_task_execution} seconds before starting the execution of {'private' if is_private else 'public'} scheduled tasks...")
|
|
1608
|
+
await asyncio.sleep(delay=self.duration_in_sec_wait_between_each_schedule_task_execution)
|
|
1609
|
+
|
|
1610
|
+
while True:
|
|
1611
|
+
out_msg: str = ""
|
|
1612
|
+
logging.info(msg="-----------------------------------------------")
|
|
1613
|
+
logging.info(msg="Checking services status and all disk usage, CPU, RAM, Swap, CPU temperature and ping of websites periodically...")
|
|
1614
|
+
try:
|
|
1615
|
+
# Services status
|
|
1616
|
+
msg: str = self.check_all_services_status_and_restart_if_down(is_private=is_private)
|
|
1617
|
+
if msg != "":
|
|
1618
|
+
logging.warning(msg=msg)
|
|
1619
|
+
if datetime_last_services_error_displayed is None or ((datetime.now() - datetime_last_services_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1620
|
+
if out_msg != "":
|
|
1621
|
+
out_msg += "\n"
|
|
1622
|
+
out_msg += msg
|
|
1623
|
+
datetime_last_services_error_displayed = datetime.now()
|
|
1624
|
+
else:
|
|
1625
|
+
logging.warning(msg="Services critical but already notified less than 12 hours ago, not notifying...")
|
|
1626
|
+
elif datetime_last_services_error_displayed is not None:
|
|
1627
|
+
msg="✅ **All services returned to normal state**"
|
|
1628
|
+
logging.info(msg=msg)
|
|
1629
|
+
if out_msg != "":
|
|
1630
|
+
out_msg += "\n"
|
|
1631
|
+
out_msg += msg
|
|
1632
|
+
datetime_last_services_error_displayed = None
|
|
1633
|
+
else:
|
|
1634
|
+
logging.info(msg="- ✅ Private services are up and running.")
|
|
1635
|
+
|
|
1636
|
+
# Disk usage
|
|
1637
|
+
msg = self.check_all_disk_usage(is_private=is_private, display_only_if_critical=True)
|
|
1638
|
+
if msg != "":
|
|
1639
|
+
logging.warning(msg=msg)
|
|
1640
|
+
if datetime_last_disk_usage_error_displayed is None or ((datetime.now() - datetime_last_disk_usage_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1641
|
+
if out_msg != "":
|
|
1642
|
+
out_msg += "\n"
|
|
1643
|
+
out_msg += msg
|
|
1644
|
+
datetime_last_disk_usage_error_displayed = datetime.now()
|
|
1645
|
+
else:
|
|
1646
|
+
logging.warning(msg="Disk usage critical but already notified less than 12 hours ago, not notifying...")
|
|
1647
|
+
elif datetime_last_disk_usage_error_displayed is not None:
|
|
1648
|
+
msg="✅ **Disk space returned to normal state**"
|
|
1649
|
+
logging.info(msg=msg)
|
|
1650
|
+
if out_msg != "":
|
|
1651
|
+
out_msg += "\n"
|
|
1652
|
+
out_msg += msg
|
|
1653
|
+
datetime_last_disk_usage_error_displayed = None
|
|
1654
|
+
else:
|
|
1655
|
+
logging.info(msg="- ✅ All disk usage are OK.")
|
|
1656
|
+
|
|
1657
|
+
# Folder usage
|
|
1658
|
+
msg = self.check_all_folder_usage(is_private=is_private, display_only_if_critical=True)
|
|
1659
|
+
if msg != "":
|
|
1660
|
+
logging.warning(msg=msg)
|
|
1661
|
+
if datetime_last_folder_usage_error_displayed is None or ((datetime.now() - datetime_last_folder_usage_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1662
|
+
if out_msg != "":
|
|
1663
|
+
out_msg += "\n"
|
|
1664
|
+
out_msg += msg
|
|
1665
|
+
datetime_last_folder_usage_error_displayed = datetime.now()
|
|
1666
|
+
else:
|
|
1667
|
+
logging.warning(msg="Folder usage critical but already notified less than 12 hours ago, not notifying...")
|
|
1668
|
+
elif datetime_last_folder_usage_error_displayed is not None:
|
|
1669
|
+
msg="✅ **Folder space returned to normal state**"
|
|
1670
|
+
logging.info(msg=msg)
|
|
1671
|
+
if out_msg != "":
|
|
1672
|
+
out_msg += "\n"
|
|
1673
|
+
out_msg += msg
|
|
1674
|
+
datetime_last_folder_usage_error_displayed = None
|
|
1675
|
+
else:
|
|
1676
|
+
logging.info(msg="- ✅ All folder usage are OK.")
|
|
1677
|
+
|
|
1678
|
+
# Certificates
|
|
1679
|
+
msg = self.check_all_certificates(is_private=is_private, display_only_if_critical=True)
|
|
1680
|
+
if msg != "":
|
|
1681
|
+
logging.warning(msg=msg)
|
|
1682
|
+
if datetime_last_certificates_error_displayed is None or ((datetime.now() - datetime_last_certificates_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1683
|
+
if out_msg != "":
|
|
1684
|
+
out_msg += "\n"
|
|
1685
|
+
out_msg += msg
|
|
1686
|
+
datetime_last_certificates_error_displayed = datetime.now()
|
|
1687
|
+
else:
|
|
1688
|
+
logging.warning(msg="Certificates critical but already notified less than 12 hours ago, not notifying...")
|
|
1689
|
+
elif datetime_last_certificates_error_displayed is not None:
|
|
1690
|
+
msg="✅ **Certificates returned to normal state**"
|
|
1691
|
+
logging.info(msg=msg)
|
|
1692
|
+
if out_msg != "":
|
|
1693
|
+
out_msg += "\n"
|
|
1694
|
+
out_msg += msg
|
|
1695
|
+
datetime_last_certificates_error_displayed = None
|
|
1696
|
+
else:
|
|
1697
|
+
logging.info(msg="- ✅ Certificates are OK.")
|
|
1698
|
+
|
|
1699
|
+
# Ping
|
|
1700
|
+
msg = self.ping_all_websites(is_private=is_private, display_only_if_critical=True)
|
|
1701
|
+
if msg != "":
|
|
1702
|
+
logging.warning(msg=msg)
|
|
1703
|
+
if datetime_last_ping_error_displayed is None or ((datetime.now() - datetime_last_ping_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1704
|
+
if out_msg != "":
|
|
1705
|
+
out_msg += "\n"
|
|
1706
|
+
out_msg += msg
|
|
1707
|
+
datetime_last_ping_error_displayed = datetime.now()
|
|
1708
|
+
else:
|
|
1709
|
+
logging.warning(msg="Ping critical but already notified less than 12 hours ago, not notifying...")
|
|
1710
|
+
elif datetime_last_ping_error_displayed is not None:
|
|
1711
|
+
msg="✅ **Website ping returned to normal state**"
|
|
1712
|
+
logging.info(msg=msg)
|
|
1713
|
+
if out_msg != "":
|
|
1714
|
+
out_msg += "\n"
|
|
1715
|
+
out_msg += msg
|
|
1716
|
+
datetime_last_ping_error_displayed = None
|
|
1717
|
+
else:
|
|
1718
|
+
logging.info(msg="- ✅ Ping of all websites are OK.")
|
|
1719
|
+
|
|
1720
|
+
# Ports
|
|
1721
|
+
msg = self.check_all_ports(is_private=is_private, display_only_if_critical=True, restart_if_down=True)
|
|
1722
|
+
if msg != "":
|
|
1723
|
+
logging.warning(msg=msg)
|
|
1724
|
+
if datetime_last_port_error_displayed is None or ((datetime.now() - datetime_last_port_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1725
|
+
if out_msg != "":
|
|
1726
|
+
out_msg += "\n"
|
|
1727
|
+
out_msg += msg
|
|
1728
|
+
datetime_last_port_error_displayed = datetime.now()
|
|
1729
|
+
else:
|
|
1730
|
+
logging.warning(msg="Ports critical but already notified less than 12 hours ago, not notifying...")
|
|
1731
|
+
elif datetime_last_port_error_displayed is not None:
|
|
1732
|
+
msg="✅ **Port returned to normal state**"
|
|
1733
|
+
logging.info(msg=msg)
|
|
1734
|
+
if out_msg != "":
|
|
1735
|
+
out_msg += "\n"
|
|
1736
|
+
out_msg += msg
|
|
1737
|
+
datetime_last_port_error_displayed = None
|
|
1738
|
+
else:
|
|
1739
|
+
logging.info(msg="- ✅ All ports are OK.")
|
|
1740
|
+
|
|
1741
|
+
# User logins
|
|
1742
|
+
if is_private:
|
|
1743
|
+
msg = self.check_all_recent_user_logins(display_only_if_critical=True)
|
|
1744
|
+
if msg != "":
|
|
1745
|
+
logging.warning(msg=msg)
|
|
1746
|
+
if datetime_last_user_logins_error_displayed is None or ((datetime.now() - datetime_last_user_logins_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1747
|
+
if out_msg != "":
|
|
1748
|
+
out_msg += "\n"
|
|
1749
|
+
out_msg += msg
|
|
1750
|
+
datetime_last_user_logins_error_displayed = datetime.now()
|
|
1751
|
+
else:
|
|
1752
|
+
logging.warning(msg="User logins critical but already notified less than 12 hours ago, not notifying...")
|
|
1753
|
+
elif datetime_last_user_logins_error_displayed is not None:
|
|
1754
|
+
msg="✅ **Last user logins returned to normal state**"
|
|
1755
|
+
logging.info(msg=msg)
|
|
1756
|
+
if out_msg != "":
|
|
1757
|
+
out_msg += "\n"
|
|
1758
|
+
out_msg += msg
|
|
1759
|
+
datetime_last_user_logins_error_displayed = None
|
|
1760
|
+
else:
|
|
1761
|
+
logging.info(msg="- ✅ All user logins are OK.")
|
|
1762
|
+
|
|
1763
|
+
# CPU
|
|
1764
|
+
if is_private:
|
|
1765
|
+
msg = self.check_cpu_usage(display_only_if_critical=True)
|
|
1766
|
+
if msg != "":
|
|
1767
|
+
logging.warning(msg=msg)
|
|
1768
|
+
if datetime_last_cpu_usage_error_displayed is None or ((datetime.now() - datetime_last_cpu_usage_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1769
|
+
if out_msg != "":
|
|
1770
|
+
out_msg += "\n"
|
|
1771
|
+
out_msg += msg
|
|
1772
|
+
datetime_last_cpu_usage_error_displayed = datetime.now()
|
|
1773
|
+
else:
|
|
1774
|
+
logging.warning(msg="CPU usage critical but already notified less than 12 hours ago, not notifying...")
|
|
1775
|
+
elif datetime_last_cpu_usage_error_displayed is not None:
|
|
1776
|
+
msg = "✅ **CPU usage returned to normal state**"
|
|
1777
|
+
logging.info(msg=msg)
|
|
1778
|
+
if out_msg != "":
|
|
1779
|
+
out_msg += "\n"
|
|
1780
|
+
out_msg += msg
|
|
1781
|
+
datetime_last_cpu_usage_error_displayed = None
|
|
1782
|
+
else:
|
|
1783
|
+
logging.info(msg="- ✅ CPU usage is OK.")
|
|
1784
|
+
|
|
1785
|
+
# RAM
|
|
1786
|
+
if is_private:
|
|
1787
|
+
msg = self.check_ram_usage(display_only_if_critical=True)
|
|
1788
|
+
if msg != "":
|
|
1789
|
+
logging.warning(msg=msg)
|
|
1790
|
+
if datetime_last_ram_usage_error_displayed is None or ((datetime.now() - datetime_last_ram_usage_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1791
|
+
if out_msg != "":
|
|
1792
|
+
out_msg += "\n"
|
|
1793
|
+
out_msg += msg
|
|
1794
|
+
datetime_last_ram_usage_error_displayed = datetime.now()
|
|
1795
|
+
else:
|
|
1796
|
+
logging.warning(msg="RAM usage critical but already notified less than 12 hours ago, not notifying...")
|
|
1797
|
+
elif datetime_last_ram_usage_error_displayed is not None:
|
|
1798
|
+
msg = "✅ **RAM usage returned to normal state**"
|
|
1799
|
+
logging.info(msg=msg)
|
|
1800
|
+
if out_msg != "":
|
|
1801
|
+
out_msg += "\n"
|
|
1802
|
+
out_msg += msg
|
|
1803
|
+
datetime_last_ram_usage_error_displayed = None
|
|
1804
|
+
else:
|
|
1805
|
+
logging.info(msg="- ✅ RAM usage is OK.")
|
|
1806
|
+
|
|
1807
|
+
# Swap
|
|
1808
|
+
if is_private:
|
|
1809
|
+
msg = self.check_swap_usage(display_only_if_critical=True)
|
|
1810
|
+
if msg != "":
|
|
1811
|
+
logging.warning(msg=msg)
|
|
1812
|
+
if datetime_last_swap_usage_error_displayed is None or ((datetime.now() - datetime_last_swap_usage_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1813
|
+
if out_msg != "":
|
|
1814
|
+
out_msg += "\n"
|
|
1815
|
+
out_msg += msg
|
|
1816
|
+
datetime_last_swap_usage_error_displayed = datetime.now()
|
|
1817
|
+
else:
|
|
1818
|
+
logging.warning(msg="Swap usage critical but already notified less than 12 hours ago, not notifying...")
|
|
1819
|
+
elif datetime_last_swap_usage_error_displayed is not None:
|
|
1820
|
+
msg = "✅ **SWAP usage returned to normal state**"
|
|
1821
|
+
logging.info(msg=msg)
|
|
1822
|
+
if out_msg != "":
|
|
1823
|
+
out_msg += "\n"
|
|
1824
|
+
out_msg += msg
|
|
1825
|
+
datetime_last_swap_usage_error_displayed = None
|
|
1826
|
+
else:
|
|
1827
|
+
logging.info(msg="- ✅ Swap usage is OK.")
|
|
1828
|
+
|
|
1829
|
+
# CPU temperature
|
|
1830
|
+
if is_private:
|
|
1831
|
+
msg = self.check_cpu_temperature(display_only_if_critical=True)
|
|
1832
|
+
if msg != "":
|
|
1833
|
+
logging.warning(msg=msg)
|
|
1834
|
+
if datetime_last_cpu_temperature_error_displayed is None or ((datetime.now() - datetime_last_cpu_temperature_error_displayed).total_seconds() > self.max_duration_seconds_showing_same_error_again_in_scheduled_tasks):
|
|
1835
|
+
if out_msg != "":
|
|
1836
|
+
out_msg += "\n"
|
|
1837
|
+
out_msg += msg
|
|
1838
|
+
datetime_last_cpu_temperature_error_displayed = datetime.now()
|
|
1839
|
+
else:
|
|
1840
|
+
logging.warning(msg="CPU temperature critical but already notified less than 12 hours ago, not notifying...")
|
|
1841
|
+
elif datetime_last_cpu_temperature_error_displayed is not None:
|
|
1842
|
+
msg = "✅ **CPU temperature returned to normal state**"
|
|
1843
|
+
logging.info(msg=msg)
|
|
1844
|
+
if out_msg != "":
|
|
1845
|
+
out_msg += "\n"
|
|
1846
|
+
out_msg += msg
|
|
1847
|
+
datetime_last_cpu_temperature_error_displayed = None
|
|
1848
|
+
else:
|
|
1849
|
+
logging.info(msg="- ✅ CPU temperature is OK.")
|
|
1850
|
+
except Exception as e:
|
|
1851
|
+
out_msg = f"**Internal error during periodic server check task**:\n```sh\n{e}\n```"
|
|
1852
|
+
logging.exception(msg=out_msg)
|
|
1853
|
+
|
|
1854
|
+
logging.info(msg="-----------------------------------------------")
|
|
1855
|
+
|
|
1856
|
+
if out_msg != "":
|
|
1857
|
+
# Send the message
|
|
1858
|
+
await handle_error_message(out_msg)
|
|
1859
|
+
|
|
1860
|
+
logging.info(msg=f"Waiting {self.duration_in_sec_wait_between_each_schedule_task_execution} seconds before next execution of {'private' if is_private else 'public'} scheduled tasks...")
|
|
1861
|
+
await asyncio.sleep(delay=self.duration_in_sec_wait_between_each_schedule_task_execution)
|
|
1862
|
+
|
|
1863
|
+
async def schedule_task_show_info(self, show_message: Callable[[str], Awaitable[None]], is_private: bool) -> None:
|
|
1864
|
+
"""
|
|
1865
|
+
Schedule task to show system information periodically.
|
|
1866
|
+
This function doesn't return anything, it will run indefinitely.
|
|
1867
|
+
|
|
1868
|
+
:param show_message: The function to show the message.
|
|
1869
|
+
:param is_private: User permission to check private or public services (True for private, False for public).
|
|
1870
|
+
"""
|
|
1871
|
+
logging.info(msg=f"Starting {'private' if is_private else 'public'} scheduled tasks every {self.duration_in_sec_wait_between_each_schedule_task_show_info_execution}sec for information purpose...")
|
|
1872
|
+
|
|
1873
|
+
if not self.allow_scheduled_task_show_info:
|
|
1874
|
+
raise Exception("Scheduled show info tasks are not allowed")
|
|
1875
|
+
|
|
1876
|
+
await asyncio.sleep(delay=10)
|
|
1877
|
+
|
|
1878
|
+
if not self.start_scheduled_task_show_info_immediately:
|
|
1879
|
+
logging.info(msg=f"Waiting for {self.duration_in_sec_wait_between_each_schedule_task_show_info_execution} seconds before starting the execution of {'private' if is_private else 'public'} show info scheduled tasks...")
|
|
1880
|
+
await asyncio.sleep(delay=self.duration_in_sec_wait_between_each_schedule_task_show_info_execution)
|
|
1881
|
+
|
|
1882
|
+
while True:
|
|
1883
|
+
try:
|
|
1884
|
+
out_msg: str = ""
|
|
1885
|
+
logging.info(msg="-----------------------------------------------")
|
|
1886
|
+
|
|
1887
|
+
# Services status
|
|
1888
|
+
msg: str = self.check_all_services_status_and_restart_if_down(is_private=is_private)
|
|
1889
|
+
if msg != "":
|
|
1890
|
+
if out_msg != "":
|
|
1891
|
+
out_msg += "\n"
|
|
1892
|
+
out_msg += msg
|
|
1893
|
+
|
|
1894
|
+
# Disk usage
|
|
1895
|
+
msg = self.check_all_disk_usage(is_private=is_private, display_only_if_critical=False)
|
|
1896
|
+
if msg != "":
|
|
1897
|
+
if out_msg != "":
|
|
1898
|
+
out_msg += "\n"
|
|
1899
|
+
out_msg += msg
|
|
1900
|
+
|
|
1901
|
+
# Folder usage
|
|
1902
|
+
msg = self.check_all_folder_usage(is_private=is_private, display_only_if_critical=False)
|
|
1903
|
+
if msg != "":
|
|
1904
|
+
if out_msg != "":
|
|
1905
|
+
out_msg += "\n"
|
|
1906
|
+
out_msg += msg
|
|
1907
|
+
|
|
1908
|
+
# Certificates
|
|
1909
|
+
msg = self.check_all_certificates(is_private=is_private, display_only_if_critical=False)
|
|
1910
|
+
if msg != "":
|
|
1911
|
+
if out_msg != "":
|
|
1912
|
+
out_msg += "\n"
|
|
1913
|
+
out_msg += msg
|
|
1914
|
+
|
|
1915
|
+
# Ping
|
|
1916
|
+
msg = self.ping_all_websites(is_private=is_private, display_only_if_critical=False)
|
|
1917
|
+
if msg != "":
|
|
1918
|
+
if out_msg != "":
|
|
1919
|
+
out_msg += "\n"
|
|
1920
|
+
out_msg += msg
|
|
1921
|
+
|
|
1922
|
+
# Ports
|
|
1923
|
+
msg = self.check_all_ports(is_private=is_private, display_only_if_critical=False, restart_if_down=False)
|
|
1924
|
+
if msg != "":
|
|
1925
|
+
if out_msg != "":
|
|
1926
|
+
out_msg += "\n"
|
|
1927
|
+
out_msg += msg
|
|
1928
|
+
|
|
1929
|
+
# User logins
|
|
1930
|
+
if is_private:
|
|
1931
|
+
msg = self.check_all_recent_user_logins(display_only_if_critical=False)
|
|
1932
|
+
if msg != "":
|
|
1933
|
+
if out_msg != "":
|
|
1934
|
+
out_msg += "\n"
|
|
1935
|
+
out_msg += msg
|
|
1936
|
+
|
|
1937
|
+
# CPU
|
|
1938
|
+
if is_private:
|
|
1939
|
+
msg = self.check_cpu_usage(display_only_if_critical=False)
|
|
1940
|
+
if msg != "":
|
|
1941
|
+
if out_msg != "":
|
|
1942
|
+
out_msg += "\n"
|
|
1943
|
+
out_msg += msg
|
|
1944
|
+
|
|
1945
|
+
# RAM
|
|
1946
|
+
if is_private:
|
|
1947
|
+
msg = self.check_ram_usage(display_only_if_critical=False)
|
|
1948
|
+
if msg != "":
|
|
1949
|
+
if out_msg != "":
|
|
1950
|
+
out_msg += "\n"
|
|
1951
|
+
out_msg += msg
|
|
1952
|
+
|
|
1953
|
+
# Swap
|
|
1954
|
+
if is_private:
|
|
1955
|
+
msg = self.check_swap_usage(display_only_if_critical=False)
|
|
1956
|
+
if msg != "":
|
|
1957
|
+
if out_msg != "":
|
|
1958
|
+
out_msg += "\n"
|
|
1959
|
+
out_msg += msg
|
|
1960
|
+
|
|
1961
|
+
# CPU temperature
|
|
1962
|
+
if is_private:
|
|
1963
|
+
msg = self.check_cpu_temperature(display_only_if_critical=False)
|
|
1964
|
+
if msg != "":
|
|
1965
|
+
if out_msg != "":
|
|
1966
|
+
out_msg += "\n"
|
|
1967
|
+
out_msg += msg
|
|
1968
|
+
|
|
1969
|
+
if out_msg != "":
|
|
1970
|
+
out_msg = f"# System state at {time.strftime('%d-%m-%Y %H:%M:%S', time.localtime())}\n{out_msg}"
|
|
1971
|
+
await show_message(out_msg)
|
|
1972
|
+
|
|
1973
|
+
logging.info(msg="-----------------------------------------------")
|
|
1974
|
+
except Exception as e:
|
|
1975
|
+
out_msg = f"**Internal error during periodic server show info task**:\n```sh\n{e}\n```"
|
|
1976
|
+
logging.exception(msg=out_msg)
|
|
1977
|
+
|
|
1978
|
+
logging.info(msg=f"Waiting {self.duration_in_sec_wait_between_each_schedule_task_show_info_execution} seconds before next execution of {'private' if is_private else 'public'} show info scheduled tasks...")
|
|
1979
|
+
await asyncio.sleep(delay=self.duration_in_sec_wait_between_each_schedule_task_show_info_execution)
|
|
1980
|
+
|
|
1981
|
+
#endregion
|
|
1982
|
+
|
|
1983
|
+
def main() -> None:
|
|
1984
|
+
parser = argparse.ArgumentParser(description='System Management CLI Tool')
|
|
1985
|
+
|
|
1986
|
+
# Define available arguments
|
|
1987
|
+
parser.add_argument('--config_file', type=str, required=True, help='Path to the configuration file (must be a JSON file)')
|
|
1988
|
+
parser.add_argument('--start_scheduled_task_check_for_issues', action='store_true', help='Start periodic task to show potential issues periodically (in background, will not stop until you stop the script)')
|
|
1989
|
+
parser.add_argument('--start_scheduled_task_show_info', action='store_true', help='Start periodic task to show system information periodically (in background, will not stop until you stop the script)')
|
|
1990
|
+
parser.add_argument('--usage', action='store_true', help='📊 View disk space, CPU, RAM, ... 📊')
|
|
1991
|
+
parser.add_argument('--os_infos', action='store_true', help='🖥 View basic system information 🖥')
|
|
1992
|
+
parser.add_argument('--users', action='store_true', help='👥 View connected users 👥')
|
|
1993
|
+
parser.add_argument('--user_logins', action='store_true', help='👥 View last user connections 👥')
|
|
1994
|
+
parser.add_argument('--ping', action='store_true', help='🌐 Ping websites 🌐')
|
|
1995
|
+
parser.add_argument('--certificates', action='store_true', help='🔒 Check SSL certificates 🔒')
|
|
1996
|
+
parser.add_argument('--reboot_server', action='store_true', help='🔄 Restart the entire server 🔄')
|
|
1997
|
+
parser.add_argument('--services_status', action='store_true', help='🩺 Check services are running 🩺')
|
|
1998
|
+
parser.add_argument('--restart_all', action='store_true', help='🚀 Restart all services 🚀')
|
|
1999
|
+
parser.add_argument('--restart_service', type=str, help='🚀 Restart a service 🚀')
|
|
2000
|
+
parser.add_argument('--list_services', action='store_true', help='📋 List all available services 📋')
|
|
2001
|
+
parser.add_argument('--ports', action='store_true', help='🔒 Check ports 🔒')
|
|
2002
|
+
parser.add_argument('--list_processes', action='store_true', help='📋 List active processes 📋')
|
|
2003
|
+
parser.add_argument('--kill_process', type=int, help='🚫 Stop a process by PID 🚫')
|
|
2004
|
+
|
|
2005
|
+
parser.add_argument('--debug', action='store_true', help='Enable debug mode')
|
|
2006
|
+
parser.add_argument('--nodebug', action='store_true', help='Disable all logs')
|
|
2007
|
+
|
|
2008
|
+
# Parse arguments
|
|
2009
|
+
args = parser.parse_args()
|
|
2010
|
+
|
|
2011
|
+
# Enable or disable debug mode
|
|
2012
|
+
if args.debug:
|
|
2013
|
+
logging.getLogger().setLevel(logging.DEBUG)
|
|
2014
|
+
elif args.nodebug:
|
|
2015
|
+
logging.getLogger().setLevel(logging.CRITICAL)
|
|
2016
|
+
else:
|
|
2017
|
+
# Default to warning
|
|
2018
|
+
logging.getLogger().setLevel(logging.WARNING)
|
|
2019
|
+
|
|
2020
|
+
# Ensure the config file is provided
|
|
2021
|
+
if not args.config_file:
|
|
2022
|
+
print("Error: --config_file is required")
|
|
2023
|
+
parser.print_help()
|
|
2024
|
+
sys.exit(1)
|
|
2025
|
+
|
|
2026
|
+
allow_scheduled_tasks_check_for_issues: bool = args.start_scheduled_task_check_for_issues is not None
|
|
2027
|
+
allow_scheduled_task_show_info: bool = args.start_scheduled_task_show_info is not None
|
|
2028
|
+
monitoring = LinuxMonitor(config_file=args.config_file, allow_scheduled_tasks_check_for_issues=allow_scheduled_tasks_check_for_issues, allow_scheduled_task_show_info=allow_scheduled_task_show_info)
|
|
2029
|
+
|
|
2030
|
+
# Handle all commands
|
|
2031
|
+
handled: bool = False
|
|
2032
|
+
|
|
2033
|
+
if args.usage:
|
|
2034
|
+
handled = True
|
|
2035
|
+
print("Viewing disk space, CPU, RAM, ...")
|
|
2036
|
+
out_msg: str = monitoring.check_all_disk_usage(is_private=True, display_only_if_critical=False)
|
|
2037
|
+
msg: str = monitoring.check_all_folder_usage(is_private=True, display_only_if_critical=False)
|
|
2038
|
+
if msg != "":
|
|
2039
|
+
if out_msg != "":
|
|
2040
|
+
out_msg += "\n"
|
|
2041
|
+
out_msg += msg
|
|
2042
|
+
|
|
2043
|
+
out_msg += "\n"
|
|
2044
|
+
out_msg += monitoring.check_cpu_usage(display_only_if_critical=False) + "\n"
|
|
2045
|
+
out_msg += monitoring.check_ram_usage(display_only_if_critical=False) + "\n"
|
|
2046
|
+
out_msg += monitoring.check_swap_usage(display_only_if_critical=False) + "\n"
|
|
2047
|
+
out_msg += monitoring.check_cpu_temperature(display_only_if_critical=False) + "\n"
|
|
2048
|
+
out_msg += monitoring.get_network_info()
|
|
2049
|
+
print(out_msg)
|
|
2050
|
+
|
|
2051
|
+
if args.os_infos:
|
|
2052
|
+
handled = True
|
|
2053
|
+
print("Viewing basic system information...")
|
|
2054
|
+
out_msg: str = monitoring.get_hostname() + "\n"
|
|
2055
|
+
out_msg += monitoring.get_os_details() + "\n"
|
|
2056
|
+
out_msg += monitoring.get_kernel_version() + "\n"
|
|
2057
|
+
out_msg += monitoring.get_uptime() + "\n"
|
|
2058
|
+
out_msg += monitoring.get_server_datetime()
|
|
2059
|
+
print(out_msg)
|
|
2060
|
+
|
|
2061
|
+
if args.users:
|
|
2062
|
+
handled = True
|
|
2063
|
+
print("Viewing connected users...")
|
|
2064
|
+
out_msg: str = monitoring.get_connected_users()
|
|
2065
|
+
print(out_msg)
|
|
2066
|
+
|
|
2067
|
+
if args.user_logins:
|
|
2068
|
+
handled = True
|
|
2069
|
+
print("Viewing last user connections...")
|
|
2070
|
+
out_msg: str = monitoring.check_all_recent_user_logins(display_only_if_critical=False)
|
|
2071
|
+
print(out_msg)
|
|
2072
|
+
|
|
2073
|
+
if args.ping:
|
|
2074
|
+
handled = True
|
|
2075
|
+
print("Pinging websites...")
|
|
2076
|
+
out_msg: str = monitoring.ping_all_websites(is_private=True, display_only_if_critical=False)
|
|
2077
|
+
print(out_msg)
|
|
2078
|
+
|
|
2079
|
+
if args.certificates:
|
|
2080
|
+
handled = True
|
|
2081
|
+
print("Checking SSL certificates...")
|
|
2082
|
+
out_msg: str = monitoring.check_all_certificates(is_private=True, display_only_if_critical=False)
|
|
2083
|
+
print(out_msg)
|
|
2084
|
+
|
|
2085
|
+
if args.reboot_server:
|
|
2086
|
+
handled = True
|
|
2087
|
+
print("Restarting the entire server...")
|
|
2088
|
+
out_msg: str = monitoring.reboot_server()
|
|
2089
|
+
print(out_msg)
|
|
2090
|
+
|
|
2091
|
+
if args.services_status:
|
|
2092
|
+
handled = True
|
|
2093
|
+
print("Checking if services are running and restart if down...")
|
|
2094
|
+
out_msg: str = monitoring.check_all_services_status_and_restart_if_down(is_private=True)
|
|
2095
|
+
print(out_msg)
|
|
2096
|
+
|
|
2097
|
+
if args.restart_all:
|
|
2098
|
+
handled = True
|
|
2099
|
+
print("Restarting all services...")
|
|
2100
|
+
out_msg: str = monitoring.restart_all_services(is_private=True)
|
|
2101
|
+
|
|
2102
|
+
if args.restart_service is not None:
|
|
2103
|
+
handled = True
|
|
2104
|
+
print(f"Restarting service: {args.restart_service}...")
|
|
2105
|
+
out_msg: str = monitoring.restart_service(is_private=True, service_name=args.restart_service)
|
|
2106
|
+
print(out_msg)
|
|
2107
|
+
|
|
2108
|
+
if args.list_services:
|
|
2109
|
+
handled = True
|
|
2110
|
+
print("Listing all available services...")
|
|
2111
|
+
out_msg: str = monitoring.get_all_services_allowed_to_restart(is_private=True)
|
|
2112
|
+
print(out_msg)
|
|
2113
|
+
|
|
2114
|
+
if args.ports:
|
|
2115
|
+
handled = True
|
|
2116
|
+
print("Checking ports...")
|
|
2117
|
+
out_msg: str = monitoring.check_all_ports(is_private=True, display_only_if_critical=False, restart_if_down=False)
|
|
2118
|
+
print(out_msg)
|
|
2119
|
+
|
|
2120
|
+
if args.list_processes:
|
|
2121
|
+
handled = True
|
|
2122
|
+
print("Listing active processes...")
|
|
2123
|
+
out_msg: str = monitoring.get_ordered_processes(get_non_consuming_processes=False)
|
|
2124
|
+
print(out_msg)
|
|
2125
|
+
|
|
2126
|
+
if args.kill_process is not None:
|
|
2127
|
+
handled = True
|
|
2128
|
+
print(f"Stopping process with PID: {args.kill_process}...")
|
|
2129
|
+
out_msg: str = monitoring.kill_process(pid=args.kill_process)
|
|
2130
|
+
print(out_msg)
|
|
2131
|
+
|
|
2132
|
+
if args.start_scheduled_task_check_for_issues:
|
|
2133
|
+
handled = True
|
|
2134
|
+
print("Starting periodic task (will show something only in case of error (or if debug enabled))...")
|
|
2135
|
+
async def async_print(msg: str) -> None:
|
|
2136
|
+
print(msg)
|
|
2137
|
+
asyncio.run(monitoring.schedule_task(handle_error_message=async_print, is_private=True))
|
|
2138
|
+
|
|
2139
|
+
if args.start_scheduled_task_show_info:
|
|
2140
|
+
handled = True
|
|
2141
|
+
print("Starting periodic task (will show system information periodically)...")
|
|
2142
|
+
async def async_print(msg: str) -> None:
|
|
2143
|
+
print(msg)
|
|
2144
|
+
asyncio.run(monitoring.schedule_task_show_info(show_message=async_print, is_private=True))
|
|
2145
|
+
|
|
2146
|
+
# Show help if no command was provided or the command was not recognized
|
|
2147
|
+
if not handled:
|
|
2148
|
+
parser.print_help()
|
|
2149
|
+
|
|
2150
|
+
if __name__ == "__main__":
|
|
2151
|
+
main()
|