psrsentinel 1.0.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. psrsentinel-1.0.3/PKG-INFO +18 -0
  2. psrsentinel-1.0.3/pyproject.toml +37 -0
  3. psrsentinel-1.0.3/setup.cfg +4 -0
  4. psrsentinel-1.0.3/src/psrsentinel/__init__.py +1 -0
  5. psrsentinel-1.0.3/src/psrsentinel/__main__.py +4 -0
  6. psrsentinel-1.0.3/src/psrsentinel/api/__init__.py +0 -0
  7. psrsentinel-1.0.3/src/psrsentinel/api/adapter.py +44 -0
  8. psrsentinel-1.0.3/src/psrsentinel/api/client.py +41 -0
  9. psrsentinel-1.0.3/src/psrsentinel/app.py +183 -0
  10. psrsentinel-1.0.3/src/psrsentinel/checks/__init__.py +0 -0
  11. psrsentinel-1.0.3/src/psrsentinel/checks/load.py +41 -0
  12. psrsentinel-1.0.3/src/psrsentinel/checks/memory.py +53 -0
  13. psrsentinel-1.0.3/src/psrsentinel/checks/mounts.py +58 -0
  14. psrsentinel-1.0.3/src/psrsentinel/checks/processes.py +87 -0
  15. psrsentinel-1.0.3/src/psrsentinel/checks/services.py +39 -0
  16. psrsentinel-1.0.3/src/psrsentinel/checks/storage.py +100 -0
  17. psrsentinel-1.0.3/src/psrsentinel/checks/time_sync.py +56 -0
  18. psrsentinel-1.0.3/src/psrsentinel/command.py +38 -0
  19. psrsentinel-1.0.3/src/psrsentinel/config.py +25 -0
  20. psrsentinel-1.0.3/src/psrsentinel/hostname.py +44 -0
  21. psrsentinel-1.0.3/src/psrsentinel/scheduler.py +34 -0
  22. psrsentinel-1.0.3/src/psrsentinel/state.py +94 -0
  23. psrsentinel-1.0.3/src/psrsentinel.egg-info/PKG-INFO +18 -0
  24. psrsentinel-1.0.3/src/psrsentinel.egg-info/SOURCES.txt +28 -0
  25. psrsentinel-1.0.3/src/psrsentinel.egg-info/dependency_links.txt +1 -0
  26. psrsentinel-1.0.3/src/psrsentinel.egg-info/entry_points.txt +2 -0
  27. psrsentinel-1.0.3/src/psrsentinel.egg-info/requires.txt +18 -0
  28. psrsentinel-1.0.3/src/psrsentinel.egg-info/top_level.txt +1 -0
  29. psrsentinel-1.0.3/tests/test_hostname.py +11 -0
  30. psrsentinel-1.0.3/tests/test_state.py +26 -0
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.4
2
+ Name: psrsentinel
3
+ Version: 1.0.3
4
+ Summary: Jodrell bank pulsar server monitor
5
+ Author: Benjamin Shaw
6
+ Requires-Python: >=3.6
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: PyYAML<6.0,>=3.13; python_version < "3.10"
9
+ Requires-Dist: PyYAML>=6.0; python_version >= "3.10"
10
+ Requires-Dist: typing_extensions>=4.0; python_version < "3.8"
11
+ Provides-Extra: dev
12
+ Requires-Dist: pytest; extra == "dev"
13
+ Requires-Dist: tox; extra == "dev"
14
+ Requires-Dist: isort; extra == "dev"
15
+ Requires-Dist: flake8; extra == "dev"
16
+ Requires-Dist: black; extra == "dev"
17
+ Requires-Dist: build; extra == "dev"
18
+ Requires-Dist: twine; extra == "dev"
@@ -0,0 +1,37 @@
1
+ [build-system]
2
+ requires = ["setuptools>=68", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "psrsentinel"
7
+ version = "1.0.3"
8
+ description = "Jodrell bank pulsar server monitor"
9
+ readme = "README.md"
10
+ requires-python = ">=3.6"
11
+ license = { file = "LICENSE" }
12
+ authors = [{ name = "Benjamin Shaw" }]
13
+ dependencies = [
14
+ "PyYAML>=3.13,<6.0; python_version < '3.10'",
15
+ "PyYAML>=6.0; python_version >= '3.10'",
16
+ "typing_extensions>=4.0; python_version<'3.8'"
17
+ ]
18
+
19
+ [project.optional-dependencies]
20
+ dev = [
21
+ "pytest",
22
+ "tox",
23
+ "isort",
24
+ "flake8",
25
+ "black",
26
+ "build",
27
+ "twine",
28
+ ]
29
+
30
+ [project.scripts]
31
+ psrsentinel = "psrsentinel.app:main"
32
+
33
+ [tool.setuptools]
34
+ package-dir = {"" = "src"}
35
+
36
+ [tool.setuptools.packages.find]
37
+ where = ["src"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,4 @@
1
+ from psrsentinel.app import main
2
+
3
+ if __name__ == "__main__":
4
+ main()
File without changes
@@ -0,0 +1,44 @@
1
+ import socket
2
+
3
+ from psrsentinel import __version__
4
+ from psrsentinel.state import (
5
+ collect_states,
6
+ combine_states,
7
+ iso_after_seconds,
8
+ make_summary,
9
+ map_to_psrmon_state,
10
+ utc_now_iso,
11
+ )
12
+
13
+
14
+ def make_psrmon_payload(machine, checks, polling_config, collector_config):
15
+ internal_state = combine_states(collect_states(checks))
16
+ hostname = machine.get("hostname") or socket.gethostname()
17
+ module_id = machine.get("module_id", "host-{0}".format(hostname))
18
+ report_timeout = int(polling_config.get("report_timeout_seconds", 45))
19
+
20
+ return {
21
+ "module_id": module_id,
22
+ "module_type": "host",
23
+ "host": hostname,
24
+ "timestamp": utc_now_iso(),
25
+ "timeout_timestamp": iso_after_seconds(report_timeout),
26
+ "overall_state": map_to_psrmon_state(internal_state),
27
+ "summary": make_summary(checks),
28
+ "diagnostics_ok": internal_state != "UNKNOWN",
29
+ "payload_version": "1.0",
30
+ "data": {
31
+ "collector": {
32
+ "type": "local_agent",
33
+ "host": socket.gethostname(),
34
+ "version": __version__,
35
+ "config_version": collector_config.get(
36
+ "config_version", "unknown"
37
+ ),
38
+ "agent": "psrsentinel",
39
+ },
40
+ "role": machine.get("role", "unknown"),
41
+ "location": machine.get("location", "unknown"),
42
+ "checks": checks,
43
+ },
44
+ }
@@ -0,0 +1,41 @@
1
+ import json
2
+ from urllib import error, request
3
+
4
+
5
+ class PostError(RuntimeError):
6
+ pass
7
+
8
+
9
+ def post_payload(base_url, api_key, payload, timeout_seconds=5):
10
+ module_id = payload["module_id"]
11
+ endpoint = "{0}/api/modules/{1}/state".format(
12
+ base_url.rstrip("/"), module_id
13
+ )
14
+
15
+ body = json.dumps(payload, separators=(",", ":")).encode("utf-8")
16
+
17
+ headers = {
18
+ "Content-Type": "application/json",
19
+ }
20
+ if api_key:
21
+ headers["Authorization"] = "Bearer {0}".format(api_key)
22
+
23
+ req = request.Request(endpoint, data=body, headers=headers, method="POST")
24
+
25
+ try:
26
+ with request.urlopen(req, timeout=timeout_seconds) as response:
27
+ raw = response.read().decode("utf-8")
28
+ if not raw:
29
+ return {}
30
+ try:
31
+ return json.loads(raw)
32
+ except ValueError:
33
+ return {"raw_response": raw}
34
+ except error.HTTPError as exc:
35
+ try:
36
+ detail = exc.read().decode("utf-8")
37
+ except Exception:
38
+ detail = str(exc)
39
+ raise PostError("HTTP {0}: {1}".format(exc.code, detail))
40
+ except error.URLError as exc:
41
+ raise PostError(str(exc))
@@ -0,0 +1,183 @@
1
+ import argparse
2
+ import json
3
+ import sys
4
+ import time
5
+
6
+ from psrsentinel.api.adapter import make_psrmon_payload
7
+ from psrsentinel.api.client import post_payload
8
+ from psrsentinel.checks.load import check_load
9
+ from psrsentinel.checks.memory import check_memory
10
+ from psrsentinel.checks.mounts import check_expected_mounts
11
+ from psrsentinel.checks.processes import check_expected_processes
12
+ from psrsentinel.checks.services import check_expected_services
13
+ from psrsentinel.checks.storage import check_storage
14
+ from psrsentinel.checks.time_sync import check_time_sync
15
+ from psrsentinel.config import get_api_key, load_config
16
+ from psrsentinel.hostname import find_machine_config
17
+ from psrsentinel.scheduler import (
18
+ CheckSchedule,
19
+ get_interval_seconds,
20
+ shortest_interval,
21
+ )
22
+
23
+ CHECKS = {
24
+ "expected_processes": check_expected_processes,
25
+ "expected_services": check_expected_services,
26
+ "expected_mounts": check_expected_mounts,
27
+ "storage": check_storage,
28
+ "memory": check_memory,
29
+ "load": check_load,
30
+ "time_sync": check_time_sync,
31
+ }
32
+
33
+
34
+ class Agent(object):
35
+ def __init__(self, config, machine, no_post=False, pretty=False):
36
+ self.config = config
37
+ self.machine = machine
38
+ self.no_post = no_post
39
+ self.pretty = pretty
40
+ self.schedule = CheckSchedule()
41
+ self.latest_checks = {}
42
+
43
+ def run_once(self, force_all=False):
44
+ checks_config = self.machine.get("checks", {})
45
+ intervals = self.config.get("polling", {}).get("intervals", {})
46
+ ran_any = False
47
+
48
+ for check_name, check_config in checks_config.items():
49
+ if not check_config.get("enabled", False):
50
+ continue
51
+
52
+ due = force_all
53
+ if not due:
54
+ interval = get_interval_seconds(check_config, intervals)
55
+ due = self.schedule.should_run(check_name, interval)
56
+
57
+ if not due:
58
+ continue
59
+
60
+ self.latest_checks[check_name] = self._run_check(
61
+ check_name, check_config
62
+ )
63
+ self.schedule.mark_run(check_name)
64
+ ran_any = True
65
+
66
+ if ran_any or force_all:
67
+ payload = make_psrmon_payload(
68
+ self.machine,
69
+ self.latest_checks,
70
+ self.config.get("polling", {}),
71
+ self.config.get("collector", {}),
72
+ )
73
+ self.emit(payload)
74
+ return payload
75
+
76
+ return None
77
+
78
+ def run_forever(self):
79
+ intervals = self.config.get("polling", {}).get("intervals", {})
80
+ sleep_seconds = shortest_interval(intervals)
81
+
82
+ # Prime the cache with all enabled checks, then move to due-only checks.
83
+ self.run_once(force_all=True)
84
+
85
+ while True:
86
+ started = time.monotonic()
87
+ self.run_once(force_all=False)
88
+ elapsed = time.monotonic() - started
89
+ time.sleep(max(1, sleep_seconds - elapsed))
90
+
91
+ def _run_check(self, check_name, check_config):
92
+ func = CHECKS.get(check_name)
93
+ if func is None:
94
+ return {
95
+ "state": "UNKNOWN",
96
+ "message": "No check registered for {0}".format(check_name),
97
+ }
98
+
99
+ try:
100
+ return func(check_config)
101
+ except Exception as exc:
102
+ return {
103
+ "state": "UNKNOWN",
104
+ "message": "Check {0} raised {1}: {2}".format(
105
+ check_name, type(exc).__name__, exc
106
+ ),
107
+ }
108
+
109
+ def emit(self, payload):
110
+ if self.pretty:
111
+ print(json.dumps(payload, indent=2, sort_keys=True))
112
+ else:
113
+ print(json.dumps(payload, sort_keys=True))
114
+ sys.stdout.flush()
115
+
116
+ api_config = self.config.get("api", {})
117
+ if self.no_post or not api_config.get("enabled", False):
118
+ return
119
+
120
+ api_key = get_api_key(api_config)
121
+ try:
122
+ post_payload(
123
+ api_config["base_url"],
124
+ api_key,
125
+ payload,
126
+ timeout_seconds=int(api_config.get("timeout_seconds", 5)),
127
+ )
128
+ except Exception as exc:
129
+ print("POST failed: {0}".format(exc), file=sys.stderr)
130
+ sys.stderr.flush()
131
+
132
+
133
+ def main():
134
+ parser = argparse.ArgumentParser(
135
+ description="Local psrsentinel health agent"
136
+ )
137
+ parser.add_argument(
138
+ "--config",
139
+ default="config/machines.yaml",
140
+ help="Path to shared YAML config",
141
+ )
142
+ parser.add_argument(
143
+ "--hostname",
144
+ help="Override local hostname when selecting machine config",
145
+ )
146
+ parser.add_argument(
147
+ "--once",
148
+ action="store_true",
149
+ help="Run all enabled checks once and exit",
150
+ )
151
+ parser.add_argument(
152
+ "--no-post",
153
+ action="store_true",
154
+ help="Do not POST to psrmon even if API is enabled",
155
+ )
156
+ parser.add_argument(
157
+ "--pretty", action="store_true", help="Pretty-print JSON"
158
+ )
159
+ parser.add_argument(
160
+ "--show-machine",
161
+ action="store_true",
162
+ help="Print the selected machine config and exit",
163
+ )
164
+
165
+ args = parser.parse_args()
166
+
167
+ config = load_config(args.config)
168
+ machine = find_machine_config(config, args.hostname)
169
+
170
+ if args.show_machine:
171
+ print(json.dumps(machine, indent=2, sort_keys=True))
172
+ return
173
+
174
+ agent = Agent(config, machine, no_post=args.no_post, pretty=args.pretty)
175
+
176
+ if args.once:
177
+ agent.run_once(force_all=True)
178
+ else:
179
+ agent.run_forever()
180
+
181
+
182
+ if __name__ == "__main__":
183
+ main()
File without changes
@@ -0,0 +1,41 @@
1
+ import os
2
+
3
+
4
+ def check_load(check_config):
5
+ try:
6
+ load_1, load_5, load_15 = os.getloadavg()
7
+ except OSError as exc:
8
+ return {"state": "UNKNOWN", "message": str(exc)}
9
+
10
+ cores = os.cpu_count() or 1
11
+ warn_per_core = float(check_config.get("warn_1min_per_core", 1.5))
12
+ fail_per_core = float(check_config.get("fail_1min_per_core", 3.0))
13
+ load_per_core = load_1 / float(cores)
14
+
15
+ if load_per_core >= fail_per_core:
16
+ state = "FAILED"
17
+ message = (
18
+ "1-minute load per core {0:.2f} >= fail threshold {1:.2f}".format(
19
+ load_per_core, fail_per_core
20
+ )
21
+ )
22
+ elif load_per_core >= warn_per_core:
23
+ state = "DEGRADED"
24
+ message = (
25
+ "1-minute load per core {0:.2f} >= warn threshold {1:.2f}".format(
26
+ load_per_core, warn_per_core
27
+ )
28
+ )
29
+ else:
30
+ state = "OK"
31
+ message = None
32
+
33
+ return {
34
+ "state": state,
35
+ "load_average": [load_1, load_5, load_15],
36
+ "cores": cores,
37
+ "load_1min_per_core": load_per_core,
38
+ "warn_1min_per_core": warn_per_core,
39
+ "fail_1min_per_core": fail_per_core,
40
+ "message": message,
41
+ }
@@ -0,0 +1,53 @@
1
+ from psrsentinel.command import run_command
2
+
3
+
4
+ def check_memory(check_config):
5
+ result = run_command(["free", "-m"])
6
+ if result.returncode != 0:
7
+ return {
8
+ "state": "UNKNOWN",
9
+ "message": result.stderr or "Could not run free",
10
+ }
11
+
12
+ parsed = parse_free_m(result.stdout)
13
+ if parsed is None:
14
+ return {"state": "UNKNOWN", "message": "Could not parse free output"}
15
+
16
+ min_available_mb = int(check_config.get("min_available_mb", 1024))
17
+ available_mb = parsed["available_mb"]
18
+
19
+ if available_mb < min_available_mb:
20
+ state = "DEGRADED"
21
+ message = "Available memory {0} MB < threshold {1} MB".format(
22
+ available_mb, min_available_mb
23
+ )
24
+ else:
25
+ state = "OK"
26
+ message = None
27
+
28
+ return {
29
+ "state": state,
30
+ "total_mb": parsed["total_mb"],
31
+ "used_mb": parsed["used_mb"],
32
+ "available_mb": available_mb,
33
+ "min_available_mb": min_available_mb,
34
+ "message": message,
35
+ }
36
+
37
+
38
+ def parse_free_m(output):
39
+ for line in output.splitlines():
40
+ if not line.startswith("Mem:"):
41
+ continue
42
+ parts = line.split()
43
+ if len(parts) < 7:
44
+ return None
45
+ try:
46
+ return {
47
+ "total_mb": int(parts[1]),
48
+ "used_mb": int(parts[2]),
49
+ "available_mb": int(parts[6]),
50
+ }
51
+ except ValueError:
52
+ return None
53
+ return None
@@ -0,0 +1,58 @@
1
+ import os
2
+
3
+
4
+ def check_expected_mounts(check_config):
5
+ items = []
6
+ for item in check_config.get("items", []) or []:
7
+ items.append(check_path(item))
8
+
9
+ states = [item.get("state", "UNKNOWN") for item in items]
10
+ if "FAILED" in states:
11
+ state = "FAILED"
12
+ elif "DEGRADED" in states:
13
+ state = "DEGRADED"
14
+ elif "UNKNOWN" in states:
15
+ state = "UNKNOWN"
16
+ else:
17
+ state = "OK"
18
+
19
+ return {"state": state, "items": items}
20
+
21
+
22
+ def check_path(item):
23
+ path = item["path"]
24
+ required = bool(item.get("required", True))
25
+ must_exist = bool(item.get("must_exist", True))
26
+ must_be_readable = bool(item.get("must_be_readable", False))
27
+ must_be_writable = bool(item.get("must_be_writable", False))
28
+
29
+ exists = os.path.exists(path)
30
+ readable = os.access(path, os.R_OK)
31
+ writable = os.access(path, os.W_OK)
32
+ mountpoint = os.path.ismount(path)
33
+
34
+ failures = []
35
+ if must_exist and not exists:
36
+ failures.append("path does not exist")
37
+ if must_be_readable and not readable:
38
+ failures.append("path is not readable")
39
+ if must_be_writable and not writable:
40
+ failures.append("path is not writable")
41
+
42
+ if failures:
43
+ state = "FAILED" if required else "DEGRADED"
44
+ message = ", ".join(failures)
45
+ else:
46
+ state = "OK"
47
+ message = None
48
+
49
+ return {
50
+ "path": path,
51
+ "state": state,
52
+ "required": required,
53
+ "exists": exists,
54
+ "readable": readable,
55
+ "writable": writable,
56
+ "is_mountpoint": mountpoint,
57
+ "message": message,
58
+ }
@@ -0,0 +1,87 @@
1
+ from psrsentinel.command import run_command
2
+
3
+
4
+ def check_expected_processes(check_config):
5
+ items = []
6
+ for proc in check_config.get("items", []) or []:
7
+ items.append(check_expected_process(proc))
8
+
9
+ states = [item.get("state", "UNKNOWN") for item in items]
10
+ if "FAILED" in states:
11
+ state = "FAILED"
12
+ elif "DEGRADED" in states:
13
+ state = "DEGRADED"
14
+ elif "UNKNOWN" in states:
15
+ state = "UNKNOWN"
16
+ else:
17
+ state = "OK"
18
+
19
+ return {"state": state, "items": items}
20
+
21
+
22
+ def check_expected_process(proc):
23
+ pattern = proc["pattern"]
24
+ min_count = int(proc.get("min_count", 1))
25
+ max_count = proc.get("max_count")
26
+ if max_count is not None:
27
+ max_count = int(max_count)
28
+ required = bool(proc.get("required", True))
29
+
30
+ result = run_command(["pgrep", "-af", pattern])
31
+
32
+ if result.returncode not in (0, 1):
33
+ return {
34
+ "name": proc.get("name", pattern),
35
+ "pattern": pattern,
36
+ "state": "UNKNOWN",
37
+ "expected": required,
38
+ "running": False,
39
+ "match_count": 0,
40
+ "matches": [],
41
+ "message": result.stderr or "Could not run pgrep",
42
+ }
43
+
44
+ matches = []
45
+ if result.returncode == 0 and result.stdout:
46
+ for line in result.stdout.splitlines():
47
+ parts = line.split(None, 1)
48
+ if not parts:
49
+ continue
50
+ try:
51
+ pid = int(parts[0])
52
+ except ValueError:
53
+ continue
54
+ command = parts[1] if len(parts) > 1 else ""
55
+ matches.append({"pid": pid, "command": command})
56
+
57
+ count = len(matches)
58
+ if count < min_count:
59
+ state = "FAILED" if required else "DEGRADED"
60
+ message = (
61
+ "Expected at least {0} matching process(es), found {1}".format(
62
+ min_count, count
63
+ )
64
+ )
65
+ elif max_count is not None and count > max_count:
66
+ state = "DEGRADED"
67
+ message = (
68
+ "Expected at most {0} matching process(es), found {1}".format(
69
+ max_count, count
70
+ )
71
+ )
72
+ else:
73
+ state = "OK"
74
+ message = None
75
+
76
+ return {
77
+ "name": proc.get("name", pattern),
78
+ "pattern": pattern,
79
+ "state": state,
80
+ "expected": required,
81
+ "running": count >= min_count,
82
+ "match_count": count,
83
+ "min_count": min_count,
84
+ "max_count": max_count,
85
+ "matches": matches,
86
+ "message": message,
87
+ }
@@ -0,0 +1,39 @@
1
+ from psrsentinel.command import run_command
2
+
3
+
4
+ def check_expected_services(check_config):
5
+ items = []
6
+ for service in check_config.get("items", []) or []:
7
+ name = service["name"]
8
+ required = bool(service.get("required", True))
9
+ result = run_command(["systemctl", "is-active", name])
10
+ active = result.stdout.strip() == "active"
11
+
12
+ if active:
13
+ state = "OK"
14
+ message = None
15
+ else:
16
+ state = "FAILED" if required else "DEGRADED"
17
+ message = result.stdout or result.stderr or "Service is not active"
18
+
19
+ items.append(
20
+ {
21
+ "name": name,
22
+ "state": state,
23
+ "active": active,
24
+ "required": required,
25
+ "message": message,
26
+ }
27
+ )
28
+
29
+ states = [item.get("state", "UNKNOWN") for item in items]
30
+ if "FAILED" in states:
31
+ state = "FAILED"
32
+ elif "DEGRADED" in states:
33
+ state = "DEGRADED"
34
+ elif "UNKNOWN" in states:
35
+ state = "UNKNOWN"
36
+ else:
37
+ state = "OK"
38
+
39
+ return {"state": state, "items": items}
@@ -0,0 +1,100 @@
1
+ from psrsentinel.command import run_command
2
+
3
+
4
+ def check_storage(check_config):
5
+ wanted = {}
6
+ for item in check_config.get("filesystems", []) or []:
7
+ wanted[item["mount"]] = item
8
+
9
+ result = run_command(
10
+ ["df", "-B1", "--output=target,size,used,avail,pcent"]
11
+ )
12
+ if result.returncode != 0:
13
+ return {
14
+ "state": "UNKNOWN",
15
+ "message": result.stderr or "Could not run df",
16
+ "filesystems": [],
17
+ }
18
+
19
+ observed = parse_df_output(result.stdout)
20
+ filesystems = []
21
+
22
+ for mount, thresholds in wanted.items():
23
+ fs = observed.get(mount)
24
+ if fs is None:
25
+ filesystems.append(
26
+ {
27
+ "mount": mount,
28
+ "state": "FAILED",
29
+ "present": False,
30
+ "message": "Expected filesystem is not present",
31
+ }
32
+ )
33
+ continue
34
+
35
+ warn = float(thresholds.get("warn_used_percent", 85))
36
+ fail = float(thresholds.get("fail_used_percent", 95))
37
+ used_percent = fs["used_percent"]
38
+
39
+ if used_percent >= fail:
40
+ state = "FAILED"
41
+ message = (
42
+ "Filesystem usage {0:.1f}% >= fail threshold {1:.1f}%".format(
43
+ used_percent, fail
44
+ )
45
+ )
46
+ elif used_percent >= warn:
47
+ state = "DEGRADED"
48
+ message = (
49
+ "Filesystem usage {0:.1f}% >= warn threshold {1:.1f}%".format(
50
+ used_percent, warn
51
+ )
52
+ )
53
+ else:
54
+ state = "OK"
55
+ message = None
56
+
57
+ fs = dict(fs)
58
+ fs.update(
59
+ {
60
+ "present": True,
61
+ "state": state,
62
+ "warn_used_percent": warn,
63
+ "fail_used_percent": fail,
64
+ "message": message,
65
+ }
66
+ )
67
+ filesystems.append(fs)
68
+
69
+ states = [item.get("state", "UNKNOWN") for item in filesystems]
70
+ if "FAILED" in states:
71
+ state = "FAILED"
72
+ elif "DEGRADED" in states:
73
+ state = "DEGRADED"
74
+ elif "UNKNOWN" in states:
75
+ state = "UNKNOWN"
76
+ else:
77
+ state = "OK"
78
+
79
+ return {"state": state, "filesystems": filesystems}
80
+
81
+
82
+ def parse_df_output(output):
83
+ filesystems = {}
84
+ for line in output.splitlines()[1:]:
85
+ parts = line.split()
86
+ if len(parts) != 5:
87
+ continue
88
+ mount, size, used, avail, percent = parts
89
+ try:
90
+ used_percent = float(percent.rstrip("%"))
91
+ filesystems[mount] = {
92
+ "mount": mount,
93
+ "size_bytes": int(size),
94
+ "used_bytes": int(used),
95
+ "available_bytes": int(avail),
96
+ "used_percent": used_percent,
97
+ }
98
+ except ValueError:
99
+ continue
100
+ return filesystems
@@ -0,0 +1,56 @@
1
+ from psrsentinel.command import run_command
2
+
3
+
4
+ def check_time_sync(check_config):
5
+ required = bool(check_config.get("required", True))
6
+
7
+ result = run_command(
8
+ ["timedatectl", "show", "-p", "NTPSynchronized", "--value"]
9
+ )
10
+ if result.returncode == 0:
11
+ value = result.stdout.strip().lower()
12
+ synchronised = value == "yes"
13
+ return make_result(synchronised, required, None)
14
+
15
+ # Fallback for machines without systemd/timedatectl but with chrony.
16
+ chrony = run_command(["chronyc", "tracking"])
17
+ if chrony.returncode == 0:
18
+ # chronyc returns useful info only when it can talk to chronyd. Treat that
19
+ # as at least a successful diagnostic, and include the raw tracking text.
20
+ return {
21
+ "state": "OK",
22
+ "synchronised": True,
23
+ "required": required,
24
+ "method": "chronyc tracking",
25
+ "message": None,
26
+ "details": chrony.stdout,
27
+ }
28
+
29
+ return {
30
+ "state": "UNKNOWN",
31
+ "synchronised": None,
32
+ "required": required,
33
+ "message": result.stderr
34
+ or chrony.stderr
35
+ or "Could not check time synchronisation",
36
+ }
37
+
38
+
39
+ def make_result(synchronised, required, details):
40
+ if synchronised:
41
+ state = "OK"
42
+ message = None
43
+ else:
44
+ state = "FAILED" if required else "DEGRADED"
45
+ message = "System clock is not NTP synchronised"
46
+
47
+ result = {
48
+ "state": state,
49
+ "synchronised": synchronised,
50
+ "required": required,
51
+ "method": "timedatectl",
52
+ "message": message,
53
+ }
54
+ if details is not None:
55
+ result["details"] = details
56
+ return result
@@ -0,0 +1,38 @@
1
+ import subprocess
2
+
3
+
4
+ class CommandResult(object):
5
+ def __init__(self, command, returncode, stdout, stderr):
6
+ self.command = command
7
+ self.returncode = returncode
8
+ self.stdout = stdout
9
+ self.stderr = stderr
10
+
11
+
12
+ def run_command(args, timeout_seconds=8):
13
+ try:
14
+ proc = subprocess.Popen(
15
+ args,
16
+ stdout=subprocess.PIPE,
17
+ stderr=subprocess.PIPE,
18
+ universal_newlines=True,
19
+ )
20
+ try:
21
+ stdout, stderr = proc.communicate(timeout=timeout_seconds)
22
+ except TypeError:
23
+ # Very old Python fallback. Python 3.6 supports timeout, but this
24
+ # keeps the helper boring if it is ever run under an older 3.x.
25
+ stdout, stderr = proc.communicate()
26
+ except subprocess.TimeoutExpired:
27
+ proc.kill()
28
+ stdout, stderr = proc.communicate()
29
+ return CommandResult(
30
+ args, 124, stdout.strip(), "Command timed out"
31
+ )
32
+
33
+ return CommandResult(
34
+ args, proc.returncode, stdout.strip(), stderr.strip()
35
+ )
36
+
37
+ except OSError as exc:
38
+ return CommandResult(args, 127, "", str(exc))
@@ -0,0 +1,25 @@
1
+ import os
2
+
3
+ import yaml
4
+
5
+
6
+ def load_config(path):
7
+ with open(path, "r") as handle:
8
+ config = yaml.safe_load(handle)
9
+
10
+ if not isinstance(config, dict):
11
+ raise ValueError("Config file must contain a YAML mapping")
12
+
13
+ if "machines" not in config or not isinstance(config["machines"], list):
14
+ raise ValueError("Config file must contain a top-level machines list")
15
+
16
+ return config
17
+
18
+
19
+ def get_api_key(api_config):
20
+ env_name = api_config.get("api_key_env")
21
+ if env_name:
22
+ value = os.environ.get(env_name)
23
+ if value:
24
+ return value
25
+ return api_config.get("api_key")
@@ -0,0 +1,44 @@
1
+ import socket
2
+
3
+
4
+ def get_local_names():
5
+ names = set()
6
+
7
+ for value in (socket.gethostname(), socket.getfqdn()):
8
+ if value:
9
+ names.add(value.lower())
10
+ names.add(value.split(".", 1)[0].lower())
11
+
12
+ return names
13
+
14
+
15
+ def find_machine_config(config, hostname_override=None):
16
+ if hostname_override:
17
+ names = set(
18
+ [
19
+ hostname_override.lower(),
20
+ hostname_override.split(".", 1)[0].lower(),
21
+ ]
22
+ )
23
+ else:
24
+ names = get_local_names()
25
+
26
+ for machine in config.get("machines", []):
27
+ candidates = set()
28
+ hostname = machine.get("hostname")
29
+ if hostname:
30
+ candidates.add(hostname.lower())
31
+ candidates.add(hostname.split(".", 1)[0].lower())
32
+
33
+ for alias in machine.get("aliases", []) or []:
34
+ candidates.add(alias.lower())
35
+ candidates.add(alias.split(".", 1)[0].lower())
36
+
37
+ if names.intersection(candidates):
38
+ return machine
39
+
40
+ raise ValueError(
41
+ "No machine entry found for local hostname(s): {0}".format(
42
+ ", ".join(sorted(names))
43
+ )
44
+ )
@@ -0,0 +1,34 @@
1
+ import time
2
+
3
+
4
+ class CheckSchedule(object):
5
+ def __init__(self):
6
+ self._last_run = {}
7
+
8
+ def should_run(self, check_name, interval_seconds):
9
+ previous = self._last_run.get(check_name)
10
+ now = time.monotonic()
11
+ if previous is None:
12
+ return True
13
+ return now - previous >= int(interval_seconds)
14
+
15
+ def mark_run(self, check_name):
16
+ self._last_run[check_name] = time.monotonic()
17
+
18
+
19
+ def get_interval_seconds(check_config, intervals):
20
+ if "interval_seconds" in check_config:
21
+ return int(check_config["interval_seconds"])
22
+
23
+ interval_name = check_config.get("interval", "normal")
24
+ if interval_name not in intervals:
25
+ raise ValueError("Unknown interval name: {0}".format(interval_name))
26
+
27
+ return int(intervals[interval_name])
28
+
29
+
30
+ def shortest_interval(intervals):
31
+ values = [int(value) for value in intervals.values()]
32
+ if not values:
33
+ return 60
34
+ return min(values)
@@ -0,0 +1,94 @@
1
+ from datetime import datetime, timedelta
2
+
3
+ try:
4
+ from datetime import timezone
5
+ except ImportError: # pragma: no cover
6
+ timezone = None
7
+
8
+
9
+ STATE_ORDER = {
10
+ "OK": 0,
11
+ "UNKNOWN": 1,
12
+ "DEGRADED": 2,
13
+ "FAILED": 3,
14
+ }
15
+
16
+ PSRMON_STATE_MAP = {
17
+ "OK": "OK",
18
+ "DEGRADED": "WARNING",
19
+ "FAILED": "ERROR",
20
+ "UNKNOWN": "UNKNOWN",
21
+ }
22
+
23
+
24
+ def utc_now():
25
+ if timezone is not None:
26
+ return datetime.now(timezone.utc)
27
+ return datetime.utcnow()
28
+
29
+
30
+ def utc_now_iso():
31
+ return utc_now().isoformat()
32
+
33
+
34
+ def iso_after_seconds(seconds):
35
+ return (utc_now() + timedelta(seconds=int(seconds))).isoformat()
36
+
37
+
38
+ def combine_states(states):
39
+ if not states:
40
+ return "UNKNOWN"
41
+ return max(states, key=lambda state: STATE_ORDER.get(state, 1))
42
+
43
+
44
+ def collect_states(value):
45
+ states = []
46
+
47
+ if isinstance(value, dict):
48
+ state = value.get("state")
49
+ if isinstance(state, str):
50
+ states.append(state)
51
+ for nested in value.values():
52
+ states.extend(collect_states(nested))
53
+
54
+ elif isinstance(value, list):
55
+ for item in value:
56
+ states.extend(collect_states(item))
57
+
58
+ return states
59
+
60
+
61
+ def map_to_psrmon_state(state):
62
+ return PSRMON_STATE_MAP.get(state, "UNKNOWN")
63
+
64
+
65
+ def state_short_name(check_name):
66
+ names = {
67
+ "reachability": "alive",
68
+ "expected_processes": "proc",
69
+ "expected_services": "svc",
70
+ "expected_mounts": "mount",
71
+ "storage": "disk",
72
+ "memory": "mem",
73
+ "load": "load",
74
+ "time_sync": "time",
75
+ }
76
+ return names.get(check_name, check_name)
77
+
78
+
79
+ def make_summary(checks):
80
+ if not checks:
81
+ return "no_checks=UNKNOWN"
82
+
83
+ parts = []
84
+ for check_name in sorted(checks.keys()):
85
+ check = checks[check_name]
86
+ state = "UNKNOWN"
87
+ if isinstance(check, dict):
88
+ state = check.get("state", "UNKNOWN")
89
+ parts.append(
90
+ "{0}={1}".format(
91
+ state_short_name(check_name), map_to_psrmon_state(state)
92
+ )
93
+ )
94
+ return " ".join(parts)
@@ -0,0 +1,18 @@
1
+ Metadata-Version: 2.4
2
+ Name: psrsentinel
3
+ Version: 1.0.3
4
+ Summary: Jodrell bank pulsar server monitor
5
+ Author: Benjamin Shaw
6
+ Requires-Python: >=3.6
7
+ Description-Content-Type: text/markdown
8
+ Requires-Dist: PyYAML<6.0,>=3.13; python_version < "3.10"
9
+ Requires-Dist: PyYAML>=6.0; python_version >= "3.10"
10
+ Requires-Dist: typing_extensions>=4.0; python_version < "3.8"
11
+ Provides-Extra: dev
12
+ Requires-Dist: pytest; extra == "dev"
13
+ Requires-Dist: tox; extra == "dev"
14
+ Requires-Dist: isort; extra == "dev"
15
+ Requires-Dist: flake8; extra == "dev"
16
+ Requires-Dist: black; extra == "dev"
17
+ Requires-Dist: build; extra == "dev"
18
+ Requires-Dist: twine; extra == "dev"
@@ -0,0 +1,28 @@
1
+ pyproject.toml
2
+ src/psrsentinel/__init__.py
3
+ src/psrsentinel/__main__.py
4
+ src/psrsentinel/app.py
5
+ src/psrsentinel/command.py
6
+ src/psrsentinel/config.py
7
+ src/psrsentinel/hostname.py
8
+ src/psrsentinel/scheduler.py
9
+ src/psrsentinel/state.py
10
+ src/psrsentinel.egg-info/PKG-INFO
11
+ src/psrsentinel.egg-info/SOURCES.txt
12
+ src/psrsentinel.egg-info/dependency_links.txt
13
+ src/psrsentinel.egg-info/entry_points.txt
14
+ src/psrsentinel.egg-info/requires.txt
15
+ src/psrsentinel.egg-info/top_level.txt
16
+ src/psrsentinel/api/__init__.py
17
+ src/psrsentinel/api/adapter.py
18
+ src/psrsentinel/api/client.py
19
+ src/psrsentinel/checks/__init__.py
20
+ src/psrsentinel/checks/load.py
21
+ src/psrsentinel/checks/memory.py
22
+ src/psrsentinel/checks/mounts.py
23
+ src/psrsentinel/checks/processes.py
24
+ src/psrsentinel/checks/services.py
25
+ src/psrsentinel/checks/storage.py
26
+ src/psrsentinel/checks/time_sync.py
27
+ tests/test_hostname.py
28
+ tests/test_state.py
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ psrsentinel = psrsentinel.app:main
@@ -0,0 +1,18 @@
1
+
2
+ [:python_version < "3.10"]
3
+ PyYAML<6.0,>=3.13
4
+
5
+ [:python_version < "3.8"]
6
+ typing_extensions>=4.0
7
+
8
+ [:python_version >= "3.10"]
9
+ PyYAML>=6.0
10
+
11
+ [dev]
12
+ pytest
13
+ tox
14
+ isort
15
+ flake8
16
+ black
17
+ build
18
+ twine
@@ -0,0 +1 @@
1
+ psrsentinel
@@ -0,0 +1,11 @@
1
+ from psrsentinel.hostname import find_machine_config
2
+
3
+
4
+ def test_find_machine_by_hostname():
5
+ config = {"machines": [{"hostname": "ector"}]}
6
+ assert find_machine_config(config, "ector")["hostname"] == "ector"
7
+
8
+
9
+ def test_find_machine_by_alias():
10
+ config = {"machines": [{"hostname": "ector", "aliases": ["ector.local"]}]}
11
+ assert find_machine_config(config, "ector.local")["hostname"] == "ector"
@@ -0,0 +1,26 @@
1
+ from psrsentinel.state import combine_states, make_summary, map_to_psrmon_state
2
+
3
+
4
+ def test_combine_states_ok():
5
+ assert combine_states(["OK", "OK"]) == "OK"
6
+
7
+
8
+ def test_combine_states_degraded():
9
+ assert combine_states(["OK", "DEGRADED"]) == "DEGRADED"
10
+
11
+
12
+ def test_combine_states_failed_wins():
13
+ assert combine_states(["OK", "DEGRADED", "FAILED"]) == "FAILED"
14
+
15
+
16
+ def test_state_mapping():
17
+ assert map_to_psrmon_state("DEGRADED") == "WARNING"
18
+ assert map_to_psrmon_state("FAILED") == "ERROR"
19
+
20
+
21
+ def test_make_summary():
22
+ summary = make_summary(
23
+ {"expected_processes": {"state": "OK"}, "storage": {"state": "FAILED"}}
24
+ )
25
+ assert "proc=OK" in summary
26
+ assert "disk=ERROR" in summary