syslog-postmortem 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syslog_postmortem-1.1.0/PKG-INFO +205 -0
- syslog_postmortem-1.1.0/README.md +179 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/pyproject.toml +25 -6
- syslog_postmortem-1.1.0/syslog_postmortem/__init__.py +5 -0
- syslog_postmortem-1.1.0/syslog_postmortem/__main__.py +3 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem/analyzer.py +47 -28
- syslog_postmortem-1.1.0/syslog_postmortem/cli.py +168 -0
- syslog_postmortem-1.1.0/syslog_postmortem/collector.py +240 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem/patterns.py +4 -2
- syslog_postmortem-1.1.0/syslog_postmortem/renderer.py +298 -0
- syslog_postmortem-1.1.0/syslog_postmortem.egg-info/PKG-INFO +205 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem.egg-info/SOURCES.txt +6 -1
- syslog_postmortem-1.1.0/syslog_postmortem.egg-info/requires.txt +5 -0
- syslog_postmortem-1.1.0/tests/test_analyzer.py +73 -0
- syslog_postmortem-1.1.0/tests/test_cli.py +51 -0
- syslog_postmortem-1.1.0/tests/test_collector.py +81 -0
- syslog_postmortem-1.1.0/tests/test_renderer.py +58 -0
- syslog_postmortem-1.0.0/PKG-INFO +0 -149
- syslog_postmortem-1.0.0/README.md +0 -129
- syslog_postmortem-1.0.0/syslog_postmortem/__init__.py +0 -4
- syslog_postmortem-1.0.0/syslog_postmortem/cli.py +0 -157
- syslog_postmortem-1.0.0/syslog_postmortem/collector.py +0 -228
- syslog_postmortem-1.0.0/syslog_postmortem/renderer.py +0 -284
- syslog_postmortem-1.0.0/syslog_postmortem.egg-info/PKG-INFO +0 -149
- syslog_postmortem-1.0.0/syslog_postmortem.egg-info/requires.txt +0 -1
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/LICENSE +0 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/setup.cfg +0 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem.egg-info/dependency_links.txt +0 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem.egg-info/entry_points.txt +0 -0
- {syslog_postmortem-1.0.0 → syslog_postmortem-1.1.0}/syslog_postmortem.egg-info/top_level.txt +0 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: syslog-postmortem
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: Generate structured postmortem drafts from journalctl, syslog, auth.log and kern.log
|
|
5
|
+
Author-email: Serber1990 <serber1990@pm.me>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/serber1990/syslog-postmortem
|
|
8
|
+
Project-URL: Bug Tracker, https://github.com/serber1990/syslog-postmortem/issues
|
|
9
|
+
Project-URL: Source Code, https://github.com/serber1990/syslog-postmortem
|
|
10
|
+
Project-URL: Changelog, https://github.com/serber1990/syslog-postmortem/blob/main/CHANGELOG.md
|
|
11
|
+
Keywords: postmortem,incident,sre,journalctl,syslog,logs,sysadmin
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
14
|
+
Classifier: Environment :: Console
|
|
15
|
+
Classifier: Intended Audience :: System Administrators
|
|
16
|
+
Classifier: Topic :: System :: Logging
|
|
17
|
+
Classifier: Topic :: System :: Systems Administration
|
|
18
|
+
Requires-Python: >=3.9
|
|
19
|
+
Description-Content-Type: text/markdown
|
|
20
|
+
License-File: LICENSE
|
|
21
|
+
Requires-Dist: shellcolorize>=1.1.0
|
|
22
|
+
Provides-Extra: dev
|
|
23
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
24
|
+
Requires-Dist: ruff>=0.4; extra == "dev"
|
|
25
|
+
Dynamic: license-file
|
|
26
|
+
|
|
27
|
+
# syslog-postmortem
|
|
28
|
+
|
|
29
|
+
[](https://github.com/serber1990/syslog-postmortem/actions/workflows/ci.yml)
|
|
30
|
+
[](https://badge.fury.io/py/syslog-postmortem)
|
|
31
|
+
[](LICENSE)
|
|
32
|
+
|
|
33
|
+
Generate a **structured postmortem draft** from system logs in seconds — no more reconstructing incident
|
|
34
|
+
timelines by hand at 3 a.m.
|
|
35
|
+
|
|
36
|
+
Reads `journalctl` plus `/var/log/syslog`, `messages`, `auth.log`, `secure` and `kern.log`, correlates
|
|
37
|
+
events, detects known failure patterns and writes a ready-to-edit Markdown or HTML postmortem.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## ✨ What it does
|
|
42
|
+
|
|
43
|
+
Given a time window (and optionally a list of services), `postmortem`:
|
|
44
|
+
|
|
45
|
+
1. **Collects** entries from `journalctl` (primary) and `/var/log` files, including their `.1` rotation
|
|
46
|
+
2. **Attributes** systemd messages to the unit they are about (not to PID 1)
|
|
47
|
+
3. **Groups** repeated messages — `Failed to start backup (×166, until 10:55)` instead of 166 lines
|
|
48
|
+
4. **Detects patterns** — OOM kills, disk full, kernel oops, crashes, restart loops, connection errors,
|
|
49
|
+
timeouts, auth bursts, TLS and database errors, high load
|
|
50
|
+
5. **Builds a timeline**, keeping the most severe events when there are too many
|
|
51
|
+
6. **Finds contributing factors** — cascading failures between services, restart loops, error bursts
|
|
52
|
+
7. **Suggests action items** and leaves Impact / Root cause / Lessons learned sections ready to fill in
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## 📥 Installation
|
|
57
|
+
|
|
58
|
+
```bash
|
|
59
|
+
pip install syslog-postmortem
|
|
60
|
+
```
|
|
61
|
+
|
|
62
|
+
Run it with `sudo` (or as a member of the `adm` / `systemd-journal` group) to read the whole system journal.
|
|
63
|
+
|
|
64
|
+
---
|
|
65
|
+
|
|
66
|
+
## 🛠 Usage
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
postmortem --last 2h # incident in progress: the last two hours
|
|
70
|
+
postmortem --from "2026-05-10 14:00" --to "2026-05-10 16:00"
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
# Focus on specific services (postgresql also matches postgresql@14-main)
|
|
75
|
+
postmortem --from "2026-05-10 14:00" --to "2026-05-10 16:00" \
|
|
76
|
+
--services nginx,postgresql,redis \
|
|
77
|
+
--title "Database outage" \
|
|
78
|
+
--output incident-2026-05-10.md
|
|
79
|
+
|
|
80
|
+
# Standalone HTML page (light/dark theme, safe to open: log content is escaped)
|
|
81
|
+
postmortem --last 1d --format html --output report.html
|
|
82
|
+
|
|
83
|
+
# Print to stdout, e.g. to paste into a ticket
|
|
84
|
+
postmortem --last 30m -o - | xclip -selection clipboard
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
---
|
|
88
|
+
|
|
89
|
+
## 📋 Options
|
|
90
|
+
|
|
91
|
+
| Option | Description |
|
|
92
|
+
|--------|-------------|
|
|
93
|
+
| `--from DATETIME` | Start of the window: `"YYYY-MM-DD HH:MM[:SS]"` |
|
|
94
|
+
| `--to DATETIME` | End of the window (default: now) |
|
|
95
|
+
| `--last DURATION` | Window ending now (or at `--to`): `30m`, `2h`, `1d` |
|
|
96
|
+
| `--title TEXT` | Postmortem title (default: `Incident YYYY-MM-DD`) |
|
|
97
|
+
| `--services LIST` | Comma-separated services to focus on |
|
|
98
|
+
| `-o`, `--output FILE` | Output path (default: `postmortem_YYYYMMDD_HHMM.md`); `-` for stdout |
|
|
99
|
+
| `--format` | `markdown` (default) or `html` |
|
|
100
|
+
| `--no-files` | Skip `/var/log` files, use journalctl only |
|
|
101
|
+
| `--priorities` | journalctl priority filter (default: `0..4` = emerg…warning) |
|
|
102
|
+
|
|
103
|
+
---
|
|
104
|
+
|
|
105
|
+
## 📄 Example
|
|
106
|
+
|
|
107
|
+
This draft was generated from a simulated incident: PostgreSQL is OOM-killed, nginx starts failing,
|
|
108
|
+
and someone tries to brute-force SSH at the same time.
|
|
109
|
+
|
|
110
|
+
```markdown
|
|
111
|
+
# Postmortem: Database outage
|
|
112
|
+
|
|
113
|
+
| | |
|
|
114
|
+
|---|---|
|
|
115
|
+
| **Date** | 2026-05-10 |
|
|
116
|
+
| **Window** | 2026-05-10 14:00 → 2026-05-10 16:00 |
|
|
117
|
+
| **Duration** | 2h |
|
|
118
|
+
| **Severity** | Critical |
|
|
119
|
+
| **Status** | Draft |
|
|
120
|
+
| **Services analysed** | all |
|
|
121
|
+
| **Services affected** | `kernel`, `nginx`, `postgresql` |
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## Summary
|
|
126
|
+
|
|
127
|
+
Analysis of **58 raw log entries** (5 unique events after deduplication) across **4 services**. First anomaly detected at **14:03:22** in `kernel` (CRITICAL). Highest error density at **14:03**.
|
|
128
|
+
|
|
129
|
+
> *Auto-generated draft — review all sections before sharing.*
|
|
130
|
+
|
|
131
|
+
---
|
|
132
|
+
|
|
133
|
+
## Timeline
|
|
134
|
+
|
|
135
|
+
| Time | Service | Severity | Event |
|
|
136
|
+
|------|---------|----------|-------|
|
|
137
|
+
| 14:03:22 | `kernel` | ⛔ CRITICAL | Out of memory: Killed process 2211 (postgres) total-vm:8123456kB |
|
|
138
|
+
| 14:03:24 | `postgresql` | ⛔ CRITICAL | server process (PID 2211) was terminated by signal 9: Killed |
|
|
139
|
+
| 14:03:45 | `nginx` | 🔴 ERROR | connect() failed (111: Connection refused) while connecting to upstream _(×40, until 14:05:42)_ |
|
|
140
|
+
| 14:04:22 | `postgresql` | ⚠️ WARNING | start request repeated too quickly, automatic restart _(×4, until 14:05:22)_ |
|
|
141
|
+
| 14:08:22 | `sshd` | ⚠️ WARNING | Failed password for invalid user admin from 203.0.113.7 port 40000 _(×12, until 14:08:33)_ |
|
|
142
|
+
|
|
143
|
+
## Contributing Factors
|
|
144
|
+
|
|
145
|
+
> *Auto-detected from log patterns — verify each before including in final report.*
|
|
146
|
+
|
|
147
|
+
- **Service instability**: `postgresql` triggered restart-loop detection **4 time(s)**
|
|
148
|
+
- **Memory pressure**: OOM killer fired **1 time(s)** during the window
|
|
149
|
+
- **Auth anomaly**: Burst of **12 authentication failure(s)** detected — possible brute-force attempt
|
|
150
|
+
- **Cascading failure**: `nginx` errors began **23s** after the first `kernel` critical event (14:03:22)
|
|
151
|
+
- **Cascading failure**: `nginx` errors began **21s** after the first `postgresql` critical event (14:03:24)
|
|
152
|
+
- **Error burst**: **42 errors** in the 5-minute window starting 14:00
|
|
153
|
+
|
|
154
|
+
## Action Items
|
|
155
|
+
|
|
156
|
+
_Generated from detected patterns — assign owner and priority._
|
|
157
|
+
|
|
158
|
+
- [ ] Investigate memory usage; consider adding swap or increasing RAM
|
|
159
|
+
- [ ] Verify the downstream service is running and listening on the expected port
|
|
160
|
+
- [ ] Service is crash-looping; check dependencies and configuration
|
|
161
|
+
- [ ] Review SSH access logs; consider IP blocking if burst detected
|
|
162
|
+
|
|
163
|
+
…
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## 🔍 Detected patterns
|
|
169
|
+
|
|
170
|
+
| Pattern | Triggers |
|
|
171
|
+
|---------|----------|
|
|
172
|
+
| OOM Killer | `out of memory`, `oom-kill`, `killed process` |
|
|
173
|
+
| Disk Full | `no space left on device`, `disk full` |
|
|
174
|
+
| Kernel Error | `BUG:`, `Oops`, `call trace`, `segfault at`, `general protection`, `kernel panic` |
|
|
175
|
+
| Service Failed | `failed to start`, `entered failed state` |
|
|
176
|
+
| Process Crash | `segmentation fault`, `core dumped`, `aborted` |
|
|
177
|
+
| Connection Refused | `connection refused`, `upstream connect error` |
|
|
178
|
+
| Timeout | `timed out`, `ETIMEDOUT`, `request timeout` |
|
|
179
|
+
| Auth Failure | `Failed password`, `authentication failure`, `invalid user` |
|
|
180
|
+
| Service Restart Loop | `start request repeated too quickly`, `automatic restart` |
|
|
181
|
+
| SSL/TLS Error | `certificate expired`, `TLS handshake failed`, `certificate verify failed` |
|
|
182
|
+
| Database Error | `could not connect to database`, `max connections reached`, `deadlock` |
|
|
183
|
+
| High Load | load average of 10 or more, `cpu throttled` |
|
|
184
|
+
|
|
185
|
+
---
|
|
186
|
+
|
|
187
|
+
## 🧪 Development
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
pip install -e ".[dev]"
|
|
191
|
+
ruff check .
|
|
192
|
+
pytest
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
See [CHANGELOG.md](CHANGELOG.md) for release notes.
|
|
196
|
+
|
|
197
|
+
---
|
|
198
|
+
|
|
199
|
+
## 📝 License
|
|
200
|
+
|
|
201
|
+
MIT — see [LICENSE](LICENSE).
|
|
202
|
+
|
|
203
|
+
## 🌐 Connect
|
|
204
|
+
|
|
205
|
+
[](https://github.com/serber1990)
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
# syslog-postmortem
|
|
2
|
+
|
|
3
|
+
[](https://github.com/serber1990/syslog-postmortem/actions/workflows/ci.yml)
|
|
4
|
+
[](https://badge.fury.io/py/syslog-postmortem)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
Generate a **structured postmortem draft** from system logs in seconds — no more reconstructing incident
|
|
8
|
+
timelines by hand at 3 a.m.
|
|
9
|
+
|
|
10
|
+
Reads `journalctl` plus `/var/log/syslog`, `messages`, `auth.log`, `secure` and `kern.log`, correlates
|
|
11
|
+
events, detects known failure patterns and writes a ready-to-edit Markdown or HTML postmortem.
|
|
12
|
+
|
|
13
|
+
---
|
|
14
|
+
|
|
15
|
+
## ✨ What it does
|
|
16
|
+
|
|
17
|
+
Given a time window (and optionally a list of services), `postmortem`:
|
|
18
|
+
|
|
19
|
+
1. **Collects** entries from `journalctl` (primary) and `/var/log` files, including their `.1` rotation
|
|
20
|
+
2. **Attributes** systemd messages to the unit they are about (not to PID 1)
|
|
21
|
+
3. **Groups** repeated messages — `Failed to start backup (×166, until 10:55)` instead of 166 lines
|
|
22
|
+
4. **Detects patterns** — OOM kills, disk full, kernel oops, crashes, restart loops, connection errors,
|
|
23
|
+
timeouts, auth bursts, TLS and database errors, high load
|
|
24
|
+
5. **Builds a timeline**, keeping the most severe events when there are too many
|
|
25
|
+
6. **Finds contributing factors** — cascading failures between services, restart loops, error bursts
|
|
26
|
+
7. **Suggests action items** and leaves Impact / Root cause / Lessons learned sections ready to fill in
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## 📥 Installation
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install syslog-postmortem
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
Run it with `sudo` (or as a member of the `adm` / `systemd-journal` group) to read the whole system journal.
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## 🛠 Usage
|
|
41
|
+
|
|
42
|
+
```bash
|
|
43
|
+
postmortem --last 2h # incident in progress: the last two hours
|
|
44
|
+
postmortem --from "2026-05-10 14:00" --to "2026-05-10 16:00"
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
```bash
|
|
48
|
+
# Focus on specific services (postgresql also matches postgresql@14-main)
|
|
49
|
+
postmortem --from "2026-05-10 14:00" --to "2026-05-10 16:00" \
|
|
50
|
+
--services nginx,postgresql,redis \
|
|
51
|
+
--title "Database outage" \
|
|
52
|
+
--output incident-2026-05-10.md
|
|
53
|
+
|
|
54
|
+
# Standalone HTML page (light/dark theme, safe to open: log content is escaped)
|
|
55
|
+
postmortem --last 1d --format html --output report.html
|
|
56
|
+
|
|
57
|
+
# Print to stdout, e.g. to paste into a ticket
|
|
58
|
+
postmortem --last 30m -o - | xclip -selection clipboard
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## 📋 Options
|
|
64
|
+
|
|
65
|
+
| Option | Description |
|
|
66
|
+
|--------|-------------|
|
|
67
|
+
| `--from DATETIME` | Start of the window: `"YYYY-MM-DD HH:MM[:SS]"` |
|
|
68
|
+
| `--to DATETIME` | End of the window (default: now) |
|
|
69
|
+
| `--last DURATION` | Window ending now (or at `--to`): `30m`, `2h`, `1d` |
|
|
70
|
+
| `--title TEXT` | Postmortem title (default: `Incident YYYY-MM-DD`) |
|
|
71
|
+
| `--services LIST` | Comma-separated services to focus on |
|
|
72
|
+
| `-o`, `--output FILE` | Output path (default: `postmortem_YYYYMMDD_HHMM.md`); `-` for stdout |
|
|
73
|
+
| `--format` | `markdown` (default) or `html` |
|
|
74
|
+
| `--no-files` | Skip `/var/log` files, use journalctl only |
|
|
75
|
+
| `--priorities` | journalctl priority filter (default: `0..4` = emerg…warning) |
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## 📄 Example
|
|
80
|
+
|
|
81
|
+
This draft was generated from a simulated incident: PostgreSQL is OOM-killed, nginx starts failing,
|
|
82
|
+
and someone tries to brute-force SSH at the same time.
|
|
83
|
+
|
|
84
|
+
```markdown
|
|
85
|
+
# Postmortem: Database outage
|
|
86
|
+
|
|
87
|
+
| | |
|
|
88
|
+
|---|---|
|
|
89
|
+
| **Date** | 2026-05-10 |
|
|
90
|
+
| **Window** | 2026-05-10 14:00 → 2026-05-10 16:00 |
|
|
91
|
+
| **Duration** | 2h |
|
|
92
|
+
| **Severity** | Critical |
|
|
93
|
+
| **Status** | Draft |
|
|
94
|
+
| **Services analysed** | all |
|
|
95
|
+
| **Services affected** | `kernel`, `nginx`, `postgresql` |
|
|
96
|
+
|
|
97
|
+
---
|
|
98
|
+
|
|
99
|
+
## Summary
|
|
100
|
+
|
|
101
|
+
Analysis of **58 raw log entries** (5 unique events after deduplication) across **4 services**. First anomaly detected at **14:03:22** in `kernel` (CRITICAL). Highest error density at **14:03**.
|
|
102
|
+
|
|
103
|
+
> *Auto-generated draft — review all sections before sharing.*
|
|
104
|
+
|
|
105
|
+
---
|
|
106
|
+
|
|
107
|
+
## Timeline
|
|
108
|
+
|
|
109
|
+
| Time | Service | Severity | Event |
|
|
110
|
+
|------|---------|----------|-------|
|
|
111
|
+
| 14:03:22 | `kernel` | ⛔ CRITICAL | Out of memory: Killed process 2211 (postgres) total-vm:8123456kB |
|
|
112
|
+
| 14:03:24 | `postgresql` | ⛔ CRITICAL | server process (PID 2211) was terminated by signal 9: Killed |
|
|
113
|
+
| 14:03:45 | `nginx` | 🔴 ERROR | connect() failed (111: Connection refused) while connecting to upstream _(×40, until 14:05:42)_ |
|
|
114
|
+
| 14:04:22 | `postgresql` | ⚠️ WARNING | start request repeated too quickly, automatic restart _(×4, until 14:05:22)_ |
|
|
115
|
+
| 14:08:22 | `sshd` | ⚠️ WARNING | Failed password for invalid user admin from 203.0.113.7 port 40000 _(×12, until 14:08:33)_ |
|
|
116
|
+
|
|
117
|
+
## Contributing Factors
|
|
118
|
+
|
|
119
|
+
> *Auto-detected from log patterns — verify each before including in final report.*
|
|
120
|
+
|
|
121
|
+
- **Service instability**: `postgresql` triggered restart-loop detection **4 time(s)**
|
|
122
|
+
- **Memory pressure**: OOM killer fired **1 time(s)** during the window
|
|
123
|
+
- **Auth anomaly**: Burst of **12 authentication failure(s)** detected — possible brute-force attempt
|
|
124
|
+
- **Cascading failure**: `nginx` errors began **23s** after the first `kernel` critical event (14:03:22)
|
|
125
|
+
- **Cascading failure**: `nginx` errors began **21s** after the first `postgresql` critical event (14:03:24)
|
|
126
|
+
- **Error burst**: **42 errors** in the 5-minute window starting 14:00
|
|
127
|
+
|
|
128
|
+
## Action Items
|
|
129
|
+
|
|
130
|
+
_Generated from detected patterns — assign owner and priority._
|
|
131
|
+
|
|
132
|
+
- [ ] Investigate memory usage; consider adding swap or increasing RAM
|
|
133
|
+
- [ ] Verify the downstream service is running and listening on the expected port
|
|
134
|
+
- [ ] Service is crash-looping; check dependencies and configuration
|
|
135
|
+
- [ ] Review SSH access logs; consider IP blocking if burst detected
|
|
136
|
+
|
|
137
|
+
…
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## 🔍 Detected patterns
|
|
143
|
+
|
|
144
|
+
| Pattern | Triggers |
|
|
145
|
+
|---------|----------|
|
|
146
|
+
| OOM Killer | `out of memory`, `oom-kill`, `killed process` |
|
|
147
|
+
| Disk Full | `no space left on device`, `disk full` |
|
|
148
|
+
| Kernel Error | `BUG:`, `Oops`, `call trace`, `segfault at`, `general protection`, `kernel panic` |
|
|
149
|
+
| Service Failed | `failed to start`, `entered failed state` |
|
|
150
|
+
| Process Crash | `segmentation fault`, `core dumped`, `aborted` |
|
|
151
|
+
| Connection Refused | `connection refused`, `upstream connect error` |
|
|
152
|
+
| Timeout | `timed out`, `ETIMEDOUT`, `request timeout` |
|
|
153
|
+
| Auth Failure | `Failed password`, `authentication failure`, `invalid user` |
|
|
154
|
+
| Service Restart Loop | `start request repeated too quickly`, `automatic restart` |
|
|
155
|
+
| SSL/TLS Error | `certificate expired`, `TLS handshake failed`, `certificate verify failed` |
|
|
156
|
+
| Database Error | `could not connect to database`, `max connections reached`, `deadlock` |
|
|
157
|
+
| High Load | load average of 10 or more, `cpu throttled` |
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## 🧪 Development
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
pip install -e ".[dev]"
|
|
165
|
+
ruff check .
|
|
166
|
+
pytest
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
See [CHANGELOG.md](CHANGELOG.md) for release notes.
|
|
170
|
+
|
|
171
|
+
---
|
|
172
|
+
|
|
173
|
+
## 📝 License
|
|
174
|
+
|
|
175
|
+
MIT — see [LICENSE](LICENSE).
|
|
176
|
+
|
|
177
|
+
## 🌐 Connect
|
|
178
|
+
|
|
179
|
+
[](https://github.com/serber1990)
|
|
@@ -1,32 +1,51 @@
|
|
|
1
1
|
[build-system]
|
|
2
|
-
requires = ["setuptools>=
|
|
2
|
+
requires = ["setuptools>=77"]
|
|
3
3
|
build-backend = "setuptools.build_meta"
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "syslog-postmortem"
|
|
7
|
-
|
|
8
|
-
description = "Generate structured postmortem drafts from journalctl, syslog, auth.log and
|
|
7
|
+
dynamic = ["version"]
|
|
8
|
+
description = "Generate structured postmortem drafts from journalctl, syslog, auth.log and kern.log"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
11
12
|
authors = [{ name = "Serber1990", email = "serber1990@pm.me" }]
|
|
12
|
-
requires-python = ">=3.
|
|
13
|
-
dependencies = ["shellcolorize"]
|
|
13
|
+
requires-python = ">=3.9"
|
|
14
|
+
dependencies = ["shellcolorize>=1.1.0"]
|
|
15
|
+
keywords = ["postmortem", "incident", "sre", "journalctl", "syslog", "logs", "sysadmin"]
|
|
14
16
|
classifiers = [
|
|
15
17
|
"Programming Language :: Python :: 3",
|
|
16
18
|
"Operating System :: POSIX :: Linux",
|
|
17
19
|
"Environment :: Console",
|
|
20
|
+
"Intended Audience :: System Administrators",
|
|
18
21
|
"Topic :: System :: Logging",
|
|
19
22
|
"Topic :: System :: Systems Administration",
|
|
20
23
|
]
|
|
21
24
|
|
|
25
|
+
[project.optional-dependencies]
|
|
26
|
+
dev = ["pytest>=7", "ruff>=0.4"]
|
|
27
|
+
|
|
22
28
|
[project.urls]
|
|
23
29
|
Homepage = "https://github.com/serber1990/syslog-postmortem"
|
|
24
30
|
"Bug Tracker" = "https://github.com/serber1990/syslog-postmortem/issues"
|
|
25
31
|
"Source Code" = "https://github.com/serber1990/syslog-postmortem"
|
|
32
|
+
Changelog = "https://github.com/serber1990/syslog-postmortem/blob/main/CHANGELOG.md"
|
|
26
33
|
|
|
27
34
|
[project.scripts]
|
|
28
35
|
postmortem = "syslog_postmortem.cli:main"
|
|
29
36
|
|
|
30
37
|
[tool.setuptools.packages.find]
|
|
31
|
-
where = ["."]
|
|
32
38
|
include = ["syslog_postmortem*"]
|
|
39
|
+
|
|
40
|
+
[tool.setuptools.dynamic]
|
|
41
|
+
version = { attr = "syslog_postmortem.__version__" }
|
|
42
|
+
|
|
43
|
+
[tool.pytest.ini_options]
|
|
44
|
+
pythonpath = ["."]
|
|
45
|
+
filterwarnings = ["error::DeprecationWarning"]
|
|
46
|
+
|
|
47
|
+
[tool.ruff]
|
|
48
|
+
target-version = "py39"
|
|
49
|
+
|
|
50
|
+
[tool.ruff.lint]
|
|
51
|
+
select = ["E9", "F", "B"]
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
"""
|
|
2
2
|
Pattern analysis, deduplication, cascade detection and contributing factor generation.
|
|
3
3
|
"""
|
|
4
|
+
import re
|
|
4
5
|
from collections import Counter, defaultdict
|
|
5
6
|
from dataclasses import dataclass, field
|
|
6
|
-
from datetime import datetime
|
|
7
|
-
from typing import
|
|
7
|
+
from datetime import datetime
|
|
8
|
+
from typing import Dict, List, Optional
|
|
8
9
|
|
|
9
10
|
from .collector import RawEntry, priority_to_severity
|
|
10
11
|
from .patterns import match_patterns, Pattern
|
|
@@ -20,7 +21,8 @@ class Event:
|
|
|
20
21
|
message: str
|
|
21
22
|
source: str
|
|
22
23
|
patterns: List[Pattern] = field(default_factory=list)
|
|
23
|
-
count: int = 1
|
|
24
|
+
count: int = 1 # after deduplication
|
|
25
|
+
last_timestamp: Optional[datetime] = None # last repetition when count > 1
|
|
24
26
|
|
|
25
27
|
|
|
26
28
|
# ── Analysis result ───────────────────────────────────────────────────────────
|
|
@@ -42,22 +44,29 @@ class AnalysisResult:
|
|
|
42
44
|
|
|
43
45
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
|
44
46
|
|
|
45
|
-
|
|
47
|
+
# Standalone integers (PIDs, ports, counters) — but not IPv4 octets, versions or times.
|
|
48
|
+
_VOLATILE_NUMBER = re.compile(r'(?<![\d.:])\d+(?![\d.:])')
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _deduplicate(events: List[Event], window_seconds: int = 300) -> List[Event]:
|
|
46
52
|
"""
|
|
47
|
-
Merge identical messages
|
|
48
|
-
|
|
53
|
+
Merge identical messages from the same service while they keep repeating
|
|
54
|
+
(each repetition less than `window_seconds` after the previous one).
|
|
55
|
+
Keeps the first occurrence with a count and the time of the last repetition.
|
|
49
56
|
"""
|
|
50
57
|
result: List[Event] = []
|
|
51
|
-
|
|
58
|
+
groups: Dict[tuple, Event] = {}
|
|
52
59
|
|
|
53
60
|
for ev in events:
|
|
54
|
-
key = (ev.service, ev.message[:
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
61
|
+
key = (ev.service, _VOLATILE_NUMBER.sub('#', ev.message[:200]))
|
|
62
|
+
group = groups.get(key)
|
|
63
|
+
if group is not None:
|
|
64
|
+
last = group.last_timestamp or group.timestamp
|
|
65
|
+
if (ev.timestamp - last).total_seconds() <= window_seconds:
|
|
66
|
+
group.count += 1
|
|
67
|
+
group.last_timestamp = ev.timestamp
|
|
59
68
|
continue
|
|
60
|
-
|
|
69
|
+
groups[key] = ev
|
|
61
70
|
result.append(ev)
|
|
62
71
|
|
|
63
72
|
return result
|
|
@@ -73,29 +82,39 @@ def _peak_minute(events: List[Event]) -> Optional[datetime]:
|
|
|
73
82
|
|
|
74
83
|
def _detect_cascades(events: List[Event], window_seconds: int = 120) -> List[str]:
|
|
75
84
|
"""
|
|
76
|
-
Detect cascading failures: service A
|
|
77
|
-
|
|
85
|
+
Detect cascading failures: service A has a critical event, service B starts
|
|
86
|
+
failing shortly after. One line per (A, B) pair, using A's first critical event.
|
|
78
87
|
"""
|
|
79
88
|
cascades = []
|
|
89
|
+
seen_pairs = set()
|
|
80
90
|
criticals = [e for e in events if e.severity == 'CRITICAL']
|
|
81
91
|
errors = [e for e in events if e.severity == 'ERROR']
|
|
82
92
|
|
|
83
93
|
for trigger in criticals:
|
|
84
94
|
for follow in errors:
|
|
85
|
-
|
|
95
|
+
pair = (trigger.service, follow.service)
|
|
96
|
+
if follow.service == trigger.service or pair in seen_pairs:
|
|
86
97
|
continue
|
|
87
98
|
delta = (follow.timestamp - trigger.timestamp).total_seconds()
|
|
88
99
|
if 0 < delta <= window_seconds:
|
|
100
|
+
seen_pairs.add(pair)
|
|
89
101
|
cascades.append(
|
|
90
|
-
f"**{follow.service}
|
|
91
|
-
f"**{int(delta)}s** after first
|
|
102
|
+
f"**Cascading failure**: `{follow.service}` errors began "
|
|
103
|
+
f"**{int(delta)}s** after the first `{trigger.service}` critical event "
|
|
92
104
|
f"({trigger.timestamp.strftime('%H:%M:%S')})"
|
|
93
105
|
)
|
|
94
|
-
break # one cascade per trigger service pair
|
|
95
|
-
|
|
96
106
|
return cascades
|
|
97
107
|
|
|
98
108
|
|
|
109
|
+
_SEV_RANK = {'CRITICAL': 0, 'ERROR': 1, 'WARNING': 2, 'INFO': 3, 'DEBUG': 4}
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def service_matches(service: str, wanted: List[str]) -> bool:
|
|
113
|
+
"""'postgresql' matches 'postgresql', 'postgresql@14-main' and 'postgresql.service'."""
|
|
114
|
+
s = service.lower()
|
|
115
|
+
return any(s == w or s.startswith((w + '@', w + '.', w + '-')) for w in wanted)
|
|
116
|
+
|
|
117
|
+
|
|
99
118
|
# ── Main analyser ─────────────────────────────────────────────────────────────
|
|
100
119
|
|
|
101
120
|
def analyze(raw: List[RawEntry], services_filter: List[str] = None) -> AnalysisResult:
|
|
@@ -103,15 +122,15 @@ def analyze(raw: List[RawEntry], services_filter: List[str] = None) -> AnalysisR
|
|
|
103
122
|
|
|
104
123
|
# 1. Normalise RawEntry → Event and run pattern matching
|
|
105
124
|
events: List[Event] = []
|
|
125
|
+
wanted = [s.lower() for s in services_filter or []]
|
|
106
126
|
for r in raw:
|
|
107
|
-
if
|
|
127
|
+
if wanted and not service_matches(r.service, wanted):
|
|
108
128
|
continue
|
|
109
129
|
severity = priority_to_severity(r.priority)
|
|
110
130
|
matched = match_patterns(r.message)
|
|
111
|
-
# Upgrade severity
|
|
112
|
-
sev_rank = {'CRITICAL': 0, 'ERROR': 1, 'WARNING': 2, 'INFO': 3, 'DEBUG': 4}
|
|
131
|
+
# Upgrade severity when a known pattern is more serious than the log priority
|
|
113
132
|
for p in matched:
|
|
114
|
-
if
|
|
133
|
+
if _SEV_RANK.get(p.severity, 4) < _SEV_RANK.get(severity, 4):
|
|
115
134
|
severity = p.severity
|
|
116
135
|
events.append(Event(
|
|
117
136
|
timestamp=r.timestamp,
|
|
@@ -156,10 +175,10 @@ def analyze(raw: List[RawEntry], services_filter: List[str] = None) -> AnalysisR
|
|
|
156
175
|
factors: List[str] = []
|
|
157
176
|
|
|
158
177
|
# Restart loops
|
|
159
|
-
restart_counts = Counter(
|
|
160
|
-
|
|
161
|
-
if any(p.name == 'service_restart' for p in e.patterns)
|
|
162
|
-
|
|
178
|
+
restart_counts: Counter = Counter()
|
|
179
|
+
for e in timeline:
|
|
180
|
+
if any(p.name == 'service_restart' for p in e.patterns):
|
|
181
|
+
restart_counts[e.service] += e.count
|
|
163
182
|
for svc, cnt in restart_counts.most_common():
|
|
164
183
|
factors.append(
|
|
165
184
|
f"**Service instability**: `{svc}` triggered restart-loop detection **{cnt} time(s)**"
|