psistat 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- psistat-1.0.0/.gitignore +77 -0
- psistat-1.0.0/LICENSE +21 -0
- psistat-1.0.0/PKG-INFO +109 -0
- psistat-1.0.0/README.md +91 -0
- psistat-1.0.0/psistat.py +375 -0
- psistat-1.0.0/pyproject.toml +47 -0
psistat-1.0.0/.gitignore
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[cod]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# Distribution / packaging
|
|
7
|
+
build/
|
|
8
|
+
develop-eggs/
|
|
9
|
+
dist/
|
|
10
|
+
downloads/
|
|
11
|
+
eggs/
|
|
12
|
+
.eggs/
|
|
13
|
+
lib/
|
|
14
|
+
lib64/
|
|
15
|
+
parts/
|
|
16
|
+
sdist/
|
|
17
|
+
var/
|
|
18
|
+
wheels/
|
|
19
|
+
share/python-wheels/
|
|
20
|
+
*.egg-info/
|
|
21
|
+
.installed.cfg
|
|
22
|
+
*.egg
|
|
23
|
+
|
|
24
|
+
# Virtual environments
|
|
25
|
+
.env
|
|
26
|
+
.venv
|
|
27
|
+
env/
|
|
28
|
+
venv/
|
|
29
|
+
ENV/
|
|
30
|
+
env.bak/
|
|
31
|
+
venv.bak/
|
|
32
|
+
|
|
33
|
+
# PyInstaller
|
|
34
|
+
*.manifest
|
|
35
|
+
*.spec
|
|
36
|
+
|
|
37
|
+
# Installer logs
|
|
38
|
+
pip-log.txt
|
|
39
|
+
pip-delete-this-directory.txt
|
|
40
|
+
|
|
41
|
+
# Unit test / coverage reports
|
|
42
|
+
htmlcov/
|
|
43
|
+
.tox/
|
|
44
|
+
.nox/
|
|
45
|
+
.coverage
|
|
46
|
+
.coverage.*
|
|
47
|
+
.cache
|
|
48
|
+
nosetests.xml
|
|
49
|
+
coverage.xml
|
|
50
|
+
*.cover
|
|
51
|
+
*.py,cover
|
|
52
|
+
.hypothesis/
|
|
53
|
+
.pytest_cache/
|
|
54
|
+
|
|
55
|
+
# Jupyter Notebook checkpoints
|
|
56
|
+
.ipynb_checkpoints
|
|
57
|
+
|
|
58
|
+
# IDE / Editor files
|
|
59
|
+
.vscode/
|
|
60
|
+
.idea/
|
|
61
|
+
*.swp
|
|
62
|
+
*.swo
|
|
63
|
+
*.bak
|
|
64
|
+
*~
|
|
65
|
+
|
|
66
|
+
# macOS / Linux / Windows system files
|
|
67
|
+
.DS_Store
|
|
68
|
+
Thumbs.db
|
|
69
|
+
desktop.ini
|
|
70
|
+
|
|
71
|
+
# Local config / secrets (optional)
|
|
72
|
+
*.env
|
|
73
|
+
.env.*
|
|
74
|
+
|
|
75
|
+
# Project-specific
|
|
76
|
+
__pypackages__/
|
|
77
|
+
|
psistat-1.0.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2022 Joe Defen
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
psistat-1.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: psistat
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: psistat - A simple command-line system status tool.
|
|
5
|
+
Keywords: system,status,cli
|
|
6
|
+
Author-email: Joe Defen <joedef@duck.com>
|
|
7
|
+
Requires-Python: >=3.8
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
Classifier: Development Status :: 3 - Alpha
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
12
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Topic :: System :: Monitoring
|
|
15
|
+
License-File: LICENSE
|
|
16
|
+
Requires-Dist: console-window >= 1.0.0
|
|
17
|
+
|
|
18
|
+
# Linux Pressure Stall Information (PSI) Status App
|
|
19
|
+
`psistat` is a python3 program to display the PSIs and to capture/display exception events. See [PSI - Pressure Stall Information — The Linux Kernel documentation](https://docs.kernel.org/accounting/psi.html) for more information.
|
|
20
|
+
|
|
21
|
+
## The Unique Value of `psistat`
|
|
22
|
+
|
|
23
|
+
`psistat` fills a critical gap in Linux monitoring by providing one simple package that offers:
|
|
24
|
+
|
|
25
|
+
* **High-Granularity, Calculated Averages:** It provides **1s and 3s averages**—data points that are more relevant for real-time latency debugging than the kernel's $60s$/$300s$ averages.
|
|
26
|
+
* **Smart Event Logging:** It automatically logs incidents that cross user-defined thresholds ($1\%-99\%$), implements a **cooldown period** to prevent spam, and provides forensic data (sequence number, absolute time, interval used).
|
|
27
|
+
* **TUI and Dump Capability:** The ability to instantly **dump the historical log** for copy/paste is a massive feature for quick troubleshooting and reporting that no other simple tool provides.
|
|
28
|
+
* **Simplicity:** It's a Python application installed via `pipx`—easy to install, easy to run, and requires zero configuration.
|
|
29
|
+
|
|
30
|
+
While the raw PSI data exists in the kernel, no tool currently offers the combination of live TUI monitoring, high-resolution calculation, and intelligent event logging that `psistat` does.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## Interpreting PSI Values
|
|
35
|
+
|
|
36
|
+
PSI percentages represent the fraction of time that some or all tasks were stalled waiting for a resource. Since PSIs are workload-dependent, there is no single "bad" value, but these are general guidelines:
|
|
37
|
+
|
|
38
|
+
* **`full` Stalls (Critical):** Any sustained value $\mathbf{\ge 1\%}$ for `cpu.full`, `io.full`, or `memory.full` on the **60s or 300s** average is a strong indication of a chronic bottleneck (starvation/deadlock) and demands investigation.
|
|
39
|
+
* **`some` Stalls (Warning/Actionable):** Sustained values on the **60s or 300s** average are a more reliable indicator of system health than momentary spikes:
|
|
40
|
+
* $\mathbf{10\%-25\%}$: Performance degradation is likely. This is often the ideal range to trigger load-shedding or autoscale.
|
|
41
|
+
* $\mathbf{\ge 25\%}$: Performance is severely impacted and the system is likely overloaded.
|
|
42
|
+
|
|
43
|
+
`psistat` provides calculated 1s, 3s, and 10s averages, which are derived from the kernel's cumulative stall count. These short-term metrics offer high-resolution visibility into immediate performance bottlenecks that are intentionally smoothed out by the kernel's built-in $60s$ and $300s$ averages. Use the **1s average** to spot **transient spikes** (e.g., a short I/O hiccup or lock contention) that cause latency for users; use the **3s and 10s averages** to confirm if a stall is **momentary** or is becoming a more sustained problem before it affects the long-term averages.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Installation
|
|
48
|
+
```
|
|
49
|
+
pipx install psistat
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
## Instruction for Use
|
|
53
|
+
Typically, simply run `psistat`. Its window looks like:
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
PSIs 18:54:04 | [t]hresh=5% [i]tvl=1 [b]rief=off [d]ump ?:help [q]uit
|
|
57
|
+
---1s ---3s --10s --60s -300s Full.Stall% ---1s ---3s --10s --60s -300s Some.Stall%
|
|
58
|
+
0.0 0.0 0.0 0.0 0.0 cpu.full 1.7 2.1 6.7 1.1 0.5 cpu.some
|
|
59
|
+
0.0 0.1 0.2 0.1 0.1 io.full 0.0 0.2 0.3 0.1 0.1 io.some
|
|
60
|
+
0.0 0.0 1.4 0.2 0.0 memory.full 0.0 0.0 1.5 0.2 0.0 memory.some
|
|
61
|
+
──────────────────────────────────────────────────────────────────────────────────────────────
|
|
62
|
+
000 8s: 10-18 18:53:56.099 ................... 10.5% cpu.some >=5% i=1s
|
|
63
|
+
001 9s: 10-18 18:53:55.097 14.2% memory.full 14.5% memory.some >=5% i=1s
|
|
64
|
+
002 9s: 10-18 18:53:55.097 ................... 40.7% cpu.some >=5% i=1s
|
|
65
|
+
003 26s: 10-18 18:53:37.995 10.0% io.full 10.5% io.some >=5% i=1s
|
|
66
|
+
004 1m26s: 10-18 18:52:37.665 5.8% io.full 6.0% io.some >=5% i=1s
|
|
67
|
+
005 3m35s: 10-18 18:50:28.977 ................... 5.2% io.some >=5% i=1s
|
|
68
|
+
006 4m35s: 10-18 18:49:28.644 ................... 5.0% io.some >=5% i=1s
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
`psistat` displays:
|
|
72
|
+
* The first line of the screen shows some of the control keys and values.
|
|
73
|
+
* `t` prompts for a new event threshold percent. Values between 1% and 99% are supported.
|
|
74
|
+
* `i` changes the event interval; e.g., if it is 10s, then only the 10s average stat is watched for events.
|
|
75
|
+
* `b` toggles *brief mode*. When enabled, only events that fit on the screen are shown (disables scrolling). When disabled (Full mode), the full history (up to 200 events) is scrollable.
|
|
76
|
+
* `d` immediately dumps the full event history (up to 1000 events) to the terminal, exits the display mode for easy copy/paste, and prompts you to return when ready.
|
|
77
|
+
A dump will drop out of the display mode so you can copy events to another app, and allow you to display mode return when ready.
|
|
78
|
+
* `?` will show all the control and navagation keys and the current values of the controls.
|
|
79
|
+
* `q` will terminated the app.
|
|
80
|
+
* On the top portion of the screen, the 1s, 3s, 10s, 60s, and 300s PSI stats (each is a percent stalled during the interval).
|
|
81
|
+
* The 60s and 300s are averages computed by the system.
|
|
82
|
+
* Below the stats is a list of exception events beginning with the most recent;
|
|
83
|
+
each event includes:
|
|
84
|
+
* A sequence number from 000 for the most recent.
|
|
85
|
+
* The relative time and absolute time of the event.
|
|
86
|
+
* For the 'full' and 'some' stats:
|
|
87
|
+
* The pressure value as a percentage that met or exceeded the current threshold.
|
|
88
|
+
* The type of event.
|
|
89
|
+
* NOTE: Since the 'full' and 'some' stats are highly correlated, you will often see both in one "event" as `psistat` collects them.
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
`psistat` has only a few options:
|
|
93
|
+
```
|
|
94
|
+
usage: psistat [-h] [-b] [-i {1,3,10,60,300}] [-t THRESHOLD_PCT]
|
|
95
|
+
|
|
96
|
+
options:
|
|
97
|
+
-h, --help show this help message and exit
|
|
98
|
+
-b, --brief whether present only as meay events as fit on the screen
|
|
99
|
+
-i {1,3,10,60,300}, --event-interval {1,3,10,60,300}
|
|
100
|
+
average interval used for events
|
|
101
|
+
-t THRESHOLD_PCT, --threshold-pct THRESHOLD_PCT
|
|
102
|
+
event threshold percent [1-99,dflt=5]
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
## Load Test
|
|
106
|
+
To create some test loads, you can use the `create_load.c` program found at
|
|
107
|
+
[psi-by-example](https://github.com/shuveb/psi-by-example).
|
|
108
|
+
|
|
109
|
+
|
psistat-1.0.0/README.md
ADDED
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
# Linux Pressure Stall Information (PSI) Status App
|
|
2
|
+
`psistat` is a python3 program to display the PSIs and to capture/display exception events. See [PSI - Pressure Stall Information — The Linux Kernel documentation](https://docs.kernel.org/accounting/psi.html) for more information.
|
|
3
|
+
|
|
4
|
+
## The Unique Value of `psistat`
|
|
5
|
+
|
|
6
|
+
`psistat` fills a critical gap in Linux monitoring by providing one simple package that offers:
|
|
7
|
+
|
|
8
|
+
* **High-Granularity, Calculated Averages:** It provides **1s and 3s averages**—data points that are more relevant for real-time latency debugging than the kernel's $60s$/$300s$ averages.
|
|
9
|
+
* **Smart Event Logging:** It automatically logs incidents that cross user-defined thresholds ($1\%-99\%$), implements a **cooldown period** to prevent spam, and provides forensic data (sequence number, absolute time, interval used).
|
|
10
|
+
* **TUI and Dump Capability:** The ability to instantly **dump the historical log** for copy/paste is a massive feature for quick troubleshooting and reporting that no other simple tool provides.
|
|
11
|
+
* **Simplicity:** It's a Python application installed via `pipx`—easy to install, easy to run, and requires zero configuration.
|
|
12
|
+
|
|
13
|
+
While the raw PSI data exists in the kernel, no tool currently offers the combination of live TUI monitoring, high-resolution calculation, and intelligent event logging that `psistat` does.
|
|
14
|
+
|
|
15
|
+
---
|
|
16
|
+
|
|
17
|
+
## Interpreting PSI Values
|
|
18
|
+
|
|
19
|
+
PSI percentages represent the fraction of time that some or all tasks were stalled waiting for a resource. Since PSIs are workload-dependent, there is no single "bad" value, but these are general guidelines:
|
|
20
|
+
|
|
21
|
+
* **`full` Stalls (Critical):** Any sustained value $\mathbf{\ge 1\%}$ for `cpu.full`, `io.full`, or `memory.full` on the **60s or 300s** average is a strong indication of a chronic bottleneck (starvation/deadlock) and demands investigation.
|
|
22
|
+
* **`some` Stalls (Warning/Actionable):** Sustained values on the **60s or 300s** average are a more reliable indicator of system health than momentary spikes:
|
|
23
|
+
* $\mathbf{10\%-25\%}$: Performance degradation is likely. This is often the ideal range to trigger load-shedding or autoscale.
|
|
24
|
+
* $\mathbf{\ge 25\%}$: Performance is severely impacted and the system is likely overloaded.
|
|
25
|
+
|
|
26
|
+
`psistat` provides calculated 1s, 3s, and 10s averages, which are derived from the kernel's cumulative stall count. These short-term metrics offer high-resolution visibility into immediate performance bottlenecks that are intentionally smoothed out by the kernel's built-in $60s$ and $300s$ averages. Use the **1s average** to spot **transient spikes** (e.g., a short I/O hiccup or lock contention) that cause latency for users; use the **3s and 10s averages** to confirm if a stall is **momentary** or is becoming a more sustained problem before it affects the long-term averages.
|
|
27
|
+
|
|
28
|
+
---
|
|
29
|
+
|
|
30
|
+
## Installation
|
|
31
|
+
```
|
|
32
|
+
pipx install psistat
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Instruction for Use
|
|
36
|
+
Typically, simply run `psistat`. Its window looks like:
|
|
37
|
+
|
|
38
|
+
```
|
|
39
|
+
PSIs 18:54:04 | [t]hresh=5% [i]tvl=1 [b]rief=off [d]ump ?:help [q]uit
|
|
40
|
+
---1s ---3s --10s --60s -300s Full.Stall% ---1s ---3s --10s --60s -300s Some.Stall%
|
|
41
|
+
0.0 0.0 0.0 0.0 0.0 cpu.full 1.7 2.1 6.7 1.1 0.5 cpu.some
|
|
42
|
+
0.0 0.1 0.2 0.1 0.1 io.full 0.0 0.2 0.3 0.1 0.1 io.some
|
|
43
|
+
0.0 0.0 1.4 0.2 0.0 memory.full 0.0 0.0 1.5 0.2 0.0 memory.some
|
|
44
|
+
──────────────────────────────────────────────────────────────────────────────────────────────
|
|
45
|
+
000 8s: 10-18 18:53:56.099 ................... 10.5% cpu.some >=5% i=1s
|
|
46
|
+
001 9s: 10-18 18:53:55.097 14.2% memory.full 14.5% memory.some >=5% i=1s
|
|
47
|
+
002 9s: 10-18 18:53:55.097 ................... 40.7% cpu.some >=5% i=1s
|
|
48
|
+
003 26s: 10-18 18:53:37.995 10.0% io.full 10.5% io.some >=5% i=1s
|
|
49
|
+
004 1m26s: 10-18 18:52:37.665 5.8% io.full 6.0% io.some >=5% i=1s
|
|
50
|
+
005 3m35s: 10-18 18:50:28.977 ................... 5.2% io.some >=5% i=1s
|
|
51
|
+
006 4m35s: 10-18 18:49:28.644 ................... 5.0% io.some >=5% i=1s
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
`psistat` displays:
|
|
55
|
+
* The first line of the screen shows some of the control keys and values.
|
|
56
|
+
* `t` prompts for a new event threshold percent. Values between 1% and 99% are supported.
|
|
57
|
+
* `i` changes the event interval; e.g., if it is 10s, then only the 10s average stat is watched for events.
|
|
58
|
+
* `b` toggles *brief mode*. When enabled, only events that fit on the screen are shown (disables scrolling). When disabled (Full mode), the full history (up to 200 events) is scrollable.
|
|
59
|
+
* `d` immediately dumps the full event history (up to 1000 events) to the terminal, exits the display mode for easy copy/paste, and prompts you to return when ready.
|
|
60
|
+
A dump will drop out of the display mode so you can copy events to another app, and allow you to display mode return when ready.
|
|
61
|
+
* `?` will show all the control and navagation keys and the current values of the controls.
|
|
62
|
+
* `q` will terminated the app.
|
|
63
|
+
* On the top portion of the screen, the 1s, 3s, 10s, 60s, and 300s PSI stats (each is a percent stalled during the interval).
|
|
64
|
+
* The 60s and 300s are averages computed by the system.
|
|
65
|
+
* Below the stats is a list of exception events beginning with the most recent;
|
|
66
|
+
each event includes:
|
|
67
|
+
* A sequence number from 000 for the most recent.
|
|
68
|
+
* The relative time and absolute time of the event.
|
|
69
|
+
* For the 'full' and 'some' stats:
|
|
70
|
+
* The pressure value as a percentage that met or exceeded the current threshold.
|
|
71
|
+
* The type of event.
|
|
72
|
+
* NOTE: Since the 'full' and 'some' stats are highly correlated, you will often see both in one "event" as `psistat` collects them.
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
`psistat` has only a few options:
|
|
76
|
+
```
|
|
77
|
+
usage: psistat [-h] [-b] [-i {1,3,10,60,300}] [-t THRESHOLD_PCT]
|
|
78
|
+
|
|
79
|
+
options:
|
|
80
|
+
-h, --help show this help message and exit
|
|
81
|
+
-b, --brief whether present only as meay events as fit on the screen
|
|
82
|
+
-i {1,3,10,60,300}, --event-interval {1,3,10,60,300}
|
|
83
|
+
average interval used for events
|
|
84
|
+
-t THRESHOLD_PCT, --threshold-pct THRESHOLD_PCT
|
|
85
|
+
event threshold percent [1-99,dflt=5]
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## Load Test
|
|
89
|
+
To create some test loads, you can use the `create_load.c` program found at
|
|
90
|
+
[psi-by-example](https://github.com/shuveb/psi-by-example).
|
|
91
|
+
|
psistat-1.0.0/psistat.py
ADDED
|
@@ -0,0 +1,375 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
# -*- coding: utf-8 -*-
|
|
3
|
+
"""
|
|
4
|
+
Copyright (c) 2022-2025 Joe Defen
|
|
5
|
+
|
|
6
|
+
Pressure Stall Indicators (PSI) status program. Shows:
|
|
7
|
+
- 1, 3, and 5 second running averages( in percent) of the 5 PSIs
|
|
8
|
+
- the recent history of PSI events that are defined by
|
|
9
|
+
exceeding a threshold percentage in a second.
|
|
10
|
+
|
|
11
|
+
Now uses the 'console-window' library for the TUI interface.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
# pylint: disable=invalid-name,too-many-instance-attributes,import-outside-toplevel,broad-except,too-many-locals
|
|
15
|
+
|
|
16
|
+
import sys
|
|
17
|
+
import os
|
|
18
|
+
import time
|
|
19
|
+
import re
|
|
20
|
+
import traceback
|
|
21
|
+
from datetime import datetime
|
|
22
|
+
from types import SimpleNamespace
|
|
23
|
+
import curses as cs
|
|
24
|
+
import locale
|
|
25
|
+
from types import SimpleNamespace
|
|
26
|
+
from console_window import ConsoleWindow, OptionSpinner
|
|
27
|
+
|
|
28
|
+
locale.setlocale(locale.LC_ALL, '') # needed to draw unicode chars
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
SAMPLES = 11
|
|
32
|
+
BILLION = 1000000000
|
|
33
|
+
|
|
34
|
+
##############################################################################
|
|
35
|
+
## Turn time differences in seconds to a compact representation;
|
|
36
|
+
## e.g., '18h·39m'
|
|
37
|
+
##############################################################################
|
|
38
|
+
def ago_str(delta_secs):
|
|
39
|
+
"""Prints a 6-character string (max) the represents how long
|
|
40
|
+
ago (i.e., delta_secs) is briefly."""
|
|
41
|
+
ago = int(max(0, round(delta_secs if delta_secs >= 0 else -delta_secs)))
|
|
42
|
+
divs = (60, 60, 24, 7, 52, 9999999)
|
|
43
|
+
units = ('s', 'm', 'h', 'd', 'w', 'y')
|
|
44
|
+
vals = (ago%60, int(ago/60)) # seed with secs, mins (step til 2nd fits)
|
|
45
|
+
uidx = 1 # best units
|
|
46
|
+
for div in divs[1:]:
|
|
47
|
+
if vals[1] < div:
|
|
48
|
+
break
|
|
49
|
+
vals = (vals[1]%div, int(vals[1]/div))
|
|
50
|
+
uidx += 1
|
|
51
|
+
rv = '{:d}{}'.format(vals[1], units[uidx]) if vals[1] else ''
|
|
52
|
+
fmt = '{:02d}{}' if vals[1] else '{:2d}{}'
|
|
53
|
+
rv += fmt.format(vals[0], units[uidx-1])
|
|
54
|
+
return rv
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class PressureGroup:
|
|
58
|
+
"""Handler for one PSI group (i.e., cpu, io, or memory)
|
|
59
|
+
"""
|
|
60
|
+
pattern = re.compile(r'^(\S+)\b.*\bavg60=(\d+\.\d+)\b.*\bavg300=(\d+\.\d+)\b.*\btotal=(\d+)$')
|
|
61
|
+
def __init__(self, tag, stats):
|
|
62
|
+
self.stats = stats
|
|
63
|
+
self.avgs = {60: 0, 300: 0} # averages from /proc/pressure/{tag}
|
|
64
|
+
self.tag = tag
|
|
65
|
+
self.fullpath = '/proc/pressure/' + tag
|
|
66
|
+
self.handle = open(self.fullpath)
|
|
67
|
+
|
|
68
|
+
def sample_ns(self):
|
|
69
|
+
""" return the initialized namespace for a sample.
|
|
70
|
+
It includes the list of recent microseconds of pressure samples and
|
|
71
|
+
the averages for 60s and 300s"""
|
|
72
|
+
return SimpleNamespace(micros=[], avgs={60: 0, 300:0})
|
|
73
|
+
|
|
74
|
+
def add_sample(self):
|
|
75
|
+
"""Read/Parse the /proc/pressure/{group} file"""
|
|
76
|
+
self.handle.seek(0)
|
|
77
|
+
document = self.handle.read()
|
|
78
|
+
allowed_subkeys = {'full', 'some'}
|
|
79
|
+
for line in document.splitlines():
|
|
80
|
+
match = self.pattern.match(line)
|
|
81
|
+
|
|
82
|
+
# NOTE: Your original regex assumed match was successful. Adding check.
|
|
83
|
+
if not match:
|
|
84
|
+
continue
|
|
85
|
+
subkey, micro = match.group(1), match.group(4)
|
|
86
|
+
avg60, avg300 = match.group(2), match.group(3)
|
|
87
|
+
if subkey not in allowed_subkeys:
|
|
88
|
+
continue
|
|
89
|
+
allowed_subkeys.discard(subkey)
|
|
90
|
+
key = f'{self.tag}.{subkey}'
|
|
91
|
+
ns = self.stats.get(key, None)
|
|
92
|
+
if not ns:
|
|
93
|
+
ns = self.stats[key] = self.sample_ns()
|
|
94
|
+
ns.micros.insert(0, int(micro))
|
|
95
|
+
del ns.micros[SAMPLES:]
|
|
96
|
+
ns.avgs[60], ns.avgs[300] = float(avg60), float(avg300)
|
|
97
|
+
assert not allowed_subkeys, f"{self.fullpath} expecting full+some values"
|
|
98
|
+
|
|
99
|
+
class PsiStat:
|
|
100
|
+
"""Class to display PSI information including
|
|
101
|
+
running averages and exception events"""
|
|
102
|
+
singleton = None
|
|
103
|
+
def __init__(self, threshold=5, brief=False, itvl=10):
|
|
104
|
+
assert self.singleton is None
|
|
105
|
+
PsiStat.singleton = self
|
|
106
|
+
self.next_mono = time.monotonic_ns() # when to sleep to next
|
|
107
|
+
self.stats = { 'cpu.full': None, 'cpu.some': None,
|
|
108
|
+
'io.full': None, 'io.some': None,
|
|
109
|
+
'memory.full': None, 'memory.some': None
|
|
110
|
+
}
|
|
111
|
+
self.monos = []
|
|
112
|
+
self.times = []
|
|
113
|
+
self.psgs = []
|
|
114
|
+
self.events = []
|
|
115
|
+
self.event_floor = {}
|
|
116
|
+
self.lineno = 0 # Not strictly needed with ConsoleWindow, but harmless.
|
|
117
|
+
self.last_sample_mono = 0
|
|
118
|
+
|
|
119
|
+
# --- CONSOLE-WINDOW SETUP ---
|
|
120
|
+
# Setup OptionSpinner for 'j' and 'k' keys
|
|
121
|
+
self.spin = OptionSpinner()
|
|
122
|
+
self.opts = self.spin.default_obj
|
|
123
|
+
self.spin.add_key('help_mode', '? - toggle help screen', vals=[False, True])
|
|
124
|
+
self.spin.add_key(attr='event_interval', descr='i - event interval (secs)',
|
|
125
|
+
vals=[10, 3, 1, 300, 60])
|
|
126
|
+
self.spin.add_key(attr='threshold', descr='t - set event threshold percent [1,99]',
|
|
127
|
+
prompt="Enter event threshold percent in range [1,99]")
|
|
128
|
+
self.spin.add_key(attr='brief', descr='b - only load events that fit on screen',
|
|
129
|
+
vals=['off', 'on'])
|
|
130
|
+
self.spin.add_key(attr='dump_now', descr='d - dump event log now',
|
|
131
|
+
vals=[False, True])
|
|
132
|
+
self.spin.add_key(attr='quit', descr='q or CTRL-c - quit program',
|
|
133
|
+
vals=[False, True])
|
|
134
|
+
|
|
135
|
+
# FIXME: keys
|
|
136
|
+
others = ''
|
|
137
|
+
other_keys = set(ord(x) for x in others)
|
|
138
|
+
# other_keys.add(cs.KEY_ENTER)
|
|
139
|
+
# other_keys.add(27) # ESCAPE
|
|
140
|
+
# other_keys.add(10) # another form of ENTER
|
|
141
|
+
self.opts.brief = 'on' if brief else 'off'
|
|
142
|
+
self.opts.threshold = threshold
|
|
143
|
+
self.opts.event_interval = itvl
|
|
144
|
+
|
|
145
|
+
# Initialize ConsoleWindow
|
|
146
|
+
self.window = ConsoleWindow(head_line=True, keys=self.spin.keys^other_keys)
|
|
147
|
+
|
|
148
|
+
for tag in 'cpu', 'io', 'memory':
|
|
149
|
+
self.psgs.append(PressureGroup(tag, self.stats))
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def dump_event_log(self, do_return=False):
|
|
153
|
+
""" drop out of Window and dump the events """
|
|
154
|
+
if self.window:
|
|
155
|
+
self.window.stop_curses()
|
|
156
|
+
os.system('clear; stty sane')
|
|
157
|
+
for event in reversed(self.events):
|
|
158
|
+
print(event[1])
|
|
159
|
+
if do_return:
|
|
160
|
+
os.system(r' /bin/echo -e "\n\n===== Press ENTER to return to psistat ====> \c"; read FOO')
|
|
161
|
+
if self.window:
|
|
162
|
+
self.window._start_curses()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def prc_samples(self, valid_sample_count):
|
|
166
|
+
"""Show the current PSI stats/events using the
|
|
167
|
+
most recent samples while also detecting new events
|
|
168
|
+
for exceptional samples. Then show the recent events.
|
|
169
|
+
"""
|
|
170
|
+
# Access threshold from OptionSpinner object
|
|
171
|
+
threshold = self.opts.threshold
|
|
172
|
+
itvl = self.opts.event_interval
|
|
173
|
+
brief = self.opts.brief
|
|
174
|
+
dt_object = datetime.fromtimestamp(time.time())
|
|
175
|
+
hhmmss = dt_object.strftime("%H:%M:%S")
|
|
176
|
+
|
|
177
|
+
title = f'PSIs {hhmmss} | [t]hresh={threshold}% [i]tvl={itvl}s [b]rief={brief} [d]ump ?:help [q]uit'
|
|
178
|
+
self.window.add_header(title)
|
|
179
|
+
|
|
180
|
+
# Draw Header
|
|
181
|
+
header = (f' {"1s":->5} {"3s":->5} {"10s":->5} {"60s":->5} {"300s":->5} {"Full.Stall%":>11}'
|
|
182
|
+
+ f' {"1s":->5} {"3s":->5} {"10s":->5} {"60s":->5} {"300s":->5} {"Some.Stall%":>11}')
|
|
183
|
+
self.window.add_header(header)
|
|
184
|
+
|
|
185
|
+
# --- Draw PSI Stats ---
|
|
186
|
+
line, event, specifically = '', '', ''
|
|
187
|
+
now = time.time()
|
|
188
|
+
for idx, key in enumerate(self.stats):
|
|
189
|
+
micros = self.stats[key].micros
|
|
190
|
+
del micros[valid_sample_count:]
|
|
191
|
+
avgs = self.stats[key].avgs
|
|
192
|
+
|
|
193
|
+
for ii in (1, 3, 10, 60, 300):
|
|
194
|
+
if ii in (1, 3, 10):
|
|
195
|
+
if len(micros) <= ii:
|
|
196
|
+
line += ' ' + 'n/a'.rjust(5)
|
|
197
|
+
continue
|
|
198
|
+
|
|
199
|
+
delta_micros = (micros[0] - micros[ii])
|
|
200
|
+
delta_monos = (self.monos[0] - self.monos[ii])
|
|
201
|
+
|
|
202
|
+
# Avoid division by zero if delta_monos is unexpectedly zero
|
|
203
|
+
if delta_monos == 0:
|
|
204
|
+
pct = 0.0
|
|
205
|
+
else:
|
|
206
|
+
pct = 100 * 1000 * delta_micros / delta_monos
|
|
207
|
+
|
|
208
|
+
else:
|
|
209
|
+
pct = avgs.get(ii, 0.0)
|
|
210
|
+
|
|
211
|
+
line += f' {pct:5.1f}'
|
|
212
|
+
|
|
213
|
+
# detect/collect the 1s samples that exceed the threshold
|
|
214
|
+
floor_key = f'{key}.{ii}'
|
|
215
|
+
floor = self.event_floor.get(floor_key, 0)
|
|
216
|
+
if ii == itvl and round(pct, 2) >= threshold and now >= floor:
|
|
217
|
+
if not event:
|
|
218
|
+
dt_object = datetime.fromtimestamp(now)
|
|
219
|
+
# Truncate microseconds to milliseconds (.mmm) with [:-3]
|
|
220
|
+
event += dt_object.strftime("%m-%d %H:%M:%S.%f")[:-3]
|
|
221
|
+
|
|
222
|
+
specifically = f'{pct:7.1f}% {key:<11s}'
|
|
223
|
+
if '.full' not in event and '.some' in key:
|
|
224
|
+
event += ' ' + '.' * len(specifically)
|
|
225
|
+
event += ' ' + specifically
|
|
226
|
+
self.event_floor[floor_key] = now + itvl
|
|
227
|
+
line += f' {key:<11}'
|
|
228
|
+
|
|
229
|
+
if idx % 2 == 0:
|
|
230
|
+
line += ' '
|
|
231
|
+
else:
|
|
232
|
+
if not self.opts.help_mode:
|
|
233
|
+
self.window.add_header(line)
|
|
234
|
+
line = ''
|
|
235
|
+
if event:
|
|
236
|
+
if '.some' not in event and '.full' in key:
|
|
237
|
+
event += ' ' + '.' * len(specifically)
|
|
238
|
+
event += f' >={threshold}% i={itvl}s'
|
|
239
|
+
self.events.insert(0, (time.monotonic_ns(), event))
|
|
240
|
+
event = ''
|
|
241
|
+
|
|
242
|
+
del self.events[1000:]
|
|
243
|
+
|
|
244
|
+
# --- Draw Events ---
|
|
245
|
+
if not self.opts.help_mode:
|
|
246
|
+
self.window.calc()
|
|
247
|
+
limit = (self.window.rows - self.window.body_base) if self.opts.brief == 'on' else 200
|
|
248
|
+
for idx, event in enumerate(self.events):
|
|
249
|
+
if idx >= limit:
|
|
250
|
+
break
|
|
251
|
+
mono_ns, descr = event
|
|
252
|
+
ago = ago_str((time.monotonic_ns() - mono_ns)/BILLION).strip()
|
|
253
|
+
self.window.put_body(f'{idx:03d} {ago:>6s}: {descr}')
|
|
254
|
+
|
|
255
|
+
def show_help_screen(self):
|
|
256
|
+
""" TBD """
|
|
257
|
+
self.window.clear()
|
|
258
|
+
self.window.set_pick_mode(False)
|
|
259
|
+
self.spin.show_help_nav_keys(self.window)
|
|
260
|
+
self.spin.show_help_body(self.window)
|
|
261
|
+
lines = [
|
|
262
|
+
' CTRL-c - quit program'
|
|
263
|
+
]
|
|
264
|
+
for line in lines:
|
|
265
|
+
self.window.put_body(line)
|
|
266
|
+
|
|
267
|
+
def count_valid_samples(self):
|
|
268
|
+
""" Valid samples are no worse that 1.25s later than the one before
|
|
269
|
+
and no more than SAMPLES """
|
|
270
|
+
# return SAMPLES-1
|
|
271
|
+
count = 1
|
|
272
|
+
del self.monos[SAMPLES:]
|
|
273
|
+
for loop in range(0, len(self.monos) - 1):
|
|
274
|
+
delta_ns = self.monos[loop] - self.monos[loop+1]
|
|
275
|
+
if delta_ns > 1000000000 * 1.25:
|
|
276
|
+
break
|
|
277
|
+
count += 1
|
|
278
|
+
return count
|
|
279
|
+
|
|
280
|
+
def process_psgs(self):
|
|
281
|
+
""" TBD """
|
|
282
|
+
for psg in self.psgs:
|
|
283
|
+
psg.add_sample()
|
|
284
|
+
|
|
285
|
+
self.times.insert(0, time.time())
|
|
286
|
+
mono = time.monotonic_ns()
|
|
287
|
+
self.monos.insert(0, mono)
|
|
288
|
+
valid_sample_count = self.count_valid_samples()
|
|
289
|
+
del self.monos[valid_sample_count:]
|
|
290
|
+
del self.times[valid_sample_count:]
|
|
291
|
+
|
|
292
|
+
if not self.opts.help_mode:
|
|
293
|
+
self.window.clear()
|
|
294
|
+
|
|
295
|
+
self.prc_samples(valid_sample_count)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def loop(self):
|
|
299
|
+
"""Do one loop of collecting samples, and then displaying running
|
|
300
|
+
averages and exception events."""
|
|
301
|
+
timeout_secs, min_timout_secs = 1.0, 0.9
|
|
302
|
+
|
|
303
|
+
if time.monotonic() - self.last_sample_mono >= min_timout_secs:
|
|
304
|
+
self.process_psgs()
|
|
305
|
+
self.last_sample_mono = time.monotonic()
|
|
306
|
+
|
|
307
|
+
if self.opts.help_mode:
|
|
308
|
+
self.show_help_screen()
|
|
309
|
+
|
|
310
|
+
# --- CONSOLE-WINDOW LOOP INTEGRATION ---
|
|
311
|
+
|
|
312
|
+
# Render the screen
|
|
313
|
+
self.window.render()
|
|
314
|
+
sleep_secs = timeout_secs - (time.monotonic() - self.last_sample_mono)
|
|
315
|
+
|
|
316
|
+
# Wait for input/timeout using ConsoleWindow.prompt()
|
|
317
|
+
keystroke = self.window.prompt(seconds=sleep_secs)
|
|
318
|
+
|
|
319
|
+
if keystroke is not None:
|
|
320
|
+
# Pass key to OptionSpinner for handling 'j' and 'k'
|
|
321
|
+
if self.spin.do_key(keystroke, self.window):
|
|
322
|
+
if isinstance(self.opts.threshold, str):
|
|
323
|
+
try:
|
|
324
|
+
self.opts.threshold = int(self.opts.threshold)
|
|
325
|
+
if self.opts.threshold < 1:
|
|
326
|
+
self.opts.threshold = 1
|
|
327
|
+
elif self.opts.threshold > 99:
|
|
328
|
+
self.opts.threshold = 99
|
|
329
|
+
except Exception:
|
|
330
|
+
self.opts.threshold = 999999
|
|
331
|
+
if self.opts.dump_now:
|
|
332
|
+
self.dump_event_log(do_return=True)
|
|
333
|
+
self.opts.dump_now = False
|
|
334
|
+
if self.opts.quit:
|
|
335
|
+
sys.exit(0)
|
|
336
|
+
|
|
337
|
+
return True
|
|
338
|
+
if keystroke in (ord('q'), ord('Q')):
|
|
339
|
+
return False # Quit
|
|
340
|
+
|
|
341
|
+
# The loop continues after timeout or non-threshold keystroke
|
|
342
|
+
return True
|
|
343
|
+
|
|
344
|
+
def main():
|
|
345
|
+
try:
|
|
346
|
+
"""Main execution function, handles CLI arguments."""
|
|
347
|
+
import argparse
|
|
348
|
+
parser = argparse.ArgumentParser()
|
|
349
|
+
parser.add_argument('-b', '--brief', action='store_true',
|
|
350
|
+
help='whether present only as meny events as fit on the screen')
|
|
351
|
+
parser.add_argument('-i', '--event-interval', default=10, type=int, choices=[1,3,10,60,300],
|
|
352
|
+
help='average interval used for events')
|
|
353
|
+
parser.add_argument('-t', '--threshold-pct', default=5, type=int,
|
|
354
|
+
help='event threshold percent [1-99,dflt=5]')
|
|
355
|
+
opts = parser.parse_args()
|
|
356
|
+
|
|
357
|
+
pct = min(max(opts.threshold_pct, 1), 99)
|
|
358
|
+
|
|
359
|
+
pstall = PsiStat(threshold=pct, brief=opts.brief, itvl=opts.event_interval)
|
|
360
|
+
|
|
361
|
+
while pstall.loop():
|
|
362
|
+
pass
|
|
363
|
+
|
|
364
|
+
except Exception as exc:
|
|
365
|
+
# Note: We no longer call Window.exit_handler(), as ConsoleWindow handles it
|
|
366
|
+
# and there is no guarantee the Window class was ever initialized.
|
|
367
|
+
if PsiStat.singleton and PsiStat.singleton.window:
|
|
368
|
+
PsiStat.singleton.window.stop_curses()
|
|
369
|
+
|
|
370
|
+
print("exception:", str(exc))
|
|
371
|
+
print(traceback.format_exc())
|
|
372
|
+
|
|
373
|
+
if __name__ == '__main__':
|
|
374
|
+
|
|
375
|
+
main()
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["flit_core >=3.9,<5"] # Updated to a current range
|
|
3
|
+
build-backend = "flit_core.buildapi"
|
|
4
|
+
|
|
5
|
+
# ----------------------------------------------------------------------
|
|
6
|
+
# FLIT MODULE CONFIGURATION
|
|
7
|
+
# Flit looks for a single file module named psistat.py
|
|
8
|
+
[tool.flit.module]
|
|
9
|
+
name = "psistat"
|
|
10
|
+
|
|
11
|
+
# ----------------------------------------------------------------------
|
|
12
|
+
# UNIFIED PROJECT METADATA
|
|
13
|
+
[project]
|
|
14
|
+
name = "psistat"
|
|
15
|
+
version = "1.0.0"
|
|
16
|
+
description = "psistat - A simple command-line system status tool."
|
|
17
|
+
readme = "README.md"
|
|
18
|
+
requires-python = ">=3.8"
|
|
19
|
+
license = {file = "LICENSE"}
|
|
20
|
+
|
|
21
|
+
# Add dependencies here if you use non-stdlib PyPI packages (e.g., requests, rich)
|
|
22
|
+
dependencies = [
|
|
23
|
+
'console-window >= 1.0.0',
|
|
24
|
+
]
|
|
25
|
+
|
|
26
|
+
authors = [
|
|
27
|
+
{name = "Joe Defen", email = "joedef@duck.com"}, # ✏️ Your info
|
|
28
|
+
]
|
|
29
|
+
|
|
30
|
+
# ✏️ Update classifiers to match your project
|
|
31
|
+
classifiers = [
|
|
32
|
+
"Development Status :: 3 - Alpha",
|
|
33
|
+
"Programming Language :: Python :: 3",
|
|
34
|
+
"Programming Language :: Python :: 3.8",
|
|
35
|
+
"License :: OSI Approved :: MIT License",
|
|
36
|
+
"Environment :: Console",
|
|
37
|
+
"Topic :: System :: Monitoring",
|
|
38
|
+
]
|
|
39
|
+
|
|
40
|
+
keywords = ["system", "status", "cli"] # ✏️ Add relevant keywords
|
|
41
|
+
|
|
42
|
+
# ----------------------------------------------------------------------
|
|
43
|
+
# COMMAND LINE SCRIPTS
|
|
44
|
+
# This tells flit to create an executable 'psistat' command
|
|
45
|
+
# that calls the 'main()' function in the 'psistat' module (psistat.py)
|
|
46
|
+
[project.scripts]
|
|
47
|
+
psistat = "psistat:main"
|