croft-signals 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- croft_signals-0.4.0/PKG-INFO +100 -0
- croft_signals-0.4.0/README.md +77 -0
- croft_signals-0.4.0/pyproject.toml +47 -0
- croft_signals-0.4.0/pyproject.toml.orig +36 -0
- croft_signals-0.4.0/src/croft/__init__.py +5 -0
- croft_signals-0.4.0/src/croft/dataset.py +301 -0
- croft_signals-0.4.0/src/croft/kube.py +31 -0
- croft_signals-0.4.0/src/croft/metrics.py +270 -0
- croft_signals-0.4.0/src/croft/plot.py +16 -0
- croft_signals-0.4.0/src/croft/prometheus.py +48 -0
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: croft-signals
|
|
3
|
+
Version: 0.4.0
|
|
4
|
+
Summary: croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames
|
|
5
|
+
Keywords: ebpf,kubernetes,autoscaling,prometheus,pandas
|
|
6
|
+
Author: Daniel Pepuho
|
|
7
|
+
License-Expression: Apache-2.0
|
|
8
|
+
Classifier: Development Status :: 3 - Alpha
|
|
9
|
+
Classifier: Intended Audience :: Science/Research
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
13
|
+
Classifier: Topic :: System :: Monitoring
|
|
14
|
+
Requires-Dist: pandas>=2.2
|
|
15
|
+
Requires-Dist: zstandard>=0.25
|
|
16
|
+
Requires-Dist: matplotlib>=3.9 ; extra == 'plot'
|
|
17
|
+
Requires-Python: >=3.12
|
|
18
|
+
Project-URL: Repository, https://github.com/danielcristho/croft
|
|
19
|
+
Project-URL: Documentation, https://croft-docs.pages.dev
|
|
20
|
+
Project-URL: Changelog, https://github.com/danielcristho/croft/blob/main/CHANGELOG.md
|
|
21
|
+
Provides-Extra: plot
|
|
22
|
+
Description-Content-Type: text/markdown
|
|
23
|
+
|
|
24
|
+
# croft for Python
|
|
25
|
+
|
|
26
|
+
The collector's signals as pandas frames, for notebooks and analysis.
|
|
27
|
+
|
|
28
|
+
```sh
|
|
29
|
+
pip install "croft-signals[plot]"
|
|
30
|
+
uv add "croft-signals[plot]"
|
|
31
|
+
uv add --editable ../python # from a project inside this repository
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## 1. Sources
|
|
35
|
+
|
|
36
|
+
| Source | Function | History | Pods |
|
|
37
|
+
|---|---|---|---|
|
|
38
|
+
| Prometheus | `metrics.signal()` and one function per signal | as long as Prometheus keeps it | joined through kube-state-metrics |
|
|
39
|
+
| collector `/metrics` | `metrics.live()` | the newest poll | `container_id` |
|
|
40
|
+
| dataset files | `dataset.signal()` and one function per signal; `dataset.read()` for raw rows | as long as the node keeps it | `container_id`, or names from `kube.names()` |
|
|
41
|
+
|
|
42
|
+
`CROFT_PROMETHEUS` sets the Prometheus URL (`PROM_HOST` is read as a fallback),
|
|
43
|
+
`CROFT_METRICS_ADDR` the collector address and `CROFT_DATASET` the dataset directory.
|
|
44
|
+
|
|
45
|
+
## 2. Signals
|
|
46
|
+
|
|
47
|
+
| Signal | Metric | Unit |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| `syscall_rate` | `croft_container_syscall_rate` | calls/s, per `syscall` |
|
|
50
|
+
| `syscall_bytes` | `croft_container_syscall_bytes_total` | bytes/s, per `syscall` |
|
|
51
|
+
| `socket_io` | `croft_container_socket_io_total` | calls/s, per `direction` |
|
|
52
|
+
| `connects` | `croft_container_connects_total` | connects/s, per `address` and `port` |
|
|
53
|
+
| `cpu_cores` | `croft_container_cpu_cores` | cores |
|
|
54
|
+
| `cpu_throttled_fraction` | `croft_container_cpu_throttled_fraction` | fraction |
|
|
55
|
+
| `cpu_pressure` | `croft_container_cpu_pressure_avg10` | % |
|
|
56
|
+
| `runqueue_wait` | `croft_container_runqueue_wait` | s/s |
|
|
57
|
+
| `inflight_requests` | `croft_container_inflight_requests` | requests |
|
|
58
|
+
|
|
59
|
+
Counters come as rates over 15 s or the step, whichever is longer; gauges are averaged over each step longer than
|
|
60
|
+
5 s. `metrics.catalog()` returns this table. The dataset holds `syscall_rate`, `runqueue_wait` and the three CPU
|
|
61
|
+
signals; the other three are on `/metrics` and Prometheus.
|
|
62
|
+
|
|
63
|
+
## 3. Usage
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from croft import metrics, plot
|
|
67
|
+
|
|
68
|
+
frame = metrics.syscall_rate("boutique", "frontend", since="1h", match={"syscall": "getpeername"})
|
|
69
|
+
plot.lines(frame, by="pod")
|
|
70
|
+
|
|
71
|
+
metrics.runqueue_wait("hotel-res", since="2026-11-18 09:00", until="2026-11-18 10:00")
|
|
72
|
+
metrics.wide(frame, by="deployment") # one column per deployment
|
|
73
|
+
metrics.query("sum(rate(container_cpu_usage_seconds_total[1m]))", since="30m")
|
|
74
|
+
metrics.live("127.0.0.1:9464") # newest poll of one collector
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Without Prometheus, the same frames come from the dataset files, with `node` and `container_id` added:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from croft import dataset, kube
|
|
81
|
+
|
|
82
|
+
names = kube.names() # container_id to namespace, pod and container, through kubectl
|
|
83
|
+
frame = dataset.syscall_rate("boutique", "frontend", since="1h", names=names, match={"syscall": "getpeername"})
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
A summed signal (`syscall_rate`, `runqueue_wait`) is the container's total per poll, averaged over the polls of
|
|
87
|
+
each step; a poll without a row for the container counts as 0, since idle processes are written every 15 polls.
|
|
88
|
+
A CPU signal is averaged over the polls that hold it. `kube.names()` knows the pods the API server holds, so
|
|
89
|
+
containers of deleted pods keep their `container_id` only.
|
|
90
|
+
|
|
91
|
+
Each row holds `time` (UTC), `namespace`, `deployment`, `pod`, `container`, the
|
|
92
|
+
signal's own labels and `value`. `deployment` comes from the pod name and is empty
|
|
93
|
+
for pods outside a Deployment. `since` and `until` take a duration before now
|
|
94
|
+
(`"90m"`), a date or a unix time; the step keeps a range under 1000 samples.
|
|
95
|
+
|
|
96
|
+
## 4. Tests
|
|
97
|
+
|
|
98
|
+
```sh
|
|
99
|
+
uv run python -m unittest discover -s tests
|
|
100
|
+
```
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# croft for Python
|
|
2
|
+
|
|
3
|
+
The collector's signals as pandas frames, for notebooks and analysis.
|
|
4
|
+
|
|
5
|
+
```sh
|
|
6
|
+
pip install "croft-signals[plot]"
|
|
7
|
+
uv add "croft-signals[plot]"
|
|
8
|
+
uv add --editable ../python # from a project inside this repository
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
## 1. Sources
|
|
12
|
+
|
|
13
|
+
| Source | Function | History | Pods |
|
|
14
|
+
|---|---|---|---|
|
|
15
|
+
| Prometheus | `metrics.signal()` and one function per signal | as long as Prometheus keeps it | joined through kube-state-metrics |
|
|
16
|
+
| collector `/metrics` | `metrics.live()` | the newest poll | `container_id` |
|
|
17
|
+
| dataset files | `dataset.signal()` and one function per signal; `dataset.read()` for raw rows | as long as the node keeps it | `container_id`, or names from `kube.names()` |
|
|
18
|
+
|
|
19
|
+
`CROFT_PROMETHEUS` sets the Prometheus URL (`PROM_HOST` is read as a fallback),
|
|
20
|
+
`CROFT_METRICS_ADDR` the collector address and `CROFT_DATASET` the dataset directory.
|
|
21
|
+
|
|
22
|
+
## 2. Signals
|
|
23
|
+
|
|
24
|
+
| Signal | Metric | Unit |
|
|
25
|
+
|---|---|---|
|
|
26
|
+
| `syscall_rate` | `croft_container_syscall_rate` | calls/s, per `syscall` |
|
|
27
|
+
| `syscall_bytes` | `croft_container_syscall_bytes_total` | bytes/s, per `syscall` |
|
|
28
|
+
| `socket_io` | `croft_container_socket_io_total` | calls/s, per `direction` |
|
|
29
|
+
| `connects` | `croft_container_connects_total` | connects/s, per `address` and `port` |
|
|
30
|
+
| `cpu_cores` | `croft_container_cpu_cores` | cores |
|
|
31
|
+
| `cpu_throttled_fraction` | `croft_container_cpu_throttled_fraction` | fraction |
|
|
32
|
+
| `cpu_pressure` | `croft_container_cpu_pressure_avg10` | % |
|
|
33
|
+
| `runqueue_wait` | `croft_container_runqueue_wait` | s/s |
|
|
34
|
+
| `inflight_requests` | `croft_container_inflight_requests` | requests |
|
|
35
|
+
|
|
36
|
+
Counters come as rates over 15 s or the step, whichever is longer; gauges are averaged over each step longer than
|
|
37
|
+
5 s. `metrics.catalog()` returns this table. The dataset holds `syscall_rate`, `runqueue_wait` and the three CPU
|
|
38
|
+
signals; the other three are on `/metrics` and Prometheus.
|
|
39
|
+
|
|
40
|
+
## 3. Usage
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from croft import metrics, plot
|
|
44
|
+
|
|
45
|
+
frame = metrics.syscall_rate("boutique", "frontend", since="1h", match={"syscall": "getpeername"})
|
|
46
|
+
plot.lines(frame, by="pod")
|
|
47
|
+
|
|
48
|
+
metrics.runqueue_wait("hotel-res", since="2026-11-18 09:00", until="2026-11-18 10:00")
|
|
49
|
+
metrics.wide(frame, by="deployment") # one column per deployment
|
|
50
|
+
metrics.query("sum(rate(container_cpu_usage_seconds_total[1m]))", since="30m")
|
|
51
|
+
metrics.live("127.0.0.1:9464") # newest poll of one collector
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
Without Prometheus, the same frames come from the dataset files, with `node` and `container_id` added:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
from croft import dataset, kube
|
|
58
|
+
|
|
59
|
+
names = kube.names() # container_id to namespace, pod and container, through kubectl
|
|
60
|
+
frame = dataset.syscall_rate("boutique", "frontend", since="1h", names=names, match={"syscall": "getpeername"})
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
A summed signal (`syscall_rate`, `runqueue_wait`) is the container's total per poll, averaged over the polls of
|
|
64
|
+
each step; a poll without a row for the container counts as 0, since idle processes are written every 15 polls.
|
|
65
|
+
A CPU signal is averaged over the polls that hold it. `kube.names()` knows the pods the API server holds, so
|
|
66
|
+
containers of deleted pods keep their `container_id` only.
|
|
67
|
+
|
|
68
|
+
Each row holds `time` (UTC), `namespace`, `deployment`, `pod`, `container`, the
|
|
69
|
+
signal's own labels and `value`. `deployment` comes from the pod name and is empty
|
|
70
|
+
for pods outside a Deployment. `since` and `until` take a duration before now
|
|
71
|
+
(`"90m"`), a date or a unix time; the step keeps a range under 1000 samples.
|
|
72
|
+
|
|
73
|
+
## 4. Tests
|
|
74
|
+
|
|
75
|
+
```sh
|
|
76
|
+
uv run python -m unittest discover -s tests
|
|
77
|
+
```
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "croft-signals"
|
|
3
|
+
version = "0.4.0"
|
|
4
|
+
description = "croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
requires-python = ">=3.12"
|
|
8
|
+
keywords = [
|
|
9
|
+
"ebpf",
|
|
10
|
+
"kubernetes",
|
|
11
|
+
"autoscaling",
|
|
12
|
+
"prometheus",
|
|
13
|
+
"pandas",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Development Status :: 3 - Alpha",
|
|
17
|
+
"Intended Audience :: Science/Research",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.12",
|
|
20
|
+
"Programming Language :: Python :: 3.13",
|
|
21
|
+
"Topic :: System :: Monitoring",
|
|
22
|
+
]
|
|
23
|
+
dependencies = [
|
|
24
|
+
"pandas>=2.2",
|
|
25
|
+
"zstandard>=0.25",
|
|
26
|
+
]
|
|
27
|
+
|
|
28
|
+
[[project.authors]]
|
|
29
|
+
name = "Daniel Pepuho"
|
|
30
|
+
|
|
31
|
+
[project.urls]
|
|
32
|
+
Repository = "https://github.com/danielcristho/croft"
|
|
33
|
+
Documentation = "https://croft-docs.pages.dev"
|
|
34
|
+
Changelog = "https://github.com/danielcristho/croft/blob/main/CHANGELOG.md"
|
|
35
|
+
|
|
36
|
+
[project.optional-dependencies]
|
|
37
|
+
plot = ["matplotlib>=3.9"]
|
|
38
|
+
|
|
39
|
+
[tool.uv.build-backend]
|
|
40
|
+
module-name = "croft"
|
|
41
|
+
|
|
42
|
+
[tool.ruff]
|
|
43
|
+
line-length = 120
|
|
44
|
+
|
|
45
|
+
[build-system]
|
|
46
|
+
requires = ["uv_build>=0.11,<0.12"]
|
|
47
|
+
build-backend = "uv_build"
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "croft-signals"
|
|
3
|
+
version = "0.4.0"
|
|
4
|
+
description = "croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
license = "Apache-2.0"
|
|
7
|
+
requires-python = ">=3.12"
|
|
8
|
+
authors = [{ name = "Daniel Pepuho" }]
|
|
9
|
+
keywords = ["ebpf", "kubernetes", "autoscaling", "prometheus", "pandas"]
|
|
10
|
+
classifiers = [
|
|
11
|
+
"Development Status :: 3 - Alpha",
|
|
12
|
+
"Intended Audience :: Science/Research",
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.12",
|
|
15
|
+
"Programming Language :: Python :: 3.13",
|
|
16
|
+
"Topic :: System :: Monitoring",
|
|
17
|
+
]
|
|
18
|
+
dependencies = ["pandas>=2.2", "zstandard>=0.25"]
|
|
19
|
+
|
|
20
|
+
[project.urls]
|
|
21
|
+
Repository = "https://github.com/danielcristho/croft"
|
|
22
|
+
Documentation = "https://croft-docs.pages.dev"
|
|
23
|
+
Changelog = "https://github.com/danielcristho/croft/blob/main/CHANGELOG.md"
|
|
24
|
+
|
|
25
|
+
[tool.uv.build-backend]
|
|
26
|
+
module-name = "croft"
|
|
27
|
+
|
|
28
|
+
[project.optional-dependencies]
|
|
29
|
+
plot = ["matplotlib>=3.9"]
|
|
30
|
+
|
|
31
|
+
[build-system]
|
|
32
|
+
requires = ["uv_build>=0.11,<0.12"]
|
|
33
|
+
build-backend = "uv_build"
|
|
34
|
+
|
|
35
|
+
[tool.ruff]
|
|
36
|
+
line-length = 120
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""Croft datasets collected per node in CROFT_DATASET: a directory of zstd segments and a live file per node, or an older <node>.jsonl."""
|
|
2
|
+
|
|
3
|
+
import contextlib
|
|
4
|
+
import hashlib
|
|
5
|
+
import io
|
|
6
|
+
import itertools
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
|
|
12
|
+
import pandas as pd
|
|
13
|
+
import zstandard
|
|
14
|
+
|
|
15
|
+
from . import kube, metrics
|
|
16
|
+
|
|
17
|
+
DEFAULT_LOCATION = "/work/data/nodes"
|
|
18
|
+
SEGMENT = re.compile(r"croft-dataset-\d{8}T\d{6}Z\.jsonl\.zst")
|
|
19
|
+
LIVE = "croft-dataset.jsonl"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def files(location=None):
|
|
23
|
+
"""Every dataset file, each node's in time order: segments by their start stamp, then the live file."""
|
|
24
|
+
root = Path(location or os.environ.get("CROFT_DATASET", DEFAULT_LOCATION))
|
|
25
|
+
if not root.is_dir():
|
|
26
|
+
return [root]
|
|
27
|
+
found = sorted(root.glob("*.jsonl"))
|
|
28
|
+
for node_dir in sorted(path for path in root.iterdir() if path.is_dir()):
|
|
29
|
+
found += sorted(path for path in node_dir.iterdir() if SEGMENT.fullmatch(path.name))
|
|
30
|
+
found += [node_dir / LIVE] if (node_dir / LIVE).exists() else []
|
|
31
|
+
return found
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def node(path):
|
|
35
|
+
"""The node a file belongs to: its directory for segments and live files, its name for <node>.jsonl."""
|
|
36
|
+
if path.name == LIVE or SEGMENT.fullmatch(path.name):
|
|
37
|
+
return path.parent.name
|
|
38
|
+
return path.name.removesuffix(".jsonl")
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def _open(path):
|
|
42
|
+
handle = path.open("rb")
|
|
43
|
+
if path.name.endswith(".zst"):
|
|
44
|
+
return io.BufferedReader(zstandard.ZstdDecompressor().stream_reader(handle, closefd=True))
|
|
45
|
+
return handle
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def fingerprint(paths):
|
|
49
|
+
digest = hashlib.sha256()
|
|
50
|
+
for path in paths:
|
|
51
|
+
stat = path.stat()
|
|
52
|
+
digest.update(f"{path.name}:{stat.st_size}:{stat.st_mtime_ns}".encode())
|
|
53
|
+
return digest.hexdigest()[:12]
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@contextlib.contextmanager
|
|
57
|
+
def open_all(paths=None):
|
|
58
|
+
handles = [_open(path) for path in (paths or files())]
|
|
59
|
+
try:
|
|
60
|
+
yield itertools.chain.from_iterable(handles)
|
|
61
|
+
finally:
|
|
62
|
+
for handle in handles:
|
|
63
|
+
handle.close()
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
UNIX_MS = re.compile(rb'"unix_ms":(\d+)')
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _ms_near(handle, offset):
|
|
70
|
+
handle.seek(offset)
|
|
71
|
+
handle.readline()
|
|
72
|
+
for _ in range(50):
|
|
73
|
+
match = UNIX_MS.search(handle.readline())
|
|
74
|
+
if match:
|
|
75
|
+
return int(match.group(1))
|
|
76
|
+
return None
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
# offset_since bisects a file, whose rows are in time order, for a byte offset at or before the first row at since_ms.
|
|
80
|
+
def offset_since(path, since_ms):
|
|
81
|
+
low, high = 0, path.stat().st_size
|
|
82
|
+
with path.open("rb") as handle:
|
|
83
|
+
while high - low > 1 << 20:
|
|
84
|
+
middle = (low + high) // 2
|
|
85
|
+
ms = _ms_near(handle, middle)
|
|
86
|
+
if ms is None or ms >= since_ms:
|
|
87
|
+
high = middle
|
|
88
|
+
else:
|
|
89
|
+
low = middle
|
|
90
|
+
return low
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def read(keep=lambda line: True, paths=None, since_ms=None, until_ms=None):
|
|
94
|
+
"""Parses the rows whose raw bytes pass keep; returns them, each with its node, and the unreadable line count.
|
|
95
|
+
|
|
96
|
+
since_ms and until_ms limit the read to a time window: a plain file skips to it by bisection, a zstd segment
|
|
97
|
+
is decompressed from its start.
|
|
98
|
+
"""
|
|
99
|
+
rows, unreadable = [], 0
|
|
100
|
+
for path in paths or files():
|
|
101
|
+
compressed = path.name.endswith(".zst")
|
|
102
|
+
with _open(path) as handle:
|
|
103
|
+
if since_ms is not None and not compressed:
|
|
104
|
+
offset = offset_since(path, since_ms)
|
|
105
|
+
handle.seek(offset)
|
|
106
|
+
if offset:
|
|
107
|
+
handle.readline()
|
|
108
|
+
for line in handle:
|
|
109
|
+
if until_ms is not None or (since_ms is not None and compressed):
|
|
110
|
+
match = UNIX_MS.search(line, 0, 40)
|
|
111
|
+
if match and until_ms is not None and int(match.group(1)) > until_ms:
|
|
112
|
+
break
|
|
113
|
+
if match and since_ms is not None and int(match.group(1)) < since_ms:
|
|
114
|
+
continue
|
|
115
|
+
if not line.strip() or not keep(line):
|
|
116
|
+
continue
|
|
117
|
+
try:
|
|
118
|
+
row = json.loads(line)
|
|
119
|
+
except ValueError:
|
|
120
|
+
unreadable += 1
|
|
121
|
+
continue
|
|
122
|
+
row["node"] = node(path)
|
|
123
|
+
rows.append(row)
|
|
124
|
+
return rows, unreadable
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
# Where each signal sits in a row, and whether a container's processes add up or share one cgroup value.
|
|
128
|
+
FIELDS = {
|
|
129
|
+
"syscall_rate": ("syscalls", None, True),
|
|
130
|
+
"runqueue_wait": ("usage", "runqueue_s", True),
|
|
131
|
+
"cpu_cores": ("cgroup_usage", "cpu_cores", False),
|
|
132
|
+
"cpu_throttled_fraction": ("cgroup_usage", "cpu_throttled_fraction", False),
|
|
133
|
+
"cpu_pressure": ("cgroup_usage", "cpu_pressure_avg10", False),
|
|
134
|
+
}
|
|
135
|
+
# Rows of one poll are written within a few milliseconds; polls are 2 s apart.
|
|
136
|
+
POLL_GAP_MS = 500
|
|
137
|
+
CONTAINER_ID = re.compile(rb'"container_id":"([0-9a-f]+)"')
|
|
138
|
+
SEGMENT_STAMP = re.compile(r"croft-dataset-(\d{8}T\d{6}Z)\.jsonl\.zst")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def signal(
|
|
142
|
+
name: str,
|
|
143
|
+
namespace: str | None = None,
|
|
144
|
+
deployment: str | None = None,
|
|
145
|
+
*,
|
|
146
|
+
since: metrics.Time = "1h",
|
|
147
|
+
until: metrics.Time | None = None,
|
|
148
|
+
step_s: float | None = None,
|
|
149
|
+
match: dict[str, str] | None = None,
|
|
150
|
+
names: kube.Names | None = None,
|
|
151
|
+
paths: list[Path] | None = None,
|
|
152
|
+
) -> pd.DataFrame:
|
|
153
|
+
"""Samples of one signal from the dataset files, in the shape of metrics.signal with node and container_id added.
|
|
154
|
+
|
|
155
|
+
A summed signal is the container's total per poll, averaged over the polls of each step; a poll without a row
|
|
156
|
+
for the container counts as 0, since idle processes are written every 15 polls. A cgroup signal is averaged over
|
|
157
|
+
the polls that hold it. names maps container_id to (namespace, pod, container), as kube.names() returns; namespace
|
|
158
|
+
and deployment need it.
|
|
159
|
+
"""
|
|
160
|
+
if name not in FIELDS:
|
|
161
|
+
raise ValueError(f"the dataset holds {', '.join(FIELDS)}; {name} is on /metrics and Prometheus only")
|
|
162
|
+
spec = metrics.SIGNALS[name]
|
|
163
|
+
unknown = set(match or {}) - set(spec.labels)
|
|
164
|
+
if unknown:
|
|
165
|
+
raise ValueError(f"{name} has no label {', '.join(sorted(unknown))}; it has {spec.labels or 'none'}")
|
|
166
|
+
if (namespace or deployment) and names is None:
|
|
167
|
+
raise ValueError("namespace and deployment need names, such as names=kube.names()")
|
|
168
|
+
start_s, end_s, step_s = metrics.window(since, until, step_s)
|
|
169
|
+
wanted = _containers(names, namespace, deployment)
|
|
170
|
+
|
|
171
|
+
rows, stamps = [], []
|
|
172
|
+
for path in between(paths or files(), start_s * 1000, end_s * 1000):
|
|
173
|
+
seen: list[int] = []
|
|
174
|
+
found, _ = read(_keeper(wanted, seen), [path], since_ms=int(start_s * 1000), until_ms=int(end_s * 1000))
|
|
175
|
+
rows += found
|
|
176
|
+
stamps += [(node(path), stamp) for stamp in seen]
|
|
177
|
+
polls = _polls(pd.DataFrame(stamps, columns=["node", "unix_ms"]), step_s)
|
|
178
|
+
frame = _binned(_samples(rows, name, match or {}), polls, spec.labels, FIELDS[name][2])
|
|
179
|
+
frame = _named(frame, names, spec.labels)
|
|
180
|
+
frame.attrs.update(signal=name, unit=spec.unit, source="dataset")
|
|
181
|
+
return frame
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def between(paths: list[Path], start_ms: float, end_ms: float) -> list[Path]:
|
|
185
|
+
"""The files that can hold rows between start_ms and end_ms: a segment spans from its stamp to the next file's."""
|
|
186
|
+
chosen = []
|
|
187
|
+
for index, path in enumerate(paths):
|
|
188
|
+
begins = _stamp_ms(path)
|
|
189
|
+
following = paths[index + 1] if index + 1 < len(paths) else None
|
|
190
|
+
ends = _stamp_ms(following) if following is not None and node(following) == node(path) else None
|
|
191
|
+
if begins is not None and begins > end_ms:
|
|
192
|
+
continue
|
|
193
|
+
if begins is not None and ends is not None and ends < start_ms:
|
|
194
|
+
continue
|
|
195
|
+
chosen.append(path)
|
|
196
|
+
return chosen
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def _stamp_ms(path: Path) -> float | None:
|
|
200
|
+
found = SEGMENT_STAMP.fullmatch(path.name)
|
|
201
|
+
return pd.Timestamp(found.group(1)).timestamp() * 1000 if found else None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
# _keeper notes every row's time in seen, so polls in which the wanted containers wrote nothing still count.
|
|
205
|
+
def _keeper(wanted: set[str] | None, seen: list[int]):
|
|
206
|
+
def keep(line: bytes) -> bool:
|
|
207
|
+
stamp = UNIX_MS.search(line, 0, 40)
|
|
208
|
+
if stamp:
|
|
209
|
+
seen.append(int(stamp.group(1)))
|
|
210
|
+
found = CONTAINER_ID.search(line)
|
|
211
|
+
return found is not None and (wanted is None or found.group(1).decode() in wanted)
|
|
212
|
+
|
|
213
|
+
return keep
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _containers(names: kube.Names | None, namespace: str | None, deployment: str | None) -> set[str] | None:
|
|
217
|
+
if names is None or not (namespace or deployment):
|
|
218
|
+
return None
|
|
219
|
+
return {
|
|
220
|
+
identity
|
|
221
|
+
for identity, (ns, pod, _) in names.items()
|
|
222
|
+
if (not namespace or ns == namespace) and (not deployment or metrics.deployment_of(pod) == deployment)
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _samples(rows: list[dict], name: str, match: dict[str, str]) -> pd.DataFrame:
|
|
227
|
+
section, key, _ = FIELDS[name]
|
|
228
|
+
samples = []
|
|
229
|
+
for row in rows:
|
|
230
|
+
base = (row["unix_ms"], row["node"], row["process"]["container_id"])
|
|
231
|
+
if key is None:
|
|
232
|
+
samples += [(*base, s["name"], s["rate_per_sec"]) for s in row.get(section) or [] if _matches(s, match)]
|
|
233
|
+
else:
|
|
234
|
+
value = (row.get(section) or {}).get(key)
|
|
235
|
+
if value is not None:
|
|
236
|
+
samples.append((*base, value))
|
|
237
|
+
labels = ["syscall"] if key is None else []
|
|
238
|
+
return pd.DataFrame(samples, columns=["unix_ms", "node", "container_id", *labels, "value"])
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
def _matches(syscall: dict, match: dict[str, str]) -> bool:
|
|
242
|
+
return "syscall" not in match or syscall["name"] == match["syscall"]
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _polls(stamps: pd.DataFrame, step_s: float) -> pd.DataFrame:
|
|
246
|
+
"""One row per poll and node: its first row's time and the step it falls in."""
|
|
247
|
+
stamps = stamps.sort_values(["node", "unix_ms"], ignore_index=True)
|
|
248
|
+
stamps["poll"] = stamps.groupby("node")["unix_ms"].diff().fillna(POLL_GAP_MS + 1).gt(POLL_GAP_MS).cumsum()
|
|
249
|
+
polls = stamps.groupby(["node", "poll"])["unix_ms"].min().rename("start_ms").reset_index()
|
|
250
|
+
polls["bin"] = (polls["start_ms"] / 1000 // step_s) * step_s
|
|
251
|
+
return polls
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _binned(samples: pd.DataFrame, polls: pd.DataFrame, labels: tuple[str, ...], summed: bool) -> pd.DataFrame:
|
|
255
|
+
keys = ["node", "container_id", *labels]
|
|
256
|
+
if samples.empty:
|
|
257
|
+
return pd.DataFrame(columns=["time", *keys, "value"]).astype({"time": "datetime64[ns, UTC]", "value": float})
|
|
258
|
+
samples = pd.merge_asof(
|
|
259
|
+
samples.sort_values("unix_ms"), polls.sort_values("start_ms"), left_on="unix_ms", right_on="start_ms", by="node"
|
|
260
|
+
)
|
|
261
|
+
per_poll = samples.groupby([*keys, "poll", "bin"])["value"].agg("sum" if summed else "max").reset_index()
|
|
262
|
+
if summed:
|
|
263
|
+
counts = polls.groupby(["node", "bin"]).size().rename("polls").reset_index()
|
|
264
|
+
frame = per_poll.groupby([*keys, "bin"])["value"].sum().reset_index().merge(counts, on=["node", "bin"])
|
|
265
|
+
frame["value"] = frame["value"] / frame["polls"]
|
|
266
|
+
else:
|
|
267
|
+
frame = per_poll.groupby([*keys, "bin"])["value"].mean().reset_index()
|
|
268
|
+
frame["time"] = pd.to_datetime(frame["bin"], unit="s", utc=True)
|
|
269
|
+
return frame[["time", *keys, "value"]].sort_values([*keys, "time"], ignore_index=True)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _named(frame: pd.DataFrame, names: kube.Names | None, labels: tuple[str, ...]) -> pd.DataFrame:
|
|
273
|
+
if names is None:
|
|
274
|
+
return frame
|
|
275
|
+
found = frame["container_id"].map(names)
|
|
276
|
+
frame["namespace"] = found.map(lambda n: n[0] if isinstance(n, tuple) else None)
|
|
277
|
+
frame["pod"] = found.map(lambda n: n[1] if isinstance(n, tuple) else None)
|
|
278
|
+
frame["container"] = found.map(lambda n: n[2] if isinstance(n, tuple) else None)
|
|
279
|
+
frame["deployment"] = frame["pod"].map(metrics.deployment_of)
|
|
280
|
+
columns = ["time", "node", "namespace", "deployment", "pod", "container", "container_id", *labels, "value"]
|
|
281
|
+
return frame[columns]
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def syscall_rate(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
285
|
+
return signal("syscall_rate", namespace, deployment, **options)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def runqueue_wait(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
289
|
+
return signal("runqueue_wait", namespace, deployment, **options)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def cpu_cores(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
293
|
+
return signal("cpu_cores", namespace, deployment, **options)
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def cpu_throttled_fraction(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
297
|
+
return signal("cpu_throttled_fraction", namespace, deployment, **options)
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def cpu_pressure(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
301
|
+
return signal("cpu_pressure", namespace, deployment, **options)
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Container names from Kubernetes, read through kubectl, to label dataset rows that carry a container_id only."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import subprocess
|
|
5
|
+
|
|
6
|
+
Names = dict[str, tuple[str, str, str]]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def names(kubeconfig: str | None = None, context: str | None = None) -> Names:
|
|
10
|
+
"""container_id to (namespace, pod, container) for every pod the API server holds, with each container's
|
|
11
|
+
previous instance; containers of deleted pods are absent."""
|
|
12
|
+
command = ["kubectl", "get", "pods", "--all-namespaces", "-o", "json"]
|
|
13
|
+
command += ["--kubeconfig", kubeconfig] if kubeconfig else []
|
|
14
|
+
command += ["--context", context] if context else []
|
|
15
|
+
result = subprocess.run(command, capture_output=True, text=True, check=False)
|
|
16
|
+
if result.returncode:
|
|
17
|
+
raise RuntimeError(f"kubectl: {result.stderr.strip()}")
|
|
18
|
+
return names_of(json.loads(result.stdout))
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def names_of(pods: dict) -> Names:
|
|
22
|
+
found: Names = {}
|
|
23
|
+
for pod in pods.get("items", []):
|
|
24
|
+
namespace, name = pod["metadata"]["namespace"], pod["metadata"]["name"]
|
|
25
|
+
status = pod.get("status", {})
|
|
26
|
+
for container in status.get("initContainerStatuses", []) + status.get("containerStatuses", []):
|
|
27
|
+
previous = container.get("lastState", {}).get("terminated", {})
|
|
28
|
+
for identity in (container.get("containerID"), previous.get("containerID")):
|
|
29
|
+
if identity:
|
|
30
|
+
found[identity.split("://")[-1]] = (namespace, name, container["name"])
|
|
31
|
+
return found
|
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""Collector signals from Prometheus as pandas frames, one row per sample, joined to the pod and container that produced it."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import re
|
|
5
|
+
import time
|
|
6
|
+
import urllib.request
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime
|
|
9
|
+
|
|
10
|
+
import pandas as pd
|
|
11
|
+
|
|
12
|
+
from .prometheus import Prometheus
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
@dataclass(frozen=True)
|
|
16
|
+
class Signal:
|
|
17
|
+
metric: str
|
|
18
|
+
labels: tuple[str, ...]
|
|
19
|
+
counter: bool
|
|
20
|
+
unit: str
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
SIGNALS = {
|
|
24
|
+
"syscall_rate": Signal("croft_container_syscall_rate", ("syscall",), False, "calls/s"),
|
|
25
|
+
"syscall_bytes": Signal("croft_container_syscall_bytes_total", ("syscall",), True, "bytes/s"),
|
|
26
|
+
"socket_io": Signal("croft_container_socket_io_total", ("direction",), True, "calls/s"),
|
|
27
|
+
"connects": Signal("croft_container_connects_total", ("address", "port"), True, "connects/s"),
|
|
28
|
+
"cpu_cores": Signal("croft_container_cpu_cores", (), False, "cores"),
|
|
29
|
+
"cpu_throttled_fraction": Signal("croft_container_cpu_throttled_fraction", (), False, "fraction"),
|
|
30
|
+
"cpu_pressure": Signal("croft_container_cpu_pressure_avg10", (), False, "%"),
|
|
31
|
+
"runqueue_wait": Signal("croft_container_runqueue_wait", (), False, "s/s"),
|
|
32
|
+
"inflight_requests": Signal("croft_container_inflight_requests", (), False, "requests"),
|
|
33
|
+
}
|
|
34
|
+
# The collector is scraped every 5 s, so 15 s holds three samples for rate().
|
|
35
|
+
RATE_WINDOW_S = 15
|
|
36
|
+
MIN_STEP_S = 5
|
|
37
|
+
MAX_POINTS = 1000
|
|
38
|
+
DURATION = re.compile(r"(\d+(?:\.\d+)?)([smhdw])")
|
|
39
|
+
SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
|
|
40
|
+
POD = re.compile(r"(.+)-[a-z0-9]{5,10}-[a-z0-9]{5}")
|
|
41
|
+
KUBE_LABELS = ("namespace", "pod", "container")
|
|
42
|
+
LINE = re.compile(r"([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(.*)\})? (\S+)")
|
|
43
|
+
LABEL = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="((?:[^"\\]|\\.)*)"')
|
|
44
|
+
ESCAPES = {"\\": "\\", '"': '"', "n": "\n"}
|
|
45
|
+
ESCAPE = re.compile(r"\\(.)")
|
|
46
|
+
|
|
47
|
+
Time = str | float | int | datetime | pd.Timestamp
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def catalog() -> pd.DataFrame:
|
|
51
|
+
"""Every signal with its Prometheus metric, extra labels, kind and unit."""
|
|
52
|
+
return pd.DataFrame(
|
|
53
|
+
[
|
|
54
|
+
{"signal": name, "metric": s.metric, "labels": ",".join(s.labels), "counter": s.counter, "unit": s.unit}
|
|
55
|
+
for name, s in SIGNALS.items()
|
|
56
|
+
]
|
|
57
|
+
).set_index("signal")
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def signal(
|
|
61
|
+
name: str,
|
|
62
|
+
namespace: str | None = None,
|
|
63
|
+
deployment: str | None = None,
|
|
64
|
+
*,
|
|
65
|
+
since: Time = "1h",
|
|
66
|
+
until: Time | None = None,
|
|
67
|
+
step_s: float | None = None,
|
|
68
|
+
match: dict[str, str] | None = None,
|
|
69
|
+
kube: bool = True,
|
|
70
|
+
prometheus: Prometheus | None = None,
|
|
71
|
+
) -> pd.DataFrame:
|
|
72
|
+
"""Samples of one signal between since and until; counters come as per-second rates.
|
|
73
|
+
|
|
74
|
+
namespace and deployment narrow the pods; match narrows the signal's own labels, such as {"syscall": "sendto"}.
|
|
75
|
+
kube=False skips the join to kube-state-metrics and keys rows by container_id, for containers outside Kubernetes.
|
|
76
|
+
"""
|
|
77
|
+
spec = SIGNALS[name]
|
|
78
|
+
start_s, end_s, step_s = window(since, until, step_s)
|
|
79
|
+
query = promql(spec, namespace, deployment, match, kube, step_s)
|
|
80
|
+
streams = (prometheus or Prometheus()).query_range(query, start_s, end_s, step_s)
|
|
81
|
+
frame = frame_of(streams, spec, kube)
|
|
82
|
+
frame.attrs.update(signal=name, unit=spec.unit, query=query)
|
|
83
|
+
return frame
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
# promql averages a gauge over each step and widens a counter's rate window to it, so a long range does not
|
|
87
|
+
# sample one scrape per step and alias periodic spikes.
|
|
88
|
+
def promql(
|
|
89
|
+
spec: Signal,
|
|
90
|
+
namespace: str | None,
|
|
91
|
+
deployment: str | None,
|
|
92
|
+
match: dict[str, str] | None,
|
|
93
|
+
kube: bool,
|
|
94
|
+
step_s: float = MIN_STEP_S,
|
|
95
|
+
) -> str:
|
|
96
|
+
unknown = set(match or {}) - set(spec.labels)
|
|
97
|
+
if unknown:
|
|
98
|
+
raise ValueError(f"{spec.metric} has no label {', '.join(sorted(unknown))}; it has {spec.labels or 'none'}")
|
|
99
|
+
expr = f"{spec.metric}{_selector(match or {})}"
|
|
100
|
+
span = f"{max(RATE_WINDOW_S, step_s) if spec.counter else step_s:g}s"
|
|
101
|
+
if spec.counter:
|
|
102
|
+
expr = f"rate({expr}[{span}])"
|
|
103
|
+
elif step_s > MIN_STEP_S:
|
|
104
|
+
expr = f"avg_over_time({expr}[{span}])"
|
|
105
|
+
if not kube:
|
|
106
|
+
return expr
|
|
107
|
+
pods = {"namespace": namespace} if namespace else {}
|
|
108
|
+
if deployment:
|
|
109
|
+
pods["pod=~"] = f"{re.escape(deployment)}-[a-z0-9]+-[a-z0-9]+"
|
|
110
|
+
info = f'label_replace(kube_pod_container_info{_selector(pods)}, "container_id", "$1", "container_id", "[a-z-]+://(.+)")'
|
|
111
|
+
labels = ", ".join(KUBE_LABELS)
|
|
112
|
+
return f"{expr} * on (container_id) group_left ({labels}) max by (container_id, {labels}) ({info})"
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def frame_of(streams: list[dict], spec: Signal, kube: bool) -> pd.DataFrame:
|
|
116
|
+
keys = (*KUBE_LABELS, *spec.labels) if kube else ("container_id", *spec.labels)
|
|
117
|
+
rows = []
|
|
118
|
+
for stream in streams:
|
|
119
|
+
labels = {key: stream["metric"].get(key) for key in keys}
|
|
120
|
+
if kube:
|
|
121
|
+
labels["deployment"] = deployment_of(labels["pod"])
|
|
122
|
+
rows += [{"time": float(t), **labels, "value": float(v)} for t, v in stream["values"] if v != "NaN"]
|
|
123
|
+
columns = ["time", *(("namespace", "deployment", "pod", "container") if kube else ("container_id",)), *spec.labels]
|
|
124
|
+
frame = pd.DataFrame(rows, columns=[*columns, "value"])
|
|
125
|
+
frame["time"] = pd.to_datetime(frame["time"], unit="s", utc=True)
|
|
126
|
+
return frame.sort_values([*columns[1:], "time"], ignore_index=True)
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def wide(frame: pd.DataFrame, by: str | list[str] = "pod") -> pd.DataFrame:
|
|
130
|
+
"""One column per value of by, summed over the other labels, ready for .plot()."""
|
|
131
|
+
table = frame.pivot_table(index="time", columns=by, values="value", aggfunc="sum")
|
|
132
|
+
table.attrs = dict(frame.attrs)
|
|
133
|
+
return table
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _unescape(value: str) -> str:
|
|
137
|
+
return ESCAPE.sub(lambda m: ESCAPES.get(m.group(1), m.group(0)), value)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def live(address: str | None = None, timeout_s: float = 10) -> pd.DataFrame:
|
|
141
|
+
"""The newest poll of one collector, read from its /metrics: one row per series, counters as running totals.
|
|
142
|
+
|
|
143
|
+
address is host:port or a URL, defaulting to CROFT_METRICS_ADDR. Rows are keyed by container_id; the collector
|
|
144
|
+
holds no history and no pod names, so time ranges and the pod join need signal() and Prometheus.
|
|
145
|
+
"""
|
|
146
|
+
address = address or os.environ.get("CROFT_METRICS_ADDR") or "127.0.0.1:9464"
|
|
147
|
+
url = address if "://" in address else f"http://{address}"
|
|
148
|
+
with urllib.request.urlopen(f"{url.rstrip('/')}/metrics", timeout=timeout_s) as response:
|
|
149
|
+
text = response.read().decode()
|
|
150
|
+
names = {spec.metric: name for name, spec in SIGNALS.items()}
|
|
151
|
+
rows = []
|
|
152
|
+
for line in text.splitlines():
|
|
153
|
+
sample = _sample(line)
|
|
154
|
+
if sample and sample[0] in names:
|
|
155
|
+
metric, labels, value = sample
|
|
156
|
+
rows.append({"signal": names[metric], **labels, "value": value})
|
|
157
|
+
frame = pd.DataFrame(rows, columns=None if rows else ["signal", "container_id", "value"])
|
|
158
|
+
poll = re.search(r"^croft_poll_unix_ms (\d+)", text, re.MULTILINE)
|
|
159
|
+
frame.attrs.update(poll_time=pd.Timestamp(int(poll.group(1)), unit="ms", tz="UTC") if poll else None)
|
|
160
|
+
return frame
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _sample(line: str) -> tuple[str, dict[str, str], float] | None:
|
|
164
|
+
found = LINE.fullmatch(line.strip())
|
|
165
|
+
if not found or line.startswith("#"):
|
|
166
|
+
return None
|
|
167
|
+
labels = {key: _unescape(value) for key, value in LABEL.findall(found.group(2) or "")}
|
|
168
|
+
# labels = {
|
|
169
|
+
# key: ESCAPE.sub(lambda m: ESCAPES.get(m.group(1), m.group(0)), value) for key, value in LABEL.findall(found.group(2) or "")
|
|
170
|
+
# }
|
|
171
|
+
return found.group(1), labels, float(found.group(3))
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def window(since: Time, until: Time | None, step_s: float | None) -> tuple[float, float, float]:
|
|
175
|
+
"""Start, end and step in unix seconds; the step keeps a range under MAX_POINTS samples."""
|
|
176
|
+
now = time.time()
|
|
177
|
+
end_s = _seconds(until, now) if until is not None else now
|
|
178
|
+
start_s = _seconds(since, end_s)
|
|
179
|
+
if start_s >= end_s:
|
|
180
|
+
raise ValueError(f"since ({since}) is not before until ({until or 'now'})")
|
|
181
|
+
return start_s, end_s, step_s or max(MIN_STEP_S, (end_s - start_s) / MAX_POINTS)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def deployment_of(pod: str | None) -> str | None:
|
|
185
|
+
found = POD.fullmatch(pod or "")
|
|
186
|
+
return found.group(1) if found else None
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def query(
|
|
190
|
+
text: str,
|
|
191
|
+
*,
|
|
192
|
+
since: Time = "1h",
|
|
193
|
+
until: Time | None = None,
|
|
194
|
+
step_s: float | None = None,
|
|
195
|
+
prometheus: Prometheus | None = None,
|
|
196
|
+
) -> pd.DataFrame:
|
|
197
|
+
"""Any PromQL range query as a long frame with one column per label."""
|
|
198
|
+
rows = [
|
|
199
|
+
{"time": float(t), **stream["metric"], "value": float(v)}
|
|
200
|
+
for stream in (prometheus or Prometheus()).query_range(text, *window(since, until, step_s))
|
|
201
|
+
for t, v in stream["values"]
|
|
202
|
+
if v != "NaN"
|
|
203
|
+
]
|
|
204
|
+
frame = pd.DataFrame(rows, columns=None if rows else ["time", "value"])
|
|
205
|
+
frame["time"] = pd.to_datetime(frame["time"], unit="s", utc=True)
|
|
206
|
+
frame.attrs.update(query=text)
|
|
207
|
+
return frame
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def syscall_rate(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
211
|
+
return signal("syscall_rate", namespace, deployment, **options)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def syscall_bytes(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
215
|
+
return signal("syscall_bytes", namespace, deployment, **options)
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def socket_io(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
219
|
+
return signal("socket_io", namespace, deployment, **options)
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def connects(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
223
|
+
return signal("connects", namespace, deployment, **options)
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def cpu_cores(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
227
|
+
return signal("cpu_cores", namespace, deployment, **options)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def cpu_throttled_fraction(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
231
|
+
return signal("cpu_throttled_fraction", namespace, deployment, **options)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def cpu_pressure(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
235
|
+
return signal("cpu_pressure", namespace, deployment, **options)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def runqueue_wait(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
239
|
+
return signal("runqueue_wait", namespace, deployment, **options)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def inflight_requests(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
|
|
243
|
+
return signal("inflight_requests", namespace, deployment, **options)
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def _selector(labels: dict[str, str]) -> str:
|
|
247
|
+
if not labels:
|
|
248
|
+
return ""
|
|
249
|
+
pairs = []
|
|
250
|
+
for key, value in labels.items():
|
|
251
|
+
name, op = (key[:-2], "=~") if key.endswith("=~") else (key, "=")
|
|
252
|
+
pairs.append(f"{name}{op}{_quote(value)}")
|
|
253
|
+
return "{" + ",".join(pairs) + "}"
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def _quote(value: str) -> str:
|
|
257
|
+
return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# _seconds reads a unix time, a datetime, or a duration such as "90m" before relative_to.
|
|
261
|
+
def _seconds(at: Time, relative_to: float | None = None) -> float:
|
|
262
|
+
if isinstance(at, (int, float)):
|
|
263
|
+
return float(at)
|
|
264
|
+
if isinstance(at, datetime):
|
|
265
|
+
return pd.Timestamp(at).timestamp()
|
|
266
|
+
if relative_to is not None:
|
|
267
|
+
parts = DURATION.findall(at)
|
|
268
|
+
if parts and "".join(n + u for n, u in parts) == at:
|
|
269
|
+
return relative_to - sum(float(n) * SECONDS[u] for n, u in parts)
|
|
270
|
+
return pd.Timestamp(at).timestamp()
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Line charts of signal frames, one line per pod or other label, with the signal's unit on the y axis."""
|
|
2
|
+
|
|
3
|
+
import pandas as pd
|
|
4
|
+
|
|
5
|
+
from .metrics import wide
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def lines(frame: pd.DataFrame, by: str | list[str] = "pod", ax=None, title: str | None = None):
|
|
9
|
+
"""Draws frame as one line per value of by and returns the axes."""
|
|
10
|
+
table = wide(frame, by)
|
|
11
|
+
ax = table.plot(ax=ax, linewidth=1.5, legend=table.shape[1] > 1)
|
|
12
|
+
ax.set_xlabel("time (UTC)")
|
|
13
|
+
ax.set_ylabel(frame.attrs.get("unit", "value"))
|
|
14
|
+
ax.set_title(title or frame.attrs.get("signal", ""))
|
|
15
|
+
ax.grid(alpha=0.3)
|
|
16
|
+
return ax
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Prometheus HTTP API client: instant and range queries, with the server's error message on failure."""
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import os
|
|
5
|
+
import urllib.error
|
|
6
|
+
import urllib.parse
|
|
7
|
+
import urllib.request
|
|
8
|
+
|
|
9
|
+
DEFAULT_URL = "http://prometheus.127.0.0.1.sslip.io:30080"
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class PrometheusError(RuntimeError):
|
|
13
|
+
pass
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class Prometheus:
|
|
17
|
+
def __init__(self, url: str | None = None, timeout_s: float = 60):
|
|
18
|
+
self.url = (url or os.environ.get("CROFT_PROMETHEUS") or os.environ.get("PROM_HOST") or DEFAULT_URL).rstrip("/")
|
|
19
|
+
self.timeout_s = timeout_s
|
|
20
|
+
|
|
21
|
+
def query(self, query: str, at_s: float | None = None) -> list[dict]:
|
|
22
|
+
params = {"query": query} if at_s is None else {"query": query, "time": at_s}
|
|
23
|
+
return self._call("query", params)
|
|
24
|
+
|
|
25
|
+
def query_range(self, query: str, start_s: float, end_s: float, step_s: float) -> list[dict]:
|
|
26
|
+
return self._call("query_range", {"query": query, "start": start_s, "end": end_s, "step": step_s})
|
|
27
|
+
|
|
28
|
+
def _call(self, path: str, params: dict) -> list[dict]:
|
|
29
|
+
body = urllib.parse.urlencode(params).encode()
|
|
30
|
+
try:
|
|
31
|
+
with urllib.request.urlopen(f"{self.url}/api/v1/{path}", body, timeout=self.timeout_s) as response:
|
|
32
|
+
answer = json.load(response)
|
|
33
|
+
except urllib.error.HTTPError as error:
|
|
34
|
+
answer = _json_or_none(error.read())
|
|
35
|
+
if answer is None:
|
|
36
|
+
raise PrometheusError(f"{self.url}: HTTP {error.code}") from error
|
|
37
|
+
except OSError as error:
|
|
38
|
+
raise PrometheusError(f"{self.url}: {error}") from error
|
|
39
|
+
if answer.get("status") != "success":
|
|
40
|
+
raise PrometheusError(f"{answer.get('errorType', 'error')}: {answer.get('error', answer)}")
|
|
41
|
+
return answer["data"]["result"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _json_or_none(raw: bytes) -> dict | None:
|
|
45
|
+
try:
|
|
46
|
+
return json.loads(raw)
|
|
47
|
+
except ValueError:
|
|
48
|
+
return None
|