croft-signals 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,100 @@
1
+ Metadata-Version: 2.4
2
+ Name: croft-signals
3
+ Version: 0.4.0
4
+ Summary: croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames
5
+ Keywords: ebpf,kubernetes,autoscaling,prometheus,pandas
6
+ Author: Daniel Pepuho
7
+ License-Expression: Apache-2.0
8
+ Classifier: Development Status :: 3 - Alpha
9
+ Classifier: Intended Audience :: Science/Research
10
+ Classifier: Programming Language :: Python :: 3
11
+ Classifier: Programming Language :: Python :: 3.12
12
+ Classifier: Programming Language :: Python :: 3.13
13
+ Classifier: Topic :: System :: Monitoring
14
+ Requires-Dist: pandas>=2.2
15
+ Requires-Dist: zstandard>=0.25
16
+ Requires-Dist: matplotlib>=3.9 ; extra == 'plot'
17
+ Requires-Python: >=3.12
18
+ Project-URL: Repository, https://github.com/danielcristho/croft
19
+ Project-URL: Documentation, https://croft-docs.pages.dev
20
+ Project-URL: Changelog, https://github.com/danielcristho/croft/blob/main/CHANGELOG.md
21
+ Provides-Extra: plot
22
+ Description-Content-Type: text/markdown
23
+
24
+ # croft for Python
25
+
26
+ The collector's signals as pandas frames, for notebooks and analysis.
27
+
28
+ ```sh
29
+ pip install "croft-signals[plot]"
30
+ uv add "croft-signals[plot]"
31
+ uv add --editable ../python # from a project inside this repository
32
+ ```
33
+
34
+ ## 1. Sources
35
+
36
+ | Source | Function | History | Pods |
37
+ |---|---|---|---|
38
+ | Prometheus | `metrics.signal()` and one function per signal | as long as Prometheus keeps it | joined through kube-state-metrics |
39
+ | collector `/metrics` | `metrics.live()` | the newest poll | `container_id` |
40
+ | dataset files | `dataset.signal()` and one function per signal; `dataset.read()` for raw rows | as long as the node keeps it | `container_id`, or names from `kube.names()` |
41
+
42
+ `CROFT_PROMETHEUS` sets the Prometheus URL (`PROM_HOST` is read as a fallback),
43
+ `CROFT_METRICS_ADDR` the collector address and `CROFT_DATASET` the dataset directory.
44
+
45
+ ## 2. Signals
46
+
47
+ | Signal | Metric | Unit |
48
+ |---|---|---|
49
+ | `syscall_rate` | `croft_container_syscall_rate` | calls/s, per `syscall` |
50
+ | `syscall_bytes` | `croft_container_syscall_bytes_total` | bytes/s, per `syscall` |
51
+ | `socket_io` | `croft_container_socket_io_total` | calls/s, per `direction` |
52
+ | `connects` | `croft_container_connects_total` | connects/s, per `address` and `port` |
53
+ | `cpu_cores` | `croft_container_cpu_cores` | cores |
54
+ | `cpu_throttled_fraction` | `croft_container_cpu_throttled_fraction` | fraction |
55
+ | `cpu_pressure` | `croft_container_cpu_pressure_avg10` | % |
56
+ | `runqueue_wait` | `croft_container_runqueue_wait` | s/s |
57
+ | `inflight_requests` | `croft_container_inflight_requests` | requests |
58
+
59
+ Counters come as rates over 15 s or the step, whichever is longer; gauges are averaged over each step longer than
60
+ 5 s. `metrics.catalog()` returns this table. The dataset holds `syscall_rate`, `runqueue_wait` and the three CPU
61
+ signals; the other three are on `/metrics` and Prometheus.
62
+
63
+ ## 3. Usage
64
+
65
+ ```python
66
+ from croft import metrics, plot
67
+
68
+ frame = metrics.syscall_rate("boutique", "frontend", since="1h", match={"syscall": "getpeername"})
69
+ plot.lines(frame, by="pod")
70
+
71
+ metrics.runqueue_wait("hotel-res", since="2026-11-18 09:00", until="2026-11-18 10:00")
72
+ metrics.wide(frame, by="deployment") # one column per deployment
73
+ metrics.query("sum(rate(container_cpu_usage_seconds_total[1m]))", since="30m")
74
+ metrics.live("127.0.0.1:9464") # newest poll of one collector
75
+ ```
76
+
77
+ Without Prometheus, the same frames come from the dataset files, with `node` and `container_id` added:
78
+
79
+ ```python
80
+ from croft import dataset, kube
81
+
82
+ names = kube.names() # container_id to namespace, pod and container, through kubectl
83
+ frame = dataset.syscall_rate("boutique", "frontend", since="1h", names=names, match={"syscall": "getpeername"})
84
+ ```
85
+
86
+ A summed signal (`syscall_rate`, `runqueue_wait`) is the container's total per poll, averaged over the polls of
87
+ each step; a poll without a row for the container counts as 0, since idle processes are written every 15 polls.
88
+ A CPU signal is averaged over the polls that hold it. `kube.names()` knows the pods the API server holds, so
89
+ containers of deleted pods keep their `container_id` only.
90
+
91
+ Each row holds `time` (UTC), `namespace`, `deployment`, `pod`, `container`, the
92
+ signal's own labels and `value`. `deployment` comes from the pod name and is empty
93
+ for pods outside a Deployment. `since` and `until` take a duration before now
94
+ (`"90m"`), a date or a unix time; the step keeps a range under 1000 samples.
95
+
96
+ ## 4. Tests
97
+
98
+ ```sh
99
+ uv run python -m unittest discover -s tests
100
+ ```
@@ -0,0 +1,77 @@
1
+ # croft for Python
2
+
3
+ The collector's signals as pandas frames, for notebooks and analysis.
4
+
5
+ ```sh
6
+ pip install "croft-signals[plot]"
7
+ uv add "croft-signals[plot]"
8
+ uv add --editable ../python # from a project inside this repository
9
+ ```
10
+
11
+ ## 1. Sources
12
+
13
+ | Source | Function | History | Pods |
14
+ |---|---|---|---|
15
+ | Prometheus | `metrics.signal()` and one function per signal | as long as Prometheus keeps it | joined through kube-state-metrics |
16
+ | collector `/metrics` | `metrics.live()` | the newest poll | `container_id` |
17
+ | dataset files | `dataset.signal()` and one function per signal; `dataset.read()` for raw rows | as long as the node keeps it | `container_id`, or names from `kube.names()` |
18
+
19
+ `CROFT_PROMETHEUS` sets the Prometheus URL (`PROM_HOST` is read as a fallback),
20
+ `CROFT_METRICS_ADDR` the collector address and `CROFT_DATASET` the dataset directory.
21
+
22
+ ## 2. Signals
23
+
24
+ | Signal | Metric | Unit |
25
+ |---|---|---|
26
+ | `syscall_rate` | `croft_container_syscall_rate` | calls/s, per `syscall` |
27
+ | `syscall_bytes` | `croft_container_syscall_bytes_total` | bytes/s, per `syscall` |
28
+ | `socket_io` | `croft_container_socket_io_total` | calls/s, per `direction` |
29
+ | `connects` | `croft_container_connects_total` | connects/s, per `address` and `port` |
30
+ | `cpu_cores` | `croft_container_cpu_cores` | cores |
31
+ | `cpu_throttled_fraction` | `croft_container_cpu_throttled_fraction` | fraction |
32
+ | `cpu_pressure` | `croft_container_cpu_pressure_avg10` | % |
33
+ | `runqueue_wait` | `croft_container_runqueue_wait` | s/s |
34
+ | `inflight_requests` | `croft_container_inflight_requests` | requests |
35
+
36
+ Counters come as rates over 15 s or the step, whichever is longer; gauges are averaged over each step longer than
37
+ 5 s. `metrics.catalog()` returns this table. The dataset holds `syscall_rate`, `runqueue_wait` and the three CPU
38
+ signals; the other three are on `/metrics` and Prometheus.
39
+
40
+ ## 3. Usage
41
+
42
+ ```python
43
+ from croft import metrics, plot
44
+
45
+ frame = metrics.syscall_rate("boutique", "frontend", since="1h", match={"syscall": "getpeername"})
46
+ plot.lines(frame, by="pod")
47
+
48
+ metrics.runqueue_wait("hotel-res", since="2026-11-18 09:00", until="2026-11-18 10:00")
49
+ metrics.wide(frame, by="deployment") # one column per deployment
50
+ metrics.query("sum(rate(container_cpu_usage_seconds_total[1m]))", since="30m")
51
+ metrics.live("127.0.0.1:9464") # newest poll of one collector
52
+ ```
53
+
54
+ Without Prometheus, the same frames come from the dataset files, with `node` and `container_id` added:
55
+
56
+ ```python
57
+ from croft import dataset, kube
58
+
59
+ names = kube.names() # container_id to namespace, pod and container, through kubectl
60
+ frame = dataset.syscall_rate("boutique", "frontend", since="1h", names=names, match={"syscall": "getpeername"})
61
+ ```
62
+
63
+ A summed signal (`syscall_rate`, `runqueue_wait`) is the container's total per poll, averaged over the polls of
64
+ each step; a poll without a row for the container counts as 0, since idle processes are written every 15 polls.
65
+ A CPU signal is averaged over the polls that hold it. `kube.names()` knows the pods the API server holds, so
66
+ containers of deleted pods keep their `container_id` only.
67
+
68
+ Each row holds `time` (UTC), `namespace`, `deployment`, `pod`, `container`, the
69
+ signal's own labels and `value`. `deployment` comes from the pod name and is empty
70
+ for pods outside a Deployment. `since` and `until` take a duration before now
71
+ (`"90m"`), a date or a unix time; the step keeps a range under 1000 samples.
72
+
73
+ ## 4. Tests
74
+
75
+ ```sh
76
+ uv run python -m unittest discover -s tests
77
+ ```
@@ -0,0 +1,47 @@
1
+ [project]
2
+ name = "croft-signals"
3
+ version = "0.4.0"
4
+ description = "croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames"
5
+ readme = "README.md"
6
+ license = "Apache-2.0"
7
+ requires-python = ">=3.12"
8
+ keywords = [
9
+ "ebpf",
10
+ "kubernetes",
11
+ "autoscaling",
12
+ "prometheus",
13
+ "pandas",
14
+ ]
15
+ classifiers = [
16
+ "Development Status :: 3 - Alpha",
17
+ "Intended Audience :: Science/Research",
18
+ "Programming Language :: Python :: 3",
19
+ "Programming Language :: Python :: 3.12",
20
+ "Programming Language :: Python :: 3.13",
21
+ "Topic :: System :: Monitoring",
22
+ ]
23
+ dependencies = [
24
+ "pandas>=2.2",
25
+ "zstandard>=0.25",
26
+ ]
27
+
28
+ [[project.authors]]
29
+ name = "Daniel Pepuho"
30
+
31
+ [project.urls]
32
+ Repository = "https://github.com/danielcristho/croft"
33
+ Documentation = "https://croft-docs.pages.dev"
34
+ Changelog = "https://github.com/danielcristho/croft/blob/main/CHANGELOG.md"
35
+
36
+ [project.optional-dependencies]
37
+ plot = ["matplotlib>=3.9"]
38
+
39
+ [tool.uv.build-backend]
40
+ module-name = "croft"
41
+
42
+ [tool.ruff]
43
+ line-length = 120
44
+
45
+ [build-system]
46
+ requires = ["uv_build>=0.11,<0.12"]
47
+ build-backend = "uv_build"
@@ -0,0 +1,36 @@
1
+ [project]
2
+ name = "croft-signals"
3
+ version = "0.4.0"
4
+ description = "croft signals for notebooks: Prometheus metrics and collector datasets as pandas frames"
5
+ readme = "README.md"
6
+ license = "Apache-2.0"
7
+ requires-python = ">=3.12"
8
+ authors = [{ name = "Daniel Pepuho" }]
9
+ keywords = ["ebpf", "kubernetes", "autoscaling", "prometheus", "pandas"]
10
+ classifiers = [
11
+ "Development Status :: 3 - Alpha",
12
+ "Intended Audience :: Science/Research",
13
+ "Programming Language :: Python :: 3",
14
+ "Programming Language :: Python :: 3.12",
15
+ "Programming Language :: Python :: 3.13",
16
+ "Topic :: System :: Monitoring",
17
+ ]
18
+ dependencies = ["pandas>=2.2", "zstandard>=0.25"]
19
+
20
+ [project.urls]
21
+ Repository = "https://github.com/danielcristho/croft"
22
+ Documentation = "https://croft-docs.pages.dev"
23
+ Changelog = "https://github.com/danielcristho/croft/blob/main/CHANGELOG.md"
24
+
25
+ [tool.uv.build-backend]
26
+ module-name = "croft"
27
+
28
+ [project.optional-dependencies]
29
+ plot = ["matplotlib>=3.9"]
30
+
31
+ [build-system]
32
+ requires = ["uv_build>=0.11,<0.12"]
33
+ build-backend = "uv_build"
34
+
35
+ [tool.ruff]
36
+ line-length = 120
@@ -0,0 +1,5 @@
1
+ """croft signals for notebooks: metrics from Prometheus, datasets from the collector's files, and charts of both."""
2
+
3
+ from . import dataset, kube, metrics
4
+
5
+ __all__ = ["dataset", "kube", "metrics"]
@@ -0,0 +1,301 @@
1
+ """Croft datasets collected per node in CROFT_DATASET: a directory of zstd segments and a live file per node, or an older <node>.jsonl."""
2
+
3
+ import contextlib
4
+ import hashlib
5
+ import io
6
+ import itertools
7
+ import json
8
+ import os
9
+ import re
10
+ from pathlib import Path
11
+
12
+ import pandas as pd
13
+ import zstandard
14
+
15
+ from . import kube, metrics
16
+
17
+ DEFAULT_LOCATION = "/work/data/nodes"
18
+ SEGMENT = re.compile(r"croft-dataset-\d{8}T\d{6}Z\.jsonl\.zst")
19
+ LIVE = "croft-dataset.jsonl"
20
+
21
+
22
+ def files(location=None):
23
+ """Every dataset file, each node's in time order: segments by their start stamp, then the live file."""
24
+ root = Path(location or os.environ.get("CROFT_DATASET", DEFAULT_LOCATION))
25
+ if not root.is_dir():
26
+ return [root]
27
+ found = sorted(root.glob("*.jsonl"))
28
+ for node_dir in sorted(path for path in root.iterdir() if path.is_dir()):
29
+ found += sorted(path for path in node_dir.iterdir() if SEGMENT.fullmatch(path.name))
30
+ found += [node_dir / LIVE] if (node_dir / LIVE).exists() else []
31
+ return found
32
+
33
+
34
+ def node(path):
35
+ """The node a file belongs to: its directory for segments and live files, its name for <node>.jsonl."""
36
+ if path.name == LIVE or SEGMENT.fullmatch(path.name):
37
+ return path.parent.name
38
+ return path.name.removesuffix(".jsonl")
39
+
40
+
41
+ def _open(path):
42
+ handle = path.open("rb")
43
+ if path.name.endswith(".zst"):
44
+ return io.BufferedReader(zstandard.ZstdDecompressor().stream_reader(handle, closefd=True))
45
+ return handle
46
+
47
+
48
+ def fingerprint(paths):
49
+ digest = hashlib.sha256()
50
+ for path in paths:
51
+ stat = path.stat()
52
+ digest.update(f"{path.name}:{stat.st_size}:{stat.st_mtime_ns}".encode())
53
+ return digest.hexdigest()[:12]
54
+
55
+
56
+ @contextlib.contextmanager
57
+ def open_all(paths=None):
58
+ handles = [_open(path) for path in (paths or files())]
59
+ try:
60
+ yield itertools.chain.from_iterable(handles)
61
+ finally:
62
+ for handle in handles:
63
+ handle.close()
64
+
65
+
66
+ UNIX_MS = re.compile(rb'"unix_ms":(\d+)')
67
+
68
+
69
+ def _ms_near(handle, offset):
70
+ handle.seek(offset)
71
+ handle.readline()
72
+ for _ in range(50):
73
+ match = UNIX_MS.search(handle.readline())
74
+ if match:
75
+ return int(match.group(1))
76
+ return None
77
+
78
+
79
+ # offset_since bisects a file, whose rows are in time order, for a byte offset at or before the first row at since_ms.
80
+ def offset_since(path, since_ms):
81
+ low, high = 0, path.stat().st_size
82
+ with path.open("rb") as handle:
83
+ while high - low > 1 << 20:
84
+ middle = (low + high) // 2
85
+ ms = _ms_near(handle, middle)
86
+ if ms is None or ms >= since_ms:
87
+ high = middle
88
+ else:
89
+ low = middle
90
+ return low
91
+
92
+
93
+ def read(keep=lambda line: True, paths=None, since_ms=None, until_ms=None):
94
+ """Parses the rows whose raw bytes pass keep; returns them, each with its node, and the unreadable line count.
95
+
96
+ since_ms and until_ms limit the read to a time window: a plain file skips to it by bisection, a zstd segment
97
+ is decompressed from its start.
98
+ """
99
+ rows, unreadable = [], 0
100
+ for path in paths or files():
101
+ compressed = path.name.endswith(".zst")
102
+ with _open(path) as handle:
103
+ if since_ms is not None and not compressed:
104
+ offset = offset_since(path, since_ms)
105
+ handle.seek(offset)
106
+ if offset:
107
+ handle.readline()
108
+ for line in handle:
109
+ if until_ms is not None or (since_ms is not None and compressed):
110
+ match = UNIX_MS.search(line, 0, 40)
111
+ if match and until_ms is not None and int(match.group(1)) > until_ms:
112
+ break
113
+ if match and since_ms is not None and int(match.group(1)) < since_ms:
114
+ continue
115
+ if not line.strip() or not keep(line):
116
+ continue
117
+ try:
118
+ row = json.loads(line)
119
+ except ValueError:
120
+ unreadable += 1
121
+ continue
122
+ row["node"] = node(path)
123
+ rows.append(row)
124
+ return rows, unreadable
125
+
126
+
127
+ # Where each signal sits in a row, and whether a container's processes add up or share one cgroup value.
128
+ FIELDS = {
129
+ "syscall_rate": ("syscalls", None, True),
130
+ "runqueue_wait": ("usage", "runqueue_s", True),
131
+ "cpu_cores": ("cgroup_usage", "cpu_cores", False),
132
+ "cpu_throttled_fraction": ("cgroup_usage", "cpu_throttled_fraction", False),
133
+ "cpu_pressure": ("cgroup_usage", "cpu_pressure_avg10", False),
134
+ }
135
+ # Rows of one poll are written within a few milliseconds; polls are 2 s apart.
136
+ POLL_GAP_MS = 500
137
+ CONTAINER_ID = re.compile(rb'"container_id":"([0-9a-f]+)"')
138
+ SEGMENT_STAMP = re.compile(r"croft-dataset-(\d{8}T\d{6}Z)\.jsonl\.zst")
139
+
140
+
141
+ def signal(
142
+ name: str,
143
+ namespace: str | None = None,
144
+ deployment: str | None = None,
145
+ *,
146
+ since: metrics.Time = "1h",
147
+ until: metrics.Time | None = None,
148
+ step_s: float | None = None,
149
+ match: dict[str, str] | None = None,
150
+ names: kube.Names | None = None,
151
+ paths: list[Path] | None = None,
152
+ ) -> pd.DataFrame:
153
+ """Samples of one signal from the dataset files, in the shape of metrics.signal with node and container_id added.
154
+
155
+ A summed signal is the container's total per poll, averaged over the polls of each step; a poll without a row
156
+ for the container counts as 0, since idle processes are written every 15 polls. A cgroup signal is averaged over
157
+ the polls that hold it. names maps container_id to (namespace, pod, container), as kube.names() returns; namespace
158
+ and deployment need it.
159
+ """
160
+ if name not in FIELDS:
161
+ raise ValueError(f"the dataset holds {', '.join(FIELDS)}; {name} is on /metrics and Prometheus only")
162
+ spec = metrics.SIGNALS[name]
163
+ unknown = set(match or {}) - set(spec.labels)
164
+ if unknown:
165
+ raise ValueError(f"{name} has no label {', '.join(sorted(unknown))}; it has {spec.labels or 'none'}")
166
+ if (namespace or deployment) and names is None:
167
+ raise ValueError("namespace and deployment need names, such as names=kube.names()")
168
+ start_s, end_s, step_s = metrics.window(since, until, step_s)
169
+ wanted = _containers(names, namespace, deployment)
170
+
171
+ rows, stamps = [], []
172
+ for path in between(paths or files(), start_s * 1000, end_s * 1000):
173
+ seen: list[int] = []
174
+ found, _ = read(_keeper(wanted, seen), [path], since_ms=int(start_s * 1000), until_ms=int(end_s * 1000))
175
+ rows += found
176
+ stamps += [(node(path), stamp) for stamp in seen]
177
+ polls = _polls(pd.DataFrame(stamps, columns=["node", "unix_ms"]), step_s)
178
+ frame = _binned(_samples(rows, name, match or {}), polls, spec.labels, FIELDS[name][2])
179
+ frame = _named(frame, names, spec.labels)
180
+ frame.attrs.update(signal=name, unit=spec.unit, source="dataset")
181
+ return frame
182
+
183
+
184
+ def between(paths: list[Path], start_ms: float, end_ms: float) -> list[Path]:
185
+ """The files that can hold rows between start_ms and end_ms: a segment spans from its stamp to the next file's."""
186
+ chosen = []
187
+ for index, path in enumerate(paths):
188
+ begins = _stamp_ms(path)
189
+ following = paths[index + 1] if index + 1 < len(paths) else None
190
+ ends = _stamp_ms(following) if following is not None and node(following) == node(path) else None
191
+ if begins is not None and begins > end_ms:
192
+ continue
193
+ if begins is not None and ends is not None and ends < start_ms:
194
+ continue
195
+ chosen.append(path)
196
+ return chosen
197
+
198
+
199
+ def _stamp_ms(path: Path) -> float | None:
200
+ found = SEGMENT_STAMP.fullmatch(path.name)
201
+ return pd.Timestamp(found.group(1)).timestamp() * 1000 if found else None
202
+
203
+
204
+ # _keeper notes every row's time in seen, so polls in which the wanted containers wrote nothing still count.
205
+ def _keeper(wanted: set[str] | None, seen: list[int]):
206
+ def keep(line: bytes) -> bool:
207
+ stamp = UNIX_MS.search(line, 0, 40)
208
+ if stamp:
209
+ seen.append(int(stamp.group(1)))
210
+ found = CONTAINER_ID.search(line)
211
+ return found is not None and (wanted is None or found.group(1).decode() in wanted)
212
+
213
+ return keep
214
+
215
+
216
+ def _containers(names: kube.Names | None, namespace: str | None, deployment: str | None) -> set[str] | None:
217
+ if names is None or not (namespace or deployment):
218
+ return None
219
+ return {
220
+ identity
221
+ for identity, (ns, pod, _) in names.items()
222
+ if (not namespace or ns == namespace) and (not deployment or metrics.deployment_of(pod) == deployment)
223
+ }
224
+
225
+
226
+ def _samples(rows: list[dict], name: str, match: dict[str, str]) -> pd.DataFrame:
227
+ section, key, _ = FIELDS[name]
228
+ samples = []
229
+ for row in rows:
230
+ base = (row["unix_ms"], row["node"], row["process"]["container_id"])
231
+ if key is None:
232
+ samples += [(*base, s["name"], s["rate_per_sec"]) for s in row.get(section) or [] if _matches(s, match)]
233
+ else:
234
+ value = (row.get(section) or {}).get(key)
235
+ if value is not None:
236
+ samples.append((*base, value))
237
+ labels = ["syscall"] if key is None else []
238
+ return pd.DataFrame(samples, columns=["unix_ms", "node", "container_id", *labels, "value"])
239
+
240
+
241
+ def _matches(syscall: dict, match: dict[str, str]) -> bool:
242
+ return "syscall" not in match or syscall["name"] == match["syscall"]
243
+
244
+
245
+ def _polls(stamps: pd.DataFrame, step_s: float) -> pd.DataFrame:
246
+ """One row per poll and node: its first row's time and the step it falls in."""
247
+ stamps = stamps.sort_values(["node", "unix_ms"], ignore_index=True)
248
+ stamps["poll"] = stamps.groupby("node")["unix_ms"].diff().fillna(POLL_GAP_MS + 1).gt(POLL_GAP_MS).cumsum()
249
+ polls = stamps.groupby(["node", "poll"])["unix_ms"].min().rename("start_ms").reset_index()
250
+ polls["bin"] = (polls["start_ms"] / 1000 // step_s) * step_s
251
+ return polls
252
+
253
+
254
+ def _binned(samples: pd.DataFrame, polls: pd.DataFrame, labels: tuple[str, ...], summed: bool) -> pd.DataFrame:
255
+ keys = ["node", "container_id", *labels]
256
+ if samples.empty:
257
+ return pd.DataFrame(columns=["time", *keys, "value"]).astype({"time": "datetime64[ns, UTC]", "value": float})
258
+ samples = pd.merge_asof(
259
+ samples.sort_values("unix_ms"), polls.sort_values("start_ms"), left_on="unix_ms", right_on="start_ms", by="node"
260
+ )
261
+ per_poll = samples.groupby([*keys, "poll", "bin"])["value"].agg("sum" if summed else "max").reset_index()
262
+ if summed:
263
+ counts = polls.groupby(["node", "bin"]).size().rename("polls").reset_index()
264
+ frame = per_poll.groupby([*keys, "bin"])["value"].sum().reset_index().merge(counts, on=["node", "bin"])
265
+ frame["value"] = frame["value"] / frame["polls"]
266
+ else:
267
+ frame = per_poll.groupby([*keys, "bin"])["value"].mean().reset_index()
268
+ frame["time"] = pd.to_datetime(frame["bin"], unit="s", utc=True)
269
+ return frame[["time", *keys, "value"]].sort_values([*keys, "time"], ignore_index=True)
270
+
271
+
272
+ def _named(frame: pd.DataFrame, names: kube.Names | None, labels: tuple[str, ...]) -> pd.DataFrame:
273
+ if names is None:
274
+ return frame
275
+ found = frame["container_id"].map(names)
276
+ frame["namespace"] = found.map(lambda n: n[0] if isinstance(n, tuple) else None)
277
+ frame["pod"] = found.map(lambda n: n[1] if isinstance(n, tuple) else None)
278
+ frame["container"] = found.map(lambda n: n[2] if isinstance(n, tuple) else None)
279
+ frame["deployment"] = frame["pod"].map(metrics.deployment_of)
280
+ columns = ["time", "node", "namespace", "deployment", "pod", "container", "container_id", *labels, "value"]
281
+ return frame[columns]
282
+
283
+
284
+ def syscall_rate(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
285
+ return signal("syscall_rate", namespace, deployment, **options)
286
+
287
+
288
+ def runqueue_wait(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
289
+ return signal("runqueue_wait", namespace, deployment, **options)
290
+
291
+
292
+ def cpu_cores(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
293
+ return signal("cpu_cores", namespace, deployment, **options)
294
+
295
+
296
+ def cpu_throttled_fraction(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
297
+ return signal("cpu_throttled_fraction", namespace, deployment, **options)
298
+
299
+
300
+ def cpu_pressure(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
301
+ return signal("cpu_pressure", namespace, deployment, **options)
@@ -0,0 +1,31 @@
1
+ """Container names from Kubernetes, read through kubectl, to label dataset rows that carry a container_id only."""
2
+
3
+ import json
4
+ import subprocess
5
+
6
+ Names = dict[str, tuple[str, str, str]]
7
+
8
+
9
+ def names(kubeconfig: str | None = None, context: str | None = None) -> Names:
10
+ """container_id to (namespace, pod, container) for every pod the API server holds, with each container's
11
+ previous instance; containers of deleted pods are absent."""
12
+ command = ["kubectl", "get", "pods", "--all-namespaces", "-o", "json"]
13
+ command += ["--kubeconfig", kubeconfig] if kubeconfig else []
14
+ command += ["--context", context] if context else []
15
+ result = subprocess.run(command, capture_output=True, text=True, check=False)
16
+ if result.returncode:
17
+ raise RuntimeError(f"kubectl: {result.stderr.strip()}")
18
+ return names_of(json.loads(result.stdout))
19
+
20
+
21
+ def names_of(pods: dict) -> Names:
22
+ found: Names = {}
23
+ for pod in pods.get("items", []):
24
+ namespace, name = pod["metadata"]["namespace"], pod["metadata"]["name"]
25
+ status = pod.get("status", {})
26
+ for container in status.get("initContainerStatuses", []) + status.get("containerStatuses", []):
27
+ previous = container.get("lastState", {}).get("terminated", {})
28
+ for identity in (container.get("containerID"), previous.get("containerID")):
29
+ if identity:
30
+ found[identity.split("://")[-1]] = (namespace, name, container["name"])
31
+ return found
@@ -0,0 +1,270 @@
1
+ """Collector signals from Prometheus as pandas frames, one row per sample, joined to the pod and container that produced it."""
2
+
3
+ import os
4
+ import re
5
+ import time
6
+ import urllib.request
7
+ from dataclasses import dataclass
8
+ from datetime import datetime
9
+
10
+ import pandas as pd
11
+
12
+ from .prometheus import Prometheus
13
+
14
+
15
+ @dataclass(frozen=True)
16
+ class Signal:
17
+ metric: str
18
+ labels: tuple[str, ...]
19
+ counter: bool
20
+ unit: str
21
+
22
+
23
+ SIGNALS = {
24
+ "syscall_rate": Signal("croft_container_syscall_rate", ("syscall",), False, "calls/s"),
25
+ "syscall_bytes": Signal("croft_container_syscall_bytes_total", ("syscall",), True, "bytes/s"),
26
+ "socket_io": Signal("croft_container_socket_io_total", ("direction",), True, "calls/s"),
27
+ "connects": Signal("croft_container_connects_total", ("address", "port"), True, "connects/s"),
28
+ "cpu_cores": Signal("croft_container_cpu_cores", (), False, "cores"),
29
+ "cpu_throttled_fraction": Signal("croft_container_cpu_throttled_fraction", (), False, "fraction"),
30
+ "cpu_pressure": Signal("croft_container_cpu_pressure_avg10", (), False, "%"),
31
+ "runqueue_wait": Signal("croft_container_runqueue_wait", (), False, "s/s"),
32
+ "inflight_requests": Signal("croft_container_inflight_requests", (), False, "requests"),
33
+ }
34
+ # The collector is scraped every 5 s, so 15 s holds three samples for rate().
35
+ RATE_WINDOW_S = 15
36
+ MIN_STEP_S = 5
37
+ MAX_POINTS = 1000
38
+ DURATION = re.compile(r"(\d+(?:\.\d+)?)([smhdw])")
39
+ SECONDS = {"s": 1, "m": 60, "h": 3600, "d": 86400, "w": 604800}
40
+ POD = re.compile(r"(.+)-[a-z0-9]{5,10}-[a-z0-9]{5}")
41
+ KUBE_LABELS = ("namespace", "pod", "container")
42
+ LINE = re.compile(r"([a-zA-Z_:][a-zA-Z0-9_:]*)(?:\{(.*)\})? (\S+)")
43
+ LABEL = re.compile(r'([a-zA-Z_][a-zA-Z0-9_]*)="((?:[^"\\]|\\.)*)"')
44
+ ESCAPES = {"\\": "\\", '"': '"', "n": "\n"}
45
+ ESCAPE = re.compile(r"\\(.)")
46
+
47
+ Time = str | float | int | datetime | pd.Timestamp
48
+
49
+
50
+ def catalog() -> pd.DataFrame:
51
+ """Every signal with its Prometheus metric, extra labels, kind and unit."""
52
+ return pd.DataFrame(
53
+ [
54
+ {"signal": name, "metric": s.metric, "labels": ",".join(s.labels), "counter": s.counter, "unit": s.unit}
55
+ for name, s in SIGNALS.items()
56
+ ]
57
+ ).set_index("signal")
58
+
59
+
60
+ def signal(
61
+ name: str,
62
+ namespace: str | None = None,
63
+ deployment: str | None = None,
64
+ *,
65
+ since: Time = "1h",
66
+ until: Time | None = None,
67
+ step_s: float | None = None,
68
+ match: dict[str, str] | None = None,
69
+ kube: bool = True,
70
+ prometheus: Prometheus | None = None,
71
+ ) -> pd.DataFrame:
72
+ """Samples of one signal between since and until; counters come as per-second rates.
73
+
74
+ namespace and deployment narrow the pods; match narrows the signal's own labels, such as {"syscall": "sendto"}.
75
+ kube=False skips the join to kube-state-metrics and keys rows by container_id, for containers outside Kubernetes.
76
+ """
77
+ spec = SIGNALS[name]
78
+ start_s, end_s, step_s = window(since, until, step_s)
79
+ query = promql(spec, namespace, deployment, match, kube, step_s)
80
+ streams = (prometheus or Prometheus()).query_range(query, start_s, end_s, step_s)
81
+ frame = frame_of(streams, spec, kube)
82
+ frame.attrs.update(signal=name, unit=spec.unit, query=query)
83
+ return frame
84
+
85
+
86
+ # promql averages a gauge over each step and widens a counter's rate window to it, so a long range does not
87
+ # sample one scrape per step and alias periodic spikes.
88
+ def promql(
89
+ spec: Signal,
90
+ namespace: str | None,
91
+ deployment: str | None,
92
+ match: dict[str, str] | None,
93
+ kube: bool,
94
+ step_s: float = MIN_STEP_S,
95
+ ) -> str:
96
+ unknown = set(match or {}) - set(spec.labels)
97
+ if unknown:
98
+ raise ValueError(f"{spec.metric} has no label {', '.join(sorted(unknown))}; it has {spec.labels or 'none'}")
99
+ expr = f"{spec.metric}{_selector(match or {})}"
100
+ span = f"{max(RATE_WINDOW_S, step_s) if spec.counter else step_s:g}s"
101
+ if spec.counter:
102
+ expr = f"rate({expr}[{span}])"
103
+ elif step_s > MIN_STEP_S:
104
+ expr = f"avg_over_time({expr}[{span}])"
105
+ if not kube:
106
+ return expr
107
+ pods = {"namespace": namespace} if namespace else {}
108
+ if deployment:
109
+ pods["pod=~"] = f"{re.escape(deployment)}-[a-z0-9]+-[a-z0-9]+"
110
+ info = f'label_replace(kube_pod_container_info{_selector(pods)}, "container_id", "$1", "container_id", "[a-z-]+://(.+)")'
111
+ labels = ", ".join(KUBE_LABELS)
112
+ return f"{expr} * on (container_id) group_left ({labels}) max by (container_id, {labels}) ({info})"
113
+
114
+
115
+ def frame_of(streams: list[dict], spec: Signal, kube: bool) -> pd.DataFrame:
116
+ keys = (*KUBE_LABELS, *spec.labels) if kube else ("container_id", *spec.labels)
117
+ rows = []
118
+ for stream in streams:
119
+ labels = {key: stream["metric"].get(key) for key in keys}
120
+ if kube:
121
+ labels["deployment"] = deployment_of(labels["pod"])
122
+ rows += [{"time": float(t), **labels, "value": float(v)} for t, v in stream["values"] if v != "NaN"]
123
+ columns = ["time", *(("namespace", "deployment", "pod", "container") if kube else ("container_id",)), *spec.labels]
124
+ frame = pd.DataFrame(rows, columns=[*columns, "value"])
125
+ frame["time"] = pd.to_datetime(frame["time"], unit="s", utc=True)
126
+ return frame.sort_values([*columns[1:], "time"], ignore_index=True)
127
+
128
+
129
+ def wide(frame: pd.DataFrame, by: str | list[str] = "pod") -> pd.DataFrame:
130
+ """One column per value of by, summed over the other labels, ready for .plot()."""
131
+ table = frame.pivot_table(index="time", columns=by, values="value", aggfunc="sum")
132
+ table.attrs = dict(frame.attrs)
133
+ return table
134
+
135
+
136
+ def _unescape(value: str) -> str:
137
+ return ESCAPE.sub(lambda m: ESCAPES.get(m.group(1), m.group(0)), value)
138
+
139
+
140
+ def live(address: str | None = None, timeout_s: float = 10) -> pd.DataFrame:
141
+ """The newest poll of one collector, read from its /metrics: one row per series, counters as running totals.
142
+
143
+ address is host:port or a URL, defaulting to CROFT_METRICS_ADDR. Rows are keyed by container_id; the collector
144
+ holds no history and no pod names, so time ranges and the pod join need signal() and Prometheus.
145
+ """
146
+ address = address or os.environ.get("CROFT_METRICS_ADDR") or "127.0.0.1:9464"
147
+ url = address if "://" in address else f"http://{address}"
148
+ with urllib.request.urlopen(f"{url.rstrip('/')}/metrics", timeout=timeout_s) as response:
149
+ text = response.read().decode()
150
+ names = {spec.metric: name for name, spec in SIGNALS.items()}
151
+ rows = []
152
+ for line in text.splitlines():
153
+ sample = _sample(line)
154
+ if sample and sample[0] in names:
155
+ metric, labels, value = sample
156
+ rows.append({"signal": names[metric], **labels, "value": value})
157
+ frame = pd.DataFrame(rows, columns=None if rows else ["signal", "container_id", "value"])
158
+ poll = re.search(r"^croft_poll_unix_ms (\d+)", text, re.MULTILINE)
159
+ frame.attrs.update(poll_time=pd.Timestamp(int(poll.group(1)), unit="ms", tz="UTC") if poll else None)
160
+ return frame
161
+
162
+
163
+ def _sample(line: str) -> tuple[str, dict[str, str], float] | None:
164
+ found = LINE.fullmatch(line.strip())
165
+ if not found or line.startswith("#"):
166
+ return None
167
+ labels = {key: _unescape(value) for key, value in LABEL.findall(found.group(2) or "")}
168
+ # labels = {
169
+ # key: ESCAPE.sub(lambda m: ESCAPES.get(m.group(1), m.group(0)), value) for key, value in LABEL.findall(found.group(2) or "")
170
+ # }
171
+ return found.group(1), labels, float(found.group(3))
172
+
173
+
174
+ def window(since: Time, until: Time | None, step_s: float | None) -> tuple[float, float, float]:
175
+ """Start, end and step in unix seconds; the step keeps a range under MAX_POINTS samples."""
176
+ now = time.time()
177
+ end_s = _seconds(until, now) if until is not None else now
178
+ start_s = _seconds(since, end_s)
179
+ if start_s >= end_s:
180
+ raise ValueError(f"since ({since}) is not before until ({until or 'now'})")
181
+ return start_s, end_s, step_s or max(MIN_STEP_S, (end_s - start_s) / MAX_POINTS)
182
+
183
+
184
+ def deployment_of(pod: str | None) -> str | None:
185
+ found = POD.fullmatch(pod or "")
186
+ return found.group(1) if found else None
187
+
188
+
189
+ def query(
190
+ text: str,
191
+ *,
192
+ since: Time = "1h",
193
+ until: Time | None = None,
194
+ step_s: float | None = None,
195
+ prometheus: Prometheus | None = None,
196
+ ) -> pd.DataFrame:
197
+ """Any PromQL range query as a long frame with one column per label."""
198
+ rows = [
199
+ {"time": float(t), **stream["metric"], "value": float(v)}
200
+ for stream in (prometheus or Prometheus()).query_range(text, *window(since, until, step_s))
201
+ for t, v in stream["values"]
202
+ if v != "NaN"
203
+ ]
204
+ frame = pd.DataFrame(rows, columns=None if rows else ["time", "value"])
205
+ frame["time"] = pd.to_datetime(frame["time"], unit="s", utc=True)
206
+ frame.attrs.update(query=text)
207
+ return frame
208
+
209
+
210
+ def syscall_rate(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
211
+ return signal("syscall_rate", namespace, deployment, **options)
212
+
213
+
214
+ def syscall_bytes(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
215
+ return signal("syscall_bytes", namespace, deployment, **options)
216
+
217
+
218
+ def socket_io(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
219
+ return signal("socket_io", namespace, deployment, **options)
220
+
221
+
222
+ def connects(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
223
+ return signal("connects", namespace, deployment, **options)
224
+
225
+
226
+ def cpu_cores(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
227
+ return signal("cpu_cores", namespace, deployment, **options)
228
+
229
+
230
+ def cpu_throttled_fraction(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
231
+ return signal("cpu_throttled_fraction", namespace, deployment, **options)
232
+
233
+
234
+ def cpu_pressure(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
235
+ return signal("cpu_pressure", namespace, deployment, **options)
236
+
237
+
238
+ def runqueue_wait(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
239
+ return signal("runqueue_wait", namespace, deployment, **options)
240
+
241
+
242
+ def inflight_requests(namespace: str | None = None, deployment: str | None = None, **options) -> pd.DataFrame:
243
+ return signal("inflight_requests", namespace, deployment, **options)
244
+
245
+
246
+ def _selector(labels: dict[str, str]) -> str:
247
+ if not labels:
248
+ return ""
249
+ pairs = []
250
+ for key, value in labels.items():
251
+ name, op = (key[:-2], "=~") if key.endswith("=~") else (key, "=")
252
+ pairs.append(f"{name}{op}{_quote(value)}")
253
+ return "{" + ",".join(pairs) + "}"
254
+
255
+
256
+ def _quote(value: str) -> str:
257
+ return '"' + value.replace("\\", "\\\\").replace('"', '\\"') + '"'
258
+
259
+
260
+ # _seconds reads a unix time, a datetime, or a duration such as "90m" before relative_to.
261
+ def _seconds(at: Time, relative_to: float | None = None) -> float:
262
+ if isinstance(at, (int, float)):
263
+ return float(at)
264
+ if isinstance(at, datetime):
265
+ return pd.Timestamp(at).timestamp()
266
+ if relative_to is not None:
267
+ parts = DURATION.findall(at)
268
+ if parts and "".join(n + u for n, u in parts) == at:
269
+ return relative_to - sum(float(n) * SECONDS[u] for n, u in parts)
270
+ return pd.Timestamp(at).timestamp()
@@ -0,0 +1,16 @@
1
+ """Line charts of signal frames, one line per pod or other label, with the signal's unit on the y axis."""
2
+
3
+ import pandas as pd
4
+
5
+ from .metrics import wide
6
+
7
+
8
+ def lines(frame: pd.DataFrame, by: str | list[str] = "pod", ax=None, title: str | None = None):
9
+ """Draws frame as one line per value of by and returns the axes."""
10
+ table = wide(frame, by)
11
+ ax = table.plot(ax=ax, linewidth=1.5, legend=table.shape[1] > 1)
12
+ ax.set_xlabel("time (UTC)")
13
+ ax.set_ylabel(frame.attrs.get("unit", "value"))
14
+ ax.set_title(title or frame.attrs.get("signal", ""))
15
+ ax.grid(alpha=0.3)
16
+ return ax
@@ -0,0 +1,48 @@
1
+ """Prometheus HTTP API client: instant and range queries, with the server's error message on failure."""
2
+
3
+ import json
4
+ import os
5
+ import urllib.error
6
+ import urllib.parse
7
+ import urllib.request
8
+
9
+ DEFAULT_URL = "http://prometheus.127.0.0.1.sslip.io:30080"
10
+
11
+
12
+ class PrometheusError(RuntimeError):
13
+ pass
14
+
15
+
16
+ class Prometheus:
17
+ def __init__(self, url: str | None = None, timeout_s: float = 60):
18
+ self.url = (url or os.environ.get("CROFT_PROMETHEUS") or os.environ.get("PROM_HOST") or DEFAULT_URL).rstrip("/")
19
+ self.timeout_s = timeout_s
20
+
21
+ def query(self, query: str, at_s: float | None = None) -> list[dict]:
22
+ params = {"query": query} if at_s is None else {"query": query, "time": at_s}
23
+ return self._call("query", params)
24
+
25
+ def query_range(self, query: str, start_s: float, end_s: float, step_s: float) -> list[dict]:
26
+ return self._call("query_range", {"query": query, "start": start_s, "end": end_s, "step": step_s})
27
+
28
+ def _call(self, path: str, params: dict) -> list[dict]:
29
+ body = urllib.parse.urlencode(params).encode()
30
+ try:
31
+ with urllib.request.urlopen(f"{self.url}/api/v1/{path}", body, timeout=self.timeout_s) as response:
32
+ answer = json.load(response)
33
+ except urllib.error.HTTPError as error:
34
+ answer = _json_or_none(error.read())
35
+ if answer is None:
36
+ raise PrometheusError(f"{self.url}: HTTP {error.code}") from error
37
+ except OSError as error:
38
+ raise PrometheusError(f"{self.url}: {error}") from error
39
+ if answer.get("status") != "success":
40
+ raise PrometheusError(f"{answer.get('errorType', 'error')}: {answer.get('error', answer)}")
41
+ return answer["data"]["result"]
42
+
43
+
44
+ def _json_or_none(raw: bytes) -> dict | None:
45
+ try:
46
+ return json.loads(raw)
47
+ except ValueError:
48
+ return None