workloadtruth-cli 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- workloadtruth/__init__.py +1 -0
- workloadtruth/audit_log.py +148 -0
- workloadtruth/benchmark.py +111 -0
- workloadtruth/classifier/__init__.py +3 -0
- workloadtruth/classifier/experimental.py +29 -0
- workloadtruth/classifier/rules.py +183 -0
- workloadtruth/cli.py +217 -0
- workloadtruth/mcp_server.py +89 -0
- workloadtruth/telemetry/__init__.py +22 -0
- workloadtruth/telemetry/base.py +46 -0
- workloadtruth/telemetry/nvml_backend.py +56 -0
- workloadtruth/telemetry/synthetic_backend.py +151 -0
- workloadtruth/types.py +50 -0
- workloadtruth_cli-0.1.0.dist-info/METADATA +220 -0
- workloadtruth_cli-0.1.0.dist-info/RECORD +18 -0
- workloadtruth_cli-0.1.0.dist-info/WHEEL +4 -0
- workloadtruth_cli-0.1.0.dist-info/entry_points.txt +2 -0
- workloadtruth_cli-0.1.0.dist-info/licenses/LICENSE +190 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""Hash-chained local audit log.
|
|
2
|
+
|
|
3
|
+
Every `workloadtruth watch` classification event is appended to a local
|
|
4
|
+
JSONL file as a hash-chained entry: each entry's hash covers its own content
|
|
5
|
+
plus the previous entry's hash, so any edit, reorder, or deletion of a past
|
|
6
|
+
entry breaks the chain from that point forward and is detected by
|
|
7
|
+
`verify_chain`. This mirrors the audit-log pattern already shipped in this
|
|
8
|
+
portfolio's `auditreach`/`tokentrust` CLIs.
|
|
9
|
+
|
|
10
|
+
This proves what was classified and when, and that the local record hasn't
|
|
11
|
+
been silently altered after the fact. It does NOT prove the classification
|
|
12
|
+
itself was correct, and it does NOT constitute regulatory compliance
|
|
13
|
+
evidence -- no regulation currently requires this. See the README's "What
|
|
14
|
+
WorkloadTruth is not" section.
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
import hashlib
|
|
20
|
+
import json
|
|
21
|
+
from dataclasses import asdict, dataclass
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
GENESIS_HASH = "0" * 64
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _canonical_json(obj: dict[str, Any]) -> str:
|
|
29
|
+
return json.dumps(obj, sort_keys=True, separators=(",", ":"))
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(frozen=True)
|
|
33
|
+
class AuditLogEntry:
|
|
34
|
+
timestamp: float
|
|
35
|
+
gpu_index: int
|
|
36
|
+
workload_type: str
|
|
37
|
+
confidence: float
|
|
38
|
+
reasons: list[str]
|
|
39
|
+
features: dict[str, float]
|
|
40
|
+
prev_hash: str
|
|
41
|
+
entry_hash: str
|
|
42
|
+
|
|
43
|
+
def to_dict(self) -> dict[str, Any]:
|
|
44
|
+
return asdict(self)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _compute_entry_hash(payload: dict[str, Any], prev_hash: str) -> str:
|
|
48
|
+
body = _canonical_json(payload) + prev_hash
|
|
49
|
+
return hashlib.sha256(body.encode("utf-8")).hexdigest()
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def build_entry(
|
|
53
|
+
*,
|
|
54
|
+
timestamp: float,
|
|
55
|
+
gpu_index: int,
|
|
56
|
+
workload_type: str,
|
|
57
|
+
confidence: float,
|
|
58
|
+
reasons: list[str],
|
|
59
|
+
features: dict[str, float],
|
|
60
|
+
prev_hash: str,
|
|
61
|
+
) -> AuditLogEntry:
|
|
62
|
+
rounded_confidence = round(confidence, 4)
|
|
63
|
+
rounded_features = {k: round(v, 4) for k, v in features.items()}
|
|
64
|
+
payload: dict[str, Any] = {
|
|
65
|
+
"timestamp": timestamp,
|
|
66
|
+
"gpu_index": gpu_index,
|
|
67
|
+
"workload_type": workload_type,
|
|
68
|
+
"confidence": rounded_confidence,
|
|
69
|
+
"reasons": reasons,
|
|
70
|
+
"features": rounded_features,
|
|
71
|
+
}
|
|
72
|
+
entry_hash = _compute_entry_hash(payload, prev_hash)
|
|
73
|
+
return AuditLogEntry(
|
|
74
|
+
timestamp=timestamp,
|
|
75
|
+
gpu_index=gpu_index,
|
|
76
|
+
workload_type=workload_type,
|
|
77
|
+
confidence=rounded_confidence,
|
|
78
|
+
reasons=reasons,
|
|
79
|
+
features=rounded_features,
|
|
80
|
+
prev_hash=prev_hash,
|
|
81
|
+
entry_hash=entry_hash,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def last_hash(log_path: Path) -> str:
|
|
86
|
+
if not log_path.exists():
|
|
87
|
+
return GENESIS_HASH
|
|
88
|
+
last_line = None
|
|
89
|
+
with log_path.open("r", encoding="utf-8") as f:
|
|
90
|
+
for line in f:
|
|
91
|
+
line = line.strip()
|
|
92
|
+
if line:
|
|
93
|
+
last_line = line
|
|
94
|
+
if last_line is None:
|
|
95
|
+
return GENESIS_HASH
|
|
96
|
+
return str(json.loads(last_line)["entry_hash"])
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def append_entry(log_path: Path, entry: AuditLogEntry) -> None:
|
|
100
|
+
log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
101
|
+
with log_path.open("a", encoding="utf-8") as f:
|
|
102
|
+
f.write(_canonical_json(entry.to_dict()) + "\n")
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def verify_chain(log_path: Path) -> tuple[bool, str, int]:
|
|
106
|
+
"""Re-derive every entry's hash and check chain linkage.
|
|
107
|
+
|
|
108
|
+
Returns (is_valid, message, entries_checked).
|
|
109
|
+
"""
|
|
110
|
+
if not log_path.exists():
|
|
111
|
+
return True, "log file does not exist yet -- nothing to verify", 0
|
|
112
|
+
|
|
113
|
+
prev_hash = GENESIS_HASH
|
|
114
|
+
count = 0
|
|
115
|
+
with log_path.open("r", encoding="utf-8") as f:
|
|
116
|
+
for line_number, raw_line in enumerate(f, start=1):
|
|
117
|
+
raw_line = raw_line.strip()
|
|
118
|
+
if not raw_line:
|
|
119
|
+
continue
|
|
120
|
+
try:
|
|
121
|
+
record = json.loads(raw_line)
|
|
122
|
+
except json.JSONDecodeError as exc:
|
|
123
|
+
return False, f"line {line_number}: invalid JSON ({exc})", count
|
|
124
|
+
|
|
125
|
+
claimed_hash = record.get("entry_hash")
|
|
126
|
+
claimed_prev = record.get("prev_hash")
|
|
127
|
+
if claimed_prev != prev_hash:
|
|
128
|
+
return (
|
|
129
|
+
False,
|
|
130
|
+
f"line {line_number}: prev_hash mismatch "
|
|
131
|
+
f"(expected {prev_hash}, found {claimed_prev})",
|
|
132
|
+
count,
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
payload = {k: v for k, v in record.items() if k not in ("prev_hash", "entry_hash")}
|
|
136
|
+
recomputed = _compute_entry_hash(payload, prev_hash)
|
|
137
|
+
if recomputed != claimed_hash:
|
|
138
|
+
return (
|
|
139
|
+
False,
|
|
140
|
+
f"line {line_number}: entry_hash mismatch -- content was modified "
|
|
141
|
+
f"after being written (expected {recomputed}, found {claimed_hash})",
|
|
142
|
+
count,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
prev_hash = claimed_hash
|
|
146
|
+
count += 1
|
|
147
|
+
|
|
148
|
+
return True, f"chain verified, {count} entries", count
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
"""Evasion-robustness benchmark suite.
|
|
2
|
+
|
|
3
|
+
Measures the rule-based classifier's accuracy against SyntheticBackend
|
|
4
|
+
traces, both "clean" and under the documented evasion transform (see
|
|
5
|
+
telemetry/synthetic_backend.py). This is WorkloadTruth's core
|
|
6
|
+
differentiation artifact: arXiv:2606.19262 measured a real accuracy drop
|
|
7
|
+
under obfuscation (98.2% clean, 43-87% evasive) on real GPU hardware and a
|
|
8
|
+
dataset that was never published. This benchmark runs the same *kind* of
|
|
9
|
+
test, on synthetic data, on whatever machine `workloadtruth benchmark` is
|
|
10
|
+
invoked on.
|
|
11
|
+
|
|
12
|
+
The numbers this benchmark produces are NOT comparable to the paper's --
|
|
13
|
+
different data, different (and much simpler) hardware simulation, no shared
|
|
14
|
+
ground truth. Report them side by side in the README, labeled separately,
|
|
15
|
+
never blended into one number. This is a binding rule from this project's
|
|
16
|
+
own anti-sycophancy audit, not a style preference.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import dataclass
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
from workloadtruth.classifier.rules import classify
|
|
25
|
+
from workloadtruth.telemetry.synthetic_backend import SyntheticBackend
|
|
26
|
+
from workloadtruth.types import WorkloadType
|
|
27
|
+
|
|
28
|
+
_PROFILE_TO_EXPECTED = {
|
|
29
|
+
"training": WorkloadType.TRAINING,
|
|
30
|
+
"inference": WorkloadType.INFERENCE,
|
|
31
|
+
"idle": WorkloadType.IDLE,
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass(frozen=True)
|
|
36
|
+
class BenchmarkCell:
|
|
37
|
+
profile: str
|
|
38
|
+
evasion: bool
|
|
39
|
+
trials: int
|
|
40
|
+
correct: int
|
|
41
|
+
|
|
42
|
+
@property
|
|
43
|
+
def accuracy(self) -> float:
|
|
44
|
+
return self.correct / self.trials if self.trials else 0.0
|
|
45
|
+
|
|
46
|
+
def to_dict(self) -> dict[str, Any]:
|
|
47
|
+
return {
|
|
48
|
+
"profile": self.profile,
|
|
49
|
+
"evasion": self.evasion,
|
|
50
|
+
"trials": self.trials,
|
|
51
|
+
"correct": self.correct,
|
|
52
|
+
"accuracy": round(self.accuracy, 4),
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@dataclass(frozen=True)
|
|
57
|
+
class BenchmarkReport:
|
|
58
|
+
cells: list[BenchmarkCell]
|
|
59
|
+
window_size: int
|
|
60
|
+
trials_per_cell: int
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def clean_accuracy(self) -> float:
|
|
64
|
+
clean_cells = [c for c in self.cells if not c.evasion]
|
|
65
|
+
total = sum(c.trials for c in clean_cells)
|
|
66
|
+
correct = sum(c.correct for c in clean_cells)
|
|
67
|
+
return correct / total if total else 0.0
|
|
68
|
+
|
|
69
|
+
@property
|
|
70
|
+
def evasion_accuracy(self) -> float:
|
|
71
|
+
evasive_cells = [c for c in self.cells if c.evasion]
|
|
72
|
+
total = sum(c.trials for c in evasive_cells)
|
|
73
|
+
correct = sum(c.correct for c in evasive_cells)
|
|
74
|
+
return correct / total if total else 0.0
|
|
75
|
+
|
|
76
|
+
def to_dict(self) -> dict[str, Any]:
|
|
77
|
+
return {
|
|
78
|
+
"source": "synthetic",
|
|
79
|
+
"note": (
|
|
80
|
+
"Measured against documented synthetic GPU telemetry traces, "
|
|
81
|
+
"not live NVIDIA hardware. Not comparable to arXiv:2606.19262's "
|
|
82
|
+
"real-hardware numbers -- see benchmark.py module docstring."
|
|
83
|
+
),
|
|
84
|
+
"window_size": self.window_size,
|
|
85
|
+
"trials_per_cell": self.trials_per_cell,
|
|
86
|
+
"clean_accuracy": round(self.clean_accuracy, 4),
|
|
87
|
+
"evasion_accuracy": round(self.evasion_accuracy, 4),
|
|
88
|
+
"cells": [c.to_dict() for c in self.cells],
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def run_benchmark(
|
|
93
|
+
trials_per_cell: int = 50, window_size: int = 30, base_seed: int = 1000
|
|
94
|
+
) -> BenchmarkReport:
|
|
95
|
+
cells: list[BenchmarkCell] = []
|
|
96
|
+
for profile, expected in _PROFILE_TO_EXPECTED.items():
|
|
97
|
+
for evasion in (False, True):
|
|
98
|
+
correct = 0
|
|
99
|
+
for trial in range(trials_per_cell):
|
|
100
|
+
seed = base_seed + hash((profile, evasion, trial)) % 100_000
|
|
101
|
+
backend = SyntheticBackend(profile=profile, evasion=evasion, seed=seed)
|
|
102
|
+
samples = [backend.sample(gpu_index=0) for _ in range(window_size)]
|
|
103
|
+
result = classify(samples)
|
|
104
|
+
if result.workload_type == expected:
|
|
105
|
+
correct += 1
|
|
106
|
+
cells.append(
|
|
107
|
+
BenchmarkCell(
|
|
108
|
+
profile=profile, evasion=evasion, trials=trials_per_cell, correct=correct
|
|
109
|
+
)
|
|
110
|
+
)
|
|
111
|
+
return BenchmarkReport(cells=cells, window_size=window_size, trials_per_cell=trials_per_cell)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Placeholder for a future ML-based classifier.
|
|
2
|
+
|
|
3
|
+
Per this project's own anti-sycophancy rules: an ML classifier is only
|
|
4
|
+
shipped once it is independently shown to beat the rule-based baseline on a
|
|
5
|
+
real, honestly-reported test set (see the execution plan's Assumption 3).
|
|
6
|
+
No such model exists yet -- arXiv:2606.19262's trained weights and dataset
|
|
7
|
+
were not released, and this project's build environment has no GPU to
|
|
8
|
+
collect real training data on. Shipping a stub that returns fabricated
|
|
9
|
+
predictions would misrepresent accuracy that was never measured, so
|
|
10
|
+
`--experimental` fails loudly instead.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from typing import Any, NoReturn
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ExperimentalClassifierUnavailable(RuntimeError):
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def classify_experimental(*_args: Any, **_kwargs: Any) -> NoReturn:
|
|
23
|
+
raise ExperimentalClassifierUnavailable(
|
|
24
|
+
"The experimental ML classifier is not available in this release. "
|
|
25
|
+
"workloadtruth ships only the rule-based classifier (see "
|
|
26
|
+
"classifier/rules.py) until an ML classifier is trained and "
|
|
27
|
+
"independently benchmarked against it on a real, disclosed test "
|
|
28
|
+
"set. Run `workloadtruth classify` without --experimental."
|
|
29
|
+
)
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
"""Rule-based workload classifier.
|
|
2
|
+
|
|
3
|
+
This is the default (and, in v0.1, the only shipped) classifier. It is a
|
|
4
|
+
transparent, documented heuristic, not a trained model -- every threshold
|
|
5
|
+
below is a constant you can read, challenge, and override, not a black box.
|
|
6
|
+
See the README's "How classification works" section for why v0.1 ships
|
|
7
|
+
this instead of the ML classifier arXiv:2606.19262 describes:
|
|
8
|
+
without the paper's training dataset (not published) or real GPU hardware
|
|
9
|
+
in this project's build environment, an from-scratch ML classifier would be
|
|
10
|
+
an unvalidated claim, not a measured one. This module is intentionally the
|
|
11
|
+
inspectable alternative.
|
|
12
|
+
|
|
13
|
+
Feature intuition (same rationale as synthetic_backend.py's profile design,
|
|
14
|
+
kept in sync deliberately):
|
|
15
|
+
|
|
16
|
+
- Training: sustained high, low-variance GPU utilization; memory usage
|
|
17
|
+
trending upward across the sampling window (activations, gradient
|
|
18
|
+
buffers, optimizer state accumulating); low-variance, near-ceiling power
|
|
19
|
+
draw.
|
|
20
|
+
- Inference: bursty, high-variance GPU utilization tracking request
|
|
21
|
+
arrival; flat memory usage (resident weights, no growing training state);
|
|
22
|
+
high-variance power draw.
|
|
23
|
+
- Idle: near-zero utilization and power draw across the whole window.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
from workloadtruth.types import ClassificationResult, TelemetrySample, WorkloadType
|
|
29
|
+
|
|
30
|
+
# Idle detection
|
|
31
|
+
IDLE_GPU_UTIL_PCT = 5.0
|
|
32
|
+
IDLE_POWER_WATTS = 50.0
|
|
33
|
+
|
|
34
|
+
# Training signal thresholds
|
|
35
|
+
TRAINING_MIN_AVG_GPU_UTIL_PCT = 65.0
|
|
36
|
+
TRAINING_MAX_GPU_UTIL_STD_PCT = 15.0
|
|
37
|
+
TRAINING_MIN_MEM_GROWTH_MIB_PER_SAMPLE = 5.0
|
|
38
|
+
TRAINING_MAX_POWER_STD_WATTS = 25.0
|
|
39
|
+
|
|
40
|
+
# Inference signal thresholds
|
|
41
|
+
INFERENCE_MAX_AVG_GPU_UTIL_PCT = 70.0
|
|
42
|
+
INFERENCE_MIN_GPU_UTIL_STD_PCT = 12.0
|
|
43
|
+
INFERENCE_MIN_POWER_STD_WATTS = 20.0
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def _mean(values: list[float]) -> float:
|
|
47
|
+
return sum(values) / len(values) if values else 0.0
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def _stdev(values: list[float]) -> float:
|
|
51
|
+
if len(values) < 2:
|
|
52
|
+
return 0.0
|
|
53
|
+
m = _mean(values)
|
|
54
|
+
variance = sum((v - m) ** 2 for v in values) / (len(values) - 1)
|
|
55
|
+
return float(variance**0.5)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _linear_slope_per_sample(values: list[float]) -> float:
|
|
59
|
+
"""Least-squares slope of `values` against sample index (0, 1, 2, ...)."""
|
|
60
|
+
n = len(values)
|
|
61
|
+
if n < 2:
|
|
62
|
+
return 0.0
|
|
63
|
+
xs = [float(i) for i in range(n)]
|
|
64
|
+
x_mean = _mean(xs)
|
|
65
|
+
y_mean = _mean(values)
|
|
66
|
+
numerator = sum((xs[i] - x_mean) * (values[i] - y_mean) for i in range(n))
|
|
67
|
+
denominator = sum((xs[i] - x_mean) ** 2 for i in range(n))
|
|
68
|
+
return numerator / denominator if denominator else 0.0
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
def extract_features(samples: list[TelemetrySample]) -> dict[str, float]:
|
|
72
|
+
gpu_util = [s.gpu_utilization_pct for s in samples]
|
|
73
|
+
mem_used = [s.memory_used_mib for s in samples]
|
|
74
|
+
power = [s.power_draw_watts for s in samples]
|
|
75
|
+
|
|
76
|
+
return {
|
|
77
|
+
"avg_gpu_util_pct": _mean(gpu_util),
|
|
78
|
+
"std_gpu_util_pct": _stdev(gpu_util),
|
|
79
|
+
"mem_growth_mib_per_sample": _linear_slope_per_sample(mem_used),
|
|
80
|
+
"avg_power_watts": _mean(power),
|
|
81
|
+
"std_power_watts": _stdev(power),
|
|
82
|
+
"avg_process_count": _mean([float(s.process_count) for s in samples]),
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def classify(samples: list[TelemetrySample]) -> ClassificationResult:
|
|
87
|
+
if not samples:
|
|
88
|
+
raise ValueError("classify() requires at least one telemetry sample")
|
|
89
|
+
|
|
90
|
+
gpu_index = samples[0].gpu_index
|
|
91
|
+
window_seconds = samples[-1].timestamp - samples[0].timestamp if len(samples) > 1 else 0.0
|
|
92
|
+
features = extract_features(samples)
|
|
93
|
+
|
|
94
|
+
if (
|
|
95
|
+
features["avg_gpu_util_pct"] <= IDLE_GPU_UTIL_PCT
|
|
96
|
+
and features["avg_power_watts"] <= IDLE_POWER_WATTS
|
|
97
|
+
):
|
|
98
|
+
return ClassificationResult(
|
|
99
|
+
workload_type=WorkloadType.IDLE,
|
|
100
|
+
confidence=1.0 - (features["avg_gpu_util_pct"] / IDLE_GPU_UTIL_PCT) * 0.3,
|
|
101
|
+
gpu_index=gpu_index,
|
|
102
|
+
window_seconds=window_seconds,
|
|
103
|
+
sample_count=len(samples),
|
|
104
|
+
reasons=[
|
|
105
|
+
f"avg GPU utilization {features['avg_gpu_util_pct']:.1f}% "
|
|
106
|
+
f"<= idle threshold {IDLE_GPU_UTIL_PCT}%",
|
|
107
|
+
f"avg power draw {features['avg_power_watts']:.1f}W "
|
|
108
|
+
f"<= idle threshold {IDLE_POWER_WATTS}W",
|
|
109
|
+
],
|
|
110
|
+
features=features,
|
|
111
|
+
)
|
|
112
|
+
|
|
113
|
+
training_votes: list[str] = []
|
|
114
|
+
inference_votes: list[str] = []
|
|
115
|
+
|
|
116
|
+
if features["avg_gpu_util_pct"] >= TRAINING_MIN_AVG_GPU_UTIL_PCT:
|
|
117
|
+
training_votes.append(
|
|
118
|
+
f"avg GPU utilization {features['avg_gpu_util_pct']:.1f}% "
|
|
119
|
+
f">= training threshold {TRAINING_MIN_AVG_GPU_UTIL_PCT}%"
|
|
120
|
+
)
|
|
121
|
+
if features["std_gpu_util_pct"] <= TRAINING_MAX_GPU_UTIL_STD_PCT:
|
|
122
|
+
training_votes.append(
|
|
123
|
+
f"low GPU utilization variance (std={features['std_gpu_util_pct']:.1f}) "
|
|
124
|
+
f"<= training ceiling {TRAINING_MAX_GPU_UTIL_STD_PCT}"
|
|
125
|
+
)
|
|
126
|
+
if features["mem_growth_mib_per_sample"] >= TRAINING_MIN_MEM_GROWTH_MIB_PER_SAMPLE:
|
|
127
|
+
training_votes.append(
|
|
128
|
+
f"memory growing {features['mem_growth_mib_per_sample']:.1f} MiB/sample "
|
|
129
|
+
f">= training threshold {TRAINING_MIN_MEM_GROWTH_MIB_PER_SAMPLE}"
|
|
130
|
+
)
|
|
131
|
+
if features["std_power_watts"] <= TRAINING_MAX_POWER_STD_WATTS:
|
|
132
|
+
training_votes.append(
|
|
133
|
+
f"low power-draw variance (std={features['std_power_watts']:.1f}W) "
|
|
134
|
+
f"<= training ceiling {TRAINING_MAX_POWER_STD_WATTS}W"
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
if features["avg_gpu_util_pct"] <= INFERENCE_MAX_AVG_GPU_UTIL_PCT:
|
|
138
|
+
inference_votes.append(
|
|
139
|
+
f"avg GPU utilization {features['avg_gpu_util_pct']:.1f}% "
|
|
140
|
+
f"<= inference ceiling {INFERENCE_MAX_AVG_GPU_UTIL_PCT}%"
|
|
141
|
+
)
|
|
142
|
+
if features["std_gpu_util_pct"] >= INFERENCE_MIN_GPU_UTIL_STD_PCT:
|
|
143
|
+
inference_votes.append(
|
|
144
|
+
f"bursty GPU utilization (std={features['std_gpu_util_pct']:.1f}) "
|
|
145
|
+
f">= inference threshold {INFERENCE_MIN_GPU_UTIL_STD_PCT}"
|
|
146
|
+
)
|
|
147
|
+
if features["mem_growth_mib_per_sample"] < TRAINING_MIN_MEM_GROWTH_MIB_PER_SAMPLE:
|
|
148
|
+
inference_votes.append(
|
|
149
|
+
f"flat memory usage ({features['mem_growth_mib_per_sample']:.1f} MiB/sample) "
|
|
150
|
+
f"< training threshold {TRAINING_MIN_MEM_GROWTH_MIB_PER_SAMPLE}"
|
|
151
|
+
)
|
|
152
|
+
if features["std_power_watts"] >= INFERENCE_MIN_POWER_STD_WATTS:
|
|
153
|
+
inference_votes.append(
|
|
154
|
+
f"bursty power draw (std={features['std_power_watts']:.1f}W) "
|
|
155
|
+
f">= inference threshold {INFERENCE_MIN_POWER_STD_WATTS}W"
|
|
156
|
+
)
|
|
157
|
+
|
|
158
|
+
total_votes = 4
|
|
159
|
+
if len(training_votes) > len(inference_votes):
|
|
160
|
+
workload_type = WorkloadType.TRAINING
|
|
161
|
+
reasons = training_votes
|
|
162
|
+
confidence = len(training_votes) / total_votes
|
|
163
|
+
elif len(inference_votes) > len(training_votes):
|
|
164
|
+
workload_type = WorkloadType.INFERENCE
|
|
165
|
+
reasons = inference_votes
|
|
166
|
+
confidence = len(inference_votes) / total_votes
|
|
167
|
+
else:
|
|
168
|
+
workload_type = WorkloadType.UNKNOWN
|
|
169
|
+
reasons = [
|
|
170
|
+
"training and inference signals tied "
|
|
171
|
+
f"({len(training_votes)} votes each) -- inconclusive"
|
|
172
|
+
]
|
|
173
|
+
confidence = 0.5
|
|
174
|
+
|
|
175
|
+
return ClassificationResult(
|
|
176
|
+
workload_type=workload_type,
|
|
177
|
+
confidence=confidence,
|
|
178
|
+
gpu_index=gpu_index,
|
|
179
|
+
window_seconds=window_seconds,
|
|
180
|
+
sample_count=len(samples),
|
|
181
|
+
reasons=reasons,
|
|
182
|
+
features=features,
|
|
183
|
+
)
|
workloadtruth/cli.py
ADDED
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""WorkloadTruth CLI entry point."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
import time
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import click
|
|
11
|
+
|
|
12
|
+
from workloadtruth import __version__
|
|
13
|
+
from workloadtruth.audit_log import append_entry, build_entry, last_hash, verify_chain
|
|
14
|
+
from workloadtruth.classifier.rules import classify
|
|
15
|
+
from workloadtruth.telemetry import get_backend
|
|
16
|
+
from workloadtruth.types import ClassificationResult
|
|
17
|
+
|
|
18
|
+
DEFAULT_LOG_PATH = Path("workloadtruth.log.jsonl")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _echo_result(result: ClassificationResult, as_json: bool) -> None:
|
|
22
|
+
if as_json:
|
|
23
|
+
click.echo(json.dumps(result.to_dict(), indent=2))
|
|
24
|
+
return
|
|
25
|
+
click.echo(f"workload_type : {result.workload_type.value}")
|
|
26
|
+
click.echo(f"confidence : {result.confidence:.2f}")
|
|
27
|
+
click.echo(f"gpu_index : {result.gpu_index}")
|
|
28
|
+
click.echo(f"samples : {result.sample_count} over {result.window_seconds:.1f}s")
|
|
29
|
+
click.echo("reasons:")
|
|
30
|
+
for reason in result.reasons:
|
|
31
|
+
click.echo(f" - {reason}")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@click.group()
|
|
35
|
+
@click.version_option(version=__version__, prog_name="workloadtruth")
|
|
36
|
+
def main() -> None:
|
|
37
|
+
"""Classify a GPU workload as INFERENCE, TRAINING, or IDLE from telemetry alone."""
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@main.command(name="classify")
|
|
41
|
+
@click.option(
|
|
42
|
+
"--backend",
|
|
43
|
+
type=click.Choice(["synthetic", "nvml"]),
|
|
44
|
+
default="nvml",
|
|
45
|
+
show_default=True,
|
|
46
|
+
help="Telemetry source. 'synthetic' needs no GPU -- useful for trying the CLI.",
|
|
47
|
+
)
|
|
48
|
+
@click.option("--profile", default="training", help="Synthetic backend profile (synthetic only).")
|
|
49
|
+
@click.option("--gpu-index", default=0, show_default=True, type=int)
|
|
50
|
+
@click.option("--samples", default=10, show_default=True, type=int, help="Samples to collect.")
|
|
51
|
+
@click.option(
|
|
52
|
+
"--interval", default=1.0, show_default=True, type=float, help="Seconds between samples."
|
|
53
|
+
)
|
|
54
|
+
@click.option(
|
|
55
|
+
"--experimental", is_flag=True, help="Use the experimental ML classifier (not yet available)."
|
|
56
|
+
)
|
|
57
|
+
@click.option("--json", "as_json", is_flag=True, help="Machine-readable JSON output.")
|
|
58
|
+
def classify_cmd(
|
|
59
|
+
backend: str,
|
|
60
|
+
profile: str,
|
|
61
|
+
gpu_index: int,
|
|
62
|
+
samples: int,
|
|
63
|
+
interval: float,
|
|
64
|
+
experimental: bool,
|
|
65
|
+
as_json: bool,
|
|
66
|
+
) -> None:
|
|
67
|
+
"""One-shot classification of the current GPU workload."""
|
|
68
|
+
if experimental:
|
|
69
|
+
from workloadtruth.classifier.experimental import classify_experimental
|
|
70
|
+
|
|
71
|
+
classify_experimental()
|
|
72
|
+
return
|
|
73
|
+
|
|
74
|
+
backend_kwargs = {"profile": profile} if backend == "synthetic" else {}
|
|
75
|
+
tb = get_backend(backend, **backend_kwargs)
|
|
76
|
+
try:
|
|
77
|
+
collected = list(tb.sample_window(gpu_index, samples, interval))
|
|
78
|
+
finally:
|
|
79
|
+
tb.close()
|
|
80
|
+
|
|
81
|
+
result = classify(collected)
|
|
82
|
+
_echo_result(result, as_json)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@main.command()
|
|
86
|
+
@click.option(
|
|
87
|
+
"--backend",
|
|
88
|
+
type=click.Choice(["synthetic", "nvml"]),
|
|
89
|
+
default="nvml",
|
|
90
|
+
show_default=True,
|
|
91
|
+
)
|
|
92
|
+
@click.option("--profile", default="training", help="Synthetic backend profile (synthetic only).")
|
|
93
|
+
@click.option("--gpu-index", default=0, show_default=True, type=int)
|
|
94
|
+
@click.option(
|
|
95
|
+
"--interval", default=1.0, show_default=True, type=float, help="Seconds between samples."
|
|
96
|
+
)
|
|
97
|
+
@click.option(
|
|
98
|
+
"--window", default=10, show_default=True, type=int, help="Samples per classification window."
|
|
99
|
+
)
|
|
100
|
+
@click.option(
|
|
101
|
+
"--iterations",
|
|
102
|
+
default=0,
|
|
103
|
+
type=int,
|
|
104
|
+
help="Stop after N classification windows (0 = run forever).",
|
|
105
|
+
)
|
|
106
|
+
@click.option(
|
|
107
|
+
"--log-file",
|
|
108
|
+
default=str(DEFAULT_LOG_PATH),
|
|
109
|
+
show_default=True,
|
|
110
|
+
type=click.Path(path_type=Path),
|
|
111
|
+
)
|
|
112
|
+
@click.option("--json", "as_json", is_flag=True, help="Machine-readable JSON output per window.")
|
|
113
|
+
def watch(
|
|
114
|
+
backend: str,
|
|
115
|
+
profile: str,
|
|
116
|
+
gpu_index: int,
|
|
117
|
+
interval: float,
|
|
118
|
+
window: int,
|
|
119
|
+
iterations: int,
|
|
120
|
+
log_file: Path,
|
|
121
|
+
as_json: bool,
|
|
122
|
+
) -> None:
|
|
123
|
+
"""Continuously classify and append hash-chained entries to the audit log."""
|
|
124
|
+
backend_kwargs = {"profile": profile} if backend == "synthetic" else {}
|
|
125
|
+
tb = get_backend(backend, **backend_kwargs)
|
|
126
|
+
iteration = 0
|
|
127
|
+
try:
|
|
128
|
+
while True:
|
|
129
|
+
collected = list(tb.sample_window(gpu_index, window, interval))
|
|
130
|
+
result = classify(collected)
|
|
131
|
+
prev = last_hash(log_file)
|
|
132
|
+
entry = build_entry(
|
|
133
|
+
timestamp=time.time(),
|
|
134
|
+
gpu_index=result.gpu_index,
|
|
135
|
+
workload_type=result.workload_type.value,
|
|
136
|
+
confidence=result.confidence,
|
|
137
|
+
reasons=result.reasons,
|
|
138
|
+
features=result.features,
|
|
139
|
+
prev_hash=prev,
|
|
140
|
+
)
|
|
141
|
+
append_entry(log_file, entry)
|
|
142
|
+
_echo_result(result, as_json)
|
|
143
|
+
iteration += 1
|
|
144
|
+
if iterations and iteration >= iterations:
|
|
145
|
+
break
|
|
146
|
+
finally:
|
|
147
|
+
tb.close()
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@main.command()
|
|
151
|
+
@click.option(
|
|
152
|
+
"--trials", default=50, show_default=True, type=int, help="Trials per profile/evasion cell."
|
|
153
|
+
)
|
|
154
|
+
@click.option(
|
|
155
|
+
"--window", default=30, show_default=True, type=int, help="Samples per classification window."
|
|
156
|
+
)
|
|
157
|
+
@click.option("--json", "as_json", is_flag=True)
|
|
158
|
+
def benchmark(trials: int, window: int, as_json: bool) -> None:
|
|
159
|
+
"""Run the evasion-robustness benchmark against synthetic telemetry."""
|
|
160
|
+
from workloadtruth.benchmark import run_benchmark
|
|
161
|
+
|
|
162
|
+
report = run_benchmark(trials_per_cell=trials, window_size=window)
|
|
163
|
+
if as_json:
|
|
164
|
+
click.echo(json.dumps(report.to_dict(), indent=2))
|
|
165
|
+
return
|
|
166
|
+
|
|
167
|
+
click.echo("WorkloadTruth benchmark (synthetic telemetry -- see --json 'note' field)")
|
|
168
|
+
click.echo(f"clean accuracy : {report.clean_accuracy:.1%}")
|
|
169
|
+
click.echo(f"evasion accuracy : {report.evasion_accuracy:.1%}")
|
|
170
|
+
click.echo("")
|
|
171
|
+
click.echo(f"{'profile':<12}{'evasion':<10}{'accuracy':<10}trials")
|
|
172
|
+
for cell in report.cells:
|
|
173
|
+
click.echo(f"{cell.profile:<12}{str(cell.evasion):<10}{cell.accuracy:<10.1%}{cell.trials}")
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@main.command(name="verify-log")
|
|
177
|
+
@click.option(
|
|
178
|
+
"--log-file",
|
|
179
|
+
default=str(DEFAULT_LOG_PATH),
|
|
180
|
+
show_default=True,
|
|
181
|
+
type=click.Path(path_type=Path),
|
|
182
|
+
)
|
|
183
|
+
@click.option("--json", "as_json", is_flag=True)
|
|
184
|
+
def verify_log(log_file: Path, as_json: bool) -> None:
|
|
185
|
+
"""Verify the hash chain of a local audit log has not been tampered with."""
|
|
186
|
+
is_valid, message, count = verify_chain(log_file)
|
|
187
|
+
if as_json:
|
|
188
|
+
click.echo(json.dumps({"valid": is_valid, "message": message, "entries": count}, indent=2))
|
|
189
|
+
else:
|
|
190
|
+
status = "OK" if is_valid else "TAMPERED"
|
|
191
|
+
click.echo(f"[{status}] {message}")
|
|
192
|
+
if not is_valid:
|
|
193
|
+
sys.exit(1)
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
@main.command()
|
|
197
|
+
@click.option(
|
|
198
|
+
"--backend",
|
|
199
|
+
type=click.Choice(["synthetic", "nvml"]),
|
|
200
|
+
default="nvml",
|
|
201
|
+
show_default=True,
|
|
202
|
+
)
|
|
203
|
+
def mcp(backend: str) -> None:
|
|
204
|
+
"""Start an MCP server exposing classify/benchmark/verify-log as agent tools."""
|
|
205
|
+
try:
|
|
206
|
+
from workloadtruth.mcp_server import run_server
|
|
207
|
+
except ImportError:
|
|
208
|
+
click.echo(
|
|
209
|
+
'The MCP server requires the "mcp" extra: pip install "workloadtruth-cli[mcp]"',
|
|
210
|
+
err=True,
|
|
211
|
+
)
|
|
212
|
+
sys.exit(1)
|
|
213
|
+
run_server(default_backend=backend)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
if __name__ == "__main__":
|
|
217
|
+
main()
|