fleet-sensor-baseline 0.2.0__tar.gz → 0.2.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/PKG-INFO +108 -13
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/README.md +107 -12
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/docs/formats.md +136 -17
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/docs/upstream-asks.md +5 -1
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/pyproject.toml +5 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/__init__.py +1 -1
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/baseline.py +2 -2
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/cli.py +95 -2
- fleet_sensor_baseline-0.2.2/src/fleet_sensor_baseline/compare.py +244 -0
- fleet_sensor_baseline-0.2.2/src/fleet_sensor_baseline/for_referee.py +199 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/formats.py +44 -2
- fleet_sensor_baseline-0.2.2/tests/test_compare.py +236 -0
- fleet_sensor_baseline-0.2.2/tests/test_for_referee.py +216 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_formats.py +196 -2
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_seam.py +202 -1
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/.githooks/commit-msg +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/.githooks/pre-commit +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/.github/workflows/checks.yml +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/.gitignore +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/CONTRIBUTING.md +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/LICENSE +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/NOTICE +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/SECURITY.md +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/collect/__init__.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/collect/backends/__init__.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/collect/backends/mock.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/collect/backends/subprocess_backend.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/collect/collector.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/drift.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/exits.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/outliers.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/report.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/store.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/verdict.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/walk.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/conftest.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_baseline.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_boundary.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_collector.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_etag_collect.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_exits.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_hygiene.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_readme.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_readme_counts.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_referee_floor.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_scenarios.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tests/test_store.py +0 -0
- {fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/tools/hygiene_check.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: fleet-sensor-baseline
|
|
3
|
-
Version: 0.2.
|
|
3
|
+
Version: 0.2.2
|
|
4
4
|
Summary: Which units differ from their cohort, and what changed across time.
|
|
5
5
|
Project-URL: Homepage, https://github.com/james-sheen/fleet-sensor-baseline
|
|
6
6
|
Project-URL: Source, https://github.com/james-sheen/fleet-sensor-baseline
|
|
@@ -25,9 +25,26 @@ Description-Content-Type: text/markdown
|
|
|
25
25
|
list of machines and the history of captures, and answers the two questions the
|
|
26
26
|
referee cannot.
|
|
27
27
|
|
|
28
|
-
**Released — 0.2.
|
|
28
|
+
**Released — 0.2.2**, tagged `v0.2.2`, Apache-2.0, on PyPI as
|
|
29
29
|
`fleet-sensor-baseline`.
|
|
30
30
|
|
|
31
|
+
**0.2.2 adds `compare`, the threshold audit across time.** `drift` reads records
|
|
32
|
+
and never opens a walk, so a threshold edited on a sensor that stayed present was
|
|
33
|
+
invisible to it — and invisible to `collect --etag-cache` too, which is why a
|
|
34
|
+
skip record says it proves membership. `compare` resolves the newest capture at
|
|
35
|
+
or before each of two times, per surface, and hands both to the referee's
|
|
36
|
+
`regression`. A record filed from a skip is refused as an input: it reuses the
|
|
37
|
+
earlier payload, so comparing against one would judge a file against itself and
|
|
38
|
+
report no drift.
|
|
39
|
+
|
|
40
|
+
**0.2.1 closes the seam with the referee, and fixes a validator that could
|
|
41
|
+
not check the one file it most needed to.** `baseline --for-referee` writes the
|
|
42
|
+
referee's declaration-source format, which both projects described and neither
|
|
43
|
+
produced; it comes out as a candidate nobody has signed, and it does not declare
|
|
44
|
+
the sensors the cohort disagreed about. Separately, `validate` refused every
|
|
45
|
+
`targets/2` file while naming `targets/2` among the formats it read — so the
|
|
46
|
+
format that exists to carry `pin_sha256` was the one the checker would not check.
|
|
47
|
+
|
|
31
48
|
**0.2.0 is a compatibility break, and it changes an answer.** The presence
|
|
32
49
|
threshold was a single line, so a sensor either belonged to the baseline or was
|
|
33
50
|
foreign to it — and at rack scale the majority won. On 24 trays where 2 lost a
|
|
@@ -64,7 +81,9 @@ pip install fleet-sensor-baseline
|
|
|
64
81
|
|
|
65
82
|
The core has **no dependencies**. Ingest, baseline, outliers, drift, verdict and
|
|
66
83
|
validate are JSON and arithmetic, so the vertical and horizontal axes run on a jump
|
|
67
|
-
host with nothing provisioned.
|
|
84
|
+
host with nothing provisioned. `collect` and `compare` both need the referee on
|
|
85
|
+
PATH -- one to walk a BMC, the other to judge two stored walks -- and the extra
|
|
86
|
+
is named for the first only because renaming it would break every install line:
|
|
68
87
|
|
|
69
88
|
```
|
|
70
89
|
pip install "fleet-sensor-baseline[collect]"
|
|
@@ -87,14 +106,56 @@ fleet-sensor-baseline outliers --store fleet-store --baseline baseline.json
|
|
|
87
106
|
# the fleet run: every expected unit must have reported
|
|
88
107
|
fleet-sensor-baseline verdict --store fleet-store --expect-units racks.txt
|
|
89
108
|
|
|
109
|
+
# did a threshold move between two stored walks? ask the referee
|
|
110
|
+
fleet-sensor-baseline compare --store fleet-store --unit h-0042 \
|
|
111
|
+
--before 2026-08-01T00:00:00Z --after 2026-08-20T00:00:00Z
|
|
112
|
+
|
|
90
113
|
# check any artifact against the format it declares
|
|
91
114
|
fleet-sensor-baseline validate baseline.json
|
|
92
115
|
|
|
116
|
+
# hand the cohort's baseline back to the referee, as a candidate
|
|
117
|
+
fleet-sensor-baseline baseline --store fleet-store --model GB200-NVL-tray \
|
|
118
|
+
--out baseline.json --for-referee declaration.json
|
|
119
|
+
|
|
93
120
|
# walk a rack of BMCs yourself, serially, and file what comes back
|
|
94
121
|
fleet-sensor-baseline collect --targets rack-17.json --store fleet-store \
|
|
95
122
|
--collector-id rack-17
|
|
96
123
|
```
|
|
97
124
|
|
|
125
|
+
### Handing a baseline back to the referee
|
|
126
|
+
|
|
127
|
+
`--for-referee PATH` writes the cohort's baseline a second time, in
|
|
128
|
+
`bmc-sensor-audit/fleet-baseline/1` — the referee's own declaration-source
|
|
129
|
+
format, for `coverage --declaration`. It is the third source in that tool's
|
|
130
|
+
precedence, under entity-manager and under a reviewed `pdr/1`.
|
|
131
|
+
|
|
132
|
+
**It comes out as a candidate.** `"reviewed": null`, so the referee refuses to
|
|
133
|
+
consume it until a person adds their name and a date. The conversion is not the
|
|
134
|
+
review: a baseline derived from a fleet of unprovisioned boards is an empty
|
|
135
|
+
declaration that reads healthy against every other unprovisioned board, and at
|
|
136
|
+
fleet scale consensus makes that answer look corroborated. A marker this tool
|
|
137
|
+
could write is a marker nobody put their name to, so there is no flag to write
|
|
138
|
+
one.
|
|
139
|
+
|
|
140
|
+
**The sensors the cohort disagreed about are not declared.** That band exists
|
|
141
|
+
because a proportion is a coarse instrument at rack scale; the referee's format
|
|
142
|
+
has two states, not three, and declaring a divergent sensor would expect it of
|
|
143
|
+
every unit while 8 percent of the fleet does not have it — charging each of
|
|
144
|
+
those units a finding for a fact about the cohort. That is the 0.1.x inversion,
|
|
145
|
+
reached through a different door. They are written into the file under
|
|
146
|
+
`divergent_not_declared`, which the referee ignores by its own `/1` rule, and
|
|
147
|
+
named on stderr at export: the reviewer is the reader who needs them.
|
|
148
|
+
|
|
149
|
+
The cohort must be scoped to one model — `--model`, or `--for-referee-platform`
|
|
150
|
+
to name it directly — because the referee requires a platform on every
|
|
151
|
+
declaration and a cohort spanning models has no single answer.
|
|
152
|
+
|
|
153
|
+
`tests/test_seam.py::TestTheExportedDeclarationIsWhatTheRefereeReads` runs the
|
|
154
|
+
real `bmc-sensor-audit` against a file this emits: refused as a candidate,
|
|
155
|
+
consumed once reviewed, and the unit that lost the divergent sensor coming back
|
|
156
|
+
clean — with the counterfactual that declares it anyway and watches the same
|
|
157
|
+
unit get charged.
|
|
158
|
+
|
|
98
159
|
### Asking before walking
|
|
99
160
|
|
|
100
161
|
```
|
|
@@ -169,8 +230,10 @@ renders incompleteness as silence renders a dead collector as a clean rack.
|
|
|
169
230
|
is never the source of judgment. Silence cannot impersonate a pass.
|
|
170
231
|
2. **Never metric values.** This audits presence and configuration, not readings.
|
|
171
232
|
3. **Never majority-truth over declarations.** A fleet-derived baseline is an
|
|
172
|
-
*additional, labeled, downgraded* declaration source
|
|
173
|
-
|
|
233
|
+
*additional, labeled, downgraded* declaration source — reachable through
|
|
234
|
+
`baseline --for-referee`, which writes the referee's own format as a
|
|
235
|
+
candidate nobody has signed yet. Wherever a manufacturer declaration exists,
|
|
236
|
+
it wins.
|
|
174
237
|
4. **Never identity inside the referee's artifacts.** The walk payload carries no
|
|
175
238
|
`unit_key`. Identity binds here.
|
|
176
239
|
|
|
@@ -210,9 +273,34 @@ key the file declares rather than on a shape guessed from the fields present.
|
|
|
210
273
|
|
|
211
274
|
See [docs/formats.md](docs/formats.md).
|
|
212
275
|
|
|
213
|
-
##
|
|
276
|
+
## Comparing across time
|
|
214
277
|
|
|
215
|
-
|
|
278
|
+
`drift` answers what changed in the RECORDS -- firmware, the sensor set, which
|
|
279
|
+
units reported. It does not open the walks. A threshold edited on a sensor that
|
|
280
|
+
stayed present is invisible to it, and invisible to `collect --etag-cache` too:
|
|
281
|
+
a collection ETag moves when membership changes, so a skip record says in as
|
|
282
|
+
many words that it proves `membership`, and the docs say membership is not
|
|
283
|
+
enough for a threshold audit.
|
|
284
|
+
|
|
285
|
+
`compare` is the missing half, and it is selection rather than judgment. It
|
|
286
|
+
resolves the newest capture at or before each of two times, pulls both payloads
|
|
287
|
+
out of the store, and hands them to `bmc-sensor-audit regression`, whose job
|
|
288
|
+
this is. `--strict-fields` and `--aggregation-prefix` pass straight through.
|
|
289
|
+
Exit codes come back in the family's vocabulary; the referee's refusals come
|
|
290
|
+
back in the referee's own words.
|
|
291
|
+
|
|
292
|
+
**A record filed from a skip is refused as an input, and that rule is the whole
|
|
293
|
+
point rather than tidiness.** A skip reuses the earlier capture's payload
|
|
294
|
+
digest, so comparing a walk against one resolves both ends to the same stored
|
|
295
|
+
object: the referee would be handed a file and itself, and would correctly
|
|
296
|
+
answer *no drift*. The question asked was whether a threshold moved. Force a
|
|
297
|
+
full walk for that surface, or pick a time when one was taken.
|
|
298
|
+
|
|
299
|
+
Pairing is surface to surface, per [a unit is a tuple](#a-unit-is-a-tuple), so a
|
|
300
|
+
machine answering on a host BMC and an HMC gets one comparison each rather than
|
|
301
|
+
one compared against the other.
|
|
302
|
+
|
|
303
|
+
## When the cohort disagrees with itself
|
|
216
304
|
|
|
217
305
|
A sensor is **expected** at or above `--present-threshold` (0.99), **foreign** at
|
|
218
306
|
or below `--absent-threshold` (0.01), and between the two the cohort simply
|
|
@@ -262,9 +350,16 @@ to surface; presence across the cohort is the union across a unit's surfaces.
|
|
|
262
350
|
## The boundary
|
|
263
351
|
|
|
264
352
|
This package **never imports `bmc_sensor_audit`**. It reads exit codes, stdout and
|
|
265
|
-
the files the tool writes. One module is exempt — `collect/backends/mock.py
|
|
266
|
-
|
|
267
|
-
|
|
353
|
+
the files the tool writes. One module is exempt — `collect/backends/mock.py` — and
|
|
354
|
+
the distinction is the design: it stands in for the thing being walked, not the
|
|
355
|
+
thing doing the walking.
|
|
356
|
+
|
|
357
|
+
It imports the referee's mock BMC *and* its `RedfishClient` and `walk_chassis`, so
|
|
358
|
+
the payloads it produces are real `walk/1` files written by the real walker rather
|
|
359
|
+
than approximations of them. The exemption is file-level and that is the actual
|
|
360
|
+
contract `tests/test_boundary.py` enforces; an earlier version of this sentence
|
|
361
|
+
named only the mock, which was narrower than the code and would have made a
|
|
362
|
+
correct file look like a violation.
|
|
268
363
|
|
|
269
364
|
`tests/test_boundary.py` asserts both halves by reading the source: that nothing
|
|
270
365
|
else imports it, and that the mock backend still does. Without the second, the
|
|
@@ -274,15 +369,15 @@ first would pass by finding nothing.
|
|
|
274
369
|
|
|
275
370
|
| | count |
|
|
276
371
|
|---|---|
|
|
277
|
-
| tests collected |
|
|
278
|
-
| of those, requiring `bmc-sensor-audit` |
|
|
372
|
+
| tests collected | 363 |
|
|
373
|
+
| of those, requiring `bmc-sensor-audit` | 40 |
|
|
279
374
|
|
|
280
375
|
**The predicate**: `pytest --collect-only` over the test files git tracks, and
|
|
281
376
|
the same again with `-m seam` for the second row. Collection rather than a pass
|
|
282
377
|
tally, because a skip count is true only on the machine that measured it —
|
|
283
378
|
`tests/test_readme_counts.py` derives both and fails if either drifts.
|
|
284
379
|
|
|
285
|
-
Run it dependency-free and the
|
|
380
|
+
Run it dependency-free and the 40 skip. Install the referee and they run.
|
|
286
381
|
|
|
287
382
|
**No pass/skip tally is quoted here on purpose.** The first version of this
|
|
288
383
|
section did, and both numbers were wrong within a day — not because tests
|
|
@@ -6,9 +6,26 @@
|
|
|
6
6
|
list of machines and the history of captures, and answers the two questions the
|
|
7
7
|
referee cannot.
|
|
8
8
|
|
|
9
|
-
**Released — 0.2.
|
|
9
|
+
**Released — 0.2.2**, tagged `v0.2.2`, Apache-2.0, on PyPI as
|
|
10
10
|
`fleet-sensor-baseline`.
|
|
11
11
|
|
|
12
|
+
**0.2.2 adds `compare`, the threshold audit across time.** `drift` reads records
|
|
13
|
+
and never opens a walk, so a threshold edited on a sensor that stayed present was
|
|
14
|
+
invisible to it — and invisible to `collect --etag-cache` too, which is why a
|
|
15
|
+
skip record says it proves membership. `compare` resolves the newest capture at
|
|
16
|
+
or before each of two times, per surface, and hands both to the referee's
|
|
17
|
+
`regression`. A record filed from a skip is refused as an input: it reuses the
|
|
18
|
+
earlier payload, so comparing against one would judge a file against itself and
|
|
19
|
+
report no drift.
|
|
20
|
+
|
|
21
|
+
**0.2.1 closes the seam with the referee, and fixes a validator that could
|
|
22
|
+
not check the one file it most needed to.** `baseline --for-referee` writes the
|
|
23
|
+
referee's declaration-source format, which both projects described and neither
|
|
24
|
+
produced; it comes out as a candidate nobody has signed, and it does not declare
|
|
25
|
+
the sensors the cohort disagreed about. Separately, `validate` refused every
|
|
26
|
+
`targets/2` file while naming `targets/2` among the formats it read — so the
|
|
27
|
+
format that exists to carry `pin_sha256` was the one the checker would not check.
|
|
28
|
+
|
|
12
29
|
**0.2.0 is a compatibility break, and it changes an answer.** The presence
|
|
13
30
|
threshold was a single line, so a sensor either belonged to the baseline or was
|
|
14
31
|
foreign to it — and at rack scale the majority won. On 24 trays where 2 lost a
|
|
@@ -45,7 +62,9 @@ pip install fleet-sensor-baseline
|
|
|
45
62
|
|
|
46
63
|
The core has **no dependencies**. Ingest, baseline, outliers, drift, verdict and
|
|
47
64
|
validate are JSON and arithmetic, so the vertical and horizontal axes run on a jump
|
|
48
|
-
host with nothing provisioned.
|
|
65
|
+
host with nothing provisioned. `collect` and `compare` both need the referee on
|
|
66
|
+
PATH -- one to walk a BMC, the other to judge two stored walks -- and the extra
|
|
67
|
+
is named for the first only because renaming it would break every install line:
|
|
49
68
|
|
|
50
69
|
```
|
|
51
70
|
pip install "fleet-sensor-baseline[collect]"
|
|
@@ -68,14 +87,56 @@ fleet-sensor-baseline outliers --store fleet-store --baseline baseline.json
|
|
|
68
87
|
# the fleet run: every expected unit must have reported
|
|
69
88
|
fleet-sensor-baseline verdict --store fleet-store --expect-units racks.txt
|
|
70
89
|
|
|
90
|
+
# did a threshold move between two stored walks? ask the referee
|
|
91
|
+
fleet-sensor-baseline compare --store fleet-store --unit h-0042 \
|
|
92
|
+
--before 2026-08-01T00:00:00Z --after 2026-08-20T00:00:00Z
|
|
93
|
+
|
|
71
94
|
# check any artifact against the format it declares
|
|
72
95
|
fleet-sensor-baseline validate baseline.json
|
|
73
96
|
|
|
97
|
+
# hand the cohort's baseline back to the referee, as a candidate
|
|
98
|
+
fleet-sensor-baseline baseline --store fleet-store --model GB200-NVL-tray \
|
|
99
|
+
--out baseline.json --for-referee declaration.json
|
|
100
|
+
|
|
74
101
|
# walk a rack of BMCs yourself, serially, and file what comes back
|
|
75
102
|
fleet-sensor-baseline collect --targets rack-17.json --store fleet-store \
|
|
76
103
|
--collector-id rack-17
|
|
77
104
|
```
|
|
78
105
|
|
|
106
|
+
### Handing a baseline back to the referee
|
|
107
|
+
|
|
108
|
+
`--for-referee PATH` writes the cohort's baseline a second time, in
|
|
109
|
+
`bmc-sensor-audit/fleet-baseline/1` — the referee's own declaration-source
|
|
110
|
+
format, for `coverage --declaration`. It is the third source in that tool's
|
|
111
|
+
precedence, under entity-manager and under a reviewed `pdr/1`.
|
|
112
|
+
|
|
113
|
+
**It comes out as a candidate.** `"reviewed": null`, so the referee refuses to
|
|
114
|
+
consume it until a person adds their name and a date. The conversion is not the
|
|
115
|
+
review: a baseline derived from a fleet of unprovisioned boards is an empty
|
|
116
|
+
declaration that reads healthy against every other unprovisioned board, and at
|
|
117
|
+
fleet scale consensus makes that answer look corroborated. A marker this tool
|
|
118
|
+
could write is a marker nobody put their name to, so there is no flag to write
|
|
119
|
+
one.
|
|
120
|
+
|
|
121
|
+
**The sensors the cohort disagreed about are not declared.** That band exists
|
|
122
|
+
because a proportion is a coarse instrument at rack scale; the referee's format
|
|
123
|
+
has two states, not three, and declaring a divergent sensor would expect it of
|
|
124
|
+
every unit while 8 percent of the fleet does not have it — charging each of
|
|
125
|
+
those units a finding for a fact about the cohort. That is the 0.1.x inversion,
|
|
126
|
+
reached through a different door. They are written into the file under
|
|
127
|
+
`divergent_not_declared`, which the referee ignores by its own `/1` rule, and
|
|
128
|
+
named on stderr at export: the reviewer is the reader who needs them.
|
|
129
|
+
|
|
130
|
+
The cohort must be scoped to one model — `--model`, or `--for-referee-platform`
|
|
131
|
+
to name it directly — because the referee requires a platform on every
|
|
132
|
+
declaration and a cohort spanning models has no single answer.
|
|
133
|
+
|
|
134
|
+
`tests/test_seam.py::TestTheExportedDeclarationIsWhatTheRefereeReads` runs the
|
|
135
|
+
real `bmc-sensor-audit` against a file this emits: refused as a candidate,
|
|
136
|
+
consumed once reviewed, and the unit that lost the divergent sensor coming back
|
|
137
|
+
clean — with the counterfactual that declares it anyway and watches the same
|
|
138
|
+
unit get charged.
|
|
139
|
+
|
|
79
140
|
### Asking before walking
|
|
80
141
|
|
|
81
142
|
```
|
|
@@ -150,8 +211,10 @@ renders incompleteness as silence renders a dead collector as a clean rack.
|
|
|
150
211
|
is never the source of judgment. Silence cannot impersonate a pass.
|
|
151
212
|
2. **Never metric values.** This audits presence and configuration, not readings.
|
|
152
213
|
3. **Never majority-truth over declarations.** A fleet-derived baseline is an
|
|
153
|
-
*additional, labeled, downgraded* declaration source
|
|
154
|
-
|
|
214
|
+
*additional, labeled, downgraded* declaration source — reachable through
|
|
215
|
+
`baseline --for-referee`, which writes the referee's own format as a
|
|
216
|
+
candidate nobody has signed yet. Wherever a manufacturer declaration exists,
|
|
217
|
+
it wins.
|
|
155
218
|
4. **Never identity inside the referee's artifacts.** The walk payload carries no
|
|
156
219
|
`unit_key`. Identity binds here.
|
|
157
220
|
|
|
@@ -191,9 +254,34 @@ key the file declares rather than on a shape guessed from the fields present.
|
|
|
191
254
|
|
|
192
255
|
See [docs/formats.md](docs/formats.md).
|
|
193
256
|
|
|
194
|
-
##
|
|
257
|
+
## Comparing across time
|
|
195
258
|
|
|
196
|
-
|
|
259
|
+
`drift` answers what changed in the RECORDS -- firmware, the sensor set, which
|
|
260
|
+
units reported. It does not open the walks. A threshold edited on a sensor that
|
|
261
|
+
stayed present is invisible to it, and invisible to `collect --etag-cache` too:
|
|
262
|
+
a collection ETag moves when membership changes, so a skip record says in as
|
|
263
|
+
many words that it proves `membership`, and the docs say membership is not
|
|
264
|
+
enough for a threshold audit.
|
|
265
|
+
|
|
266
|
+
`compare` is the missing half, and it is selection rather than judgment. It
|
|
267
|
+
resolves the newest capture at or before each of two times, pulls both payloads
|
|
268
|
+
out of the store, and hands them to `bmc-sensor-audit regression`, whose job
|
|
269
|
+
this is. `--strict-fields` and `--aggregation-prefix` pass straight through.
|
|
270
|
+
Exit codes come back in the family's vocabulary; the referee's refusals come
|
|
271
|
+
back in the referee's own words.
|
|
272
|
+
|
|
273
|
+
**A record filed from a skip is refused as an input, and that rule is the whole
|
|
274
|
+
point rather than tidiness.** A skip reuses the earlier capture's payload
|
|
275
|
+
digest, so comparing a walk against one resolves both ends to the same stored
|
|
276
|
+
object: the referee would be handed a file and itself, and would correctly
|
|
277
|
+
answer *no drift*. The question asked was whether a threshold moved. Force a
|
|
278
|
+
full walk for that surface, or pick a time when one was taken.
|
|
279
|
+
|
|
280
|
+
Pairing is surface to surface, per [a unit is a tuple](#a-unit-is-a-tuple), so a
|
|
281
|
+
machine answering on a host BMC and an HMC gets one comparison each rather than
|
|
282
|
+
one compared against the other.
|
|
283
|
+
|
|
284
|
+
## When the cohort disagrees with itself
|
|
197
285
|
|
|
198
286
|
A sensor is **expected** at or above `--present-threshold` (0.99), **foreign** at
|
|
199
287
|
or below `--absent-threshold` (0.01), and between the two the cohort simply
|
|
@@ -243,9 +331,16 @@ to surface; presence across the cohort is the union across a unit's surfaces.
|
|
|
243
331
|
## The boundary
|
|
244
332
|
|
|
245
333
|
This package **never imports `bmc_sensor_audit`**. It reads exit codes, stdout and
|
|
246
|
-
the files the tool writes. One module is exempt — `collect/backends/mock.py
|
|
247
|
-
|
|
248
|
-
|
|
334
|
+
the files the tool writes. One module is exempt — `collect/backends/mock.py` — and
|
|
335
|
+
the distinction is the design: it stands in for the thing being walked, not the
|
|
336
|
+
thing doing the walking.
|
|
337
|
+
|
|
338
|
+
It imports the referee's mock BMC *and* its `RedfishClient` and `walk_chassis`, so
|
|
339
|
+
the payloads it produces are real `walk/1` files written by the real walker rather
|
|
340
|
+
than approximations of them. The exemption is file-level and that is the actual
|
|
341
|
+
contract `tests/test_boundary.py` enforces; an earlier version of this sentence
|
|
342
|
+
named only the mock, which was narrower than the code and would have made a
|
|
343
|
+
correct file look like a violation.
|
|
249
344
|
|
|
250
345
|
`tests/test_boundary.py` asserts both halves by reading the source: that nothing
|
|
251
346
|
else imports it, and that the mock backend still does. Without the second, the
|
|
@@ -255,15 +350,15 @@ first would pass by finding nothing.
|
|
|
255
350
|
|
|
256
351
|
| | count |
|
|
257
352
|
|---|---|
|
|
258
|
-
| tests collected |
|
|
259
|
-
| of those, requiring `bmc-sensor-audit` |
|
|
353
|
+
| tests collected | 363 |
|
|
354
|
+
| of those, requiring `bmc-sensor-audit` | 40 |
|
|
260
355
|
|
|
261
356
|
**The predicate**: `pytest --collect-only` over the test files git tracks, and
|
|
262
357
|
the same again with `-m seam` for the second row. Collection rather than a pass
|
|
263
358
|
tally, because a skip count is true only on the machine that measured it —
|
|
264
359
|
`tests/test_readme_counts.py` derives both and fails if either drifts.
|
|
265
360
|
|
|
266
|
-
Run it dependency-free and the
|
|
361
|
+
Run it dependency-free and the 40 skip. Install the referee and they run.
|
|
267
362
|
|
|
268
363
|
**No pass/skip tally is quoted here on purpose.** The first version of this
|
|
269
364
|
section did, and both numbers were wrong within a day — not because tests
|
|
@@ -1,10 +1,30 @@
|
|
|
1
1
|
# Formats
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
**the person who receives the file is the one who needs to
|
|
5
|
-
plane ingesting thousands of records has to be able to
|
|
6
|
-
using the format's own words, not a shape it inferred
|
|
7
|
-
happened to arrive first.
|
|
3
|
+
Six this build reads, each versioned in its own key and each with a validator
|
|
4
|
+
that ships, because **the person who receives the file is the one who needs to
|
|
5
|
+
check it.** A control plane ingesting thousands of records has to be able to
|
|
6
|
+
refuse a malformed one using the format's own words, not a shape it inferred
|
|
7
|
+
from the files that happened to arrive first.
|
|
8
|
+
|
|
9
|
+
| format | is |
|
|
10
|
+
|---|---|
|
|
11
|
+
| `fleet-sensor-baseline/fleet-record/1` | one capture of one unit |
|
|
12
|
+
| `fleet-sensor-baseline/fleet-baseline/1` | **superseded; refused, not upgraded** |
|
|
13
|
+
| `fleet-sensor-baseline/fleet-baseline/2` | a derived declaration, carrying `divergent` |
|
|
14
|
+
| `fleet-sensor-baseline/summary/1` | a verdict, in the family's one vocabulary |
|
|
15
|
+
| `fleet-sensor-baseline/targets/1` | the collector's rack list |
|
|
16
|
+
| `fleet-sensor-baseline/targets/2` | the same, and may declare `pin_sha256` |
|
|
17
|
+
|
|
18
|
+
and one it **writes for somebody else to read**:
|
|
19
|
+
`bmc-sensor-audit/fleet-baseline/1`, produced by `baseline --for-referee`. That
|
|
20
|
+
one is not validated here; the tool that consumes it is the tool that checks it.
|
|
21
|
+
|
|
22
|
+
`tests/test_formats.py::TestThisDocumentNamesEveryFormat` derives this list from
|
|
23
|
+
the code. It exists because the list was hand-written and went stale: for two
|
|
24
|
+
releases this page opened *Four*, presented `fleet-baseline/1` as current, and
|
|
25
|
+
did not mention `targets/2` or the `divergent` band at all — while the README,
|
|
26
|
+
whose count IS pinned by a test, said six. Two records of one fact, and the
|
|
27
|
+
unguarded one drifted.
|
|
8
28
|
|
|
9
29
|
Every validator returns *problems* rather than raising, so a caller reports all
|
|
10
30
|
of them at once instead of one per run. Every one checks **malformation only**:
|
|
@@ -91,43 +111,95 @@ per identity, and position in the file decides, not the timestamp.
|
|
|
91
111
|
|
|
92
112
|
---
|
|
93
113
|
|
|
94
|
-
## `fleet-sensor-baseline/fleet-baseline/
|
|
114
|
+
## `fleet-sensor-baseline/fleet-baseline/2`
|
|
95
115
|
|
|
96
116
|
A derived declaration, labeled as such.
|
|
97
117
|
|
|
98
118
|
```json
|
|
99
119
|
{
|
|
100
|
-
"format": "fleet-sensor-baseline/fleet-baseline/
|
|
120
|
+
"format": "fleet-sensor-baseline/fleet-baseline/2",
|
|
101
121
|
"scope": {"model": "GB200-NVL-tray", "firmware_range": ">=1.4,<1.5"},
|
|
102
122
|
"derived": {"units": 1987, "present_threshold": 0.99,
|
|
123
|
+
"absent_threshold": 0.01,
|
|
103
124
|
"captured_between": ["2026-08-01T00:00:00Z",
|
|
104
125
|
"2026-08-21T00:00:00Z"]},
|
|
105
126
|
"sensors": [{"name": "Fan_CPU_1", "uri_suffix": "/Sensors/Fan_CPU_1",
|
|
106
127
|
"present_ratio": 0.9975}],
|
|
128
|
+
"divergent": [{"name": "Fan_CPU_2", "present_ratio": 0.916667,
|
|
129
|
+
"present_on": 22, "of": 24}],
|
|
107
130
|
"provenance": "fleet-derived",
|
|
108
131
|
"notice": "This baseline was derived from the fleet, not declared by a manufacturer. It cannot see an absence the whole cohort shares."
|
|
109
132
|
}
|
|
110
133
|
```
|
|
111
134
|
|
|
112
135
|
**The `notice` sentence is part of the format.** Every consumer that judges
|
|
113
|
-
against a
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
correctly.
|
|
136
|
+
against a baseline prints it verbatim, and the validator refuses a baseline that
|
|
137
|
+
lost it or reworded it. That is not pedantry about prose: a baseline without the
|
|
138
|
+
sentence validates as a manufacturer declaration everywhere downstream, and the
|
|
139
|
+
loss is invisible because every other field still reads correctly.
|
|
118
140
|
|
|
119
|
-
`provenance` carries the same fact in a machine-readable field, and every
|
|
120
|
-
|
|
141
|
+
`provenance` carries the same fact in a machine-readable field, and every summary
|
|
142
|
+
this layer emits records which kind of truth decided it.
|
|
121
143
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
144
|
+
### `divergent` — the third state, and why the format had to move
|
|
145
|
+
|
|
146
|
+
A sensor is **expected** at or above `present_threshold`, **foreign** at or below
|
|
147
|
+
`absent_threshold`, and between the two the cohort **disagrees with itself**. That
|
|
148
|
+
middle band is reported once, against the cohort, naming the minority, and charged
|
|
149
|
+
to no unit.
|
|
150
|
+
|
|
151
|
+
Two states were not enough, and the failure was not a miss — it was an inversion.
|
|
152
|
+
With 22 of 24 trays carrying a sensor the ratio is 0.9167, below the 0.99 default,
|
|
153
|
+
so the sensor left the baseline entirely and every tray that **had** it was
|
|
154
|
+
reported as carrying something unexpected while the two that had lost it came back
|
|
155
|
+
clean. A proportion is a coarse instrument at rack scale: 0.99 of 24 is 23.76, so
|
|
156
|
+
one deviant unit crosses it, and `--floor` admits cohorts from 20.
|
|
157
|
+
|
|
158
|
+
**`divergent` is why this is `/2` and not a new key on `/1`.** An unknown format
|
|
159
|
+
is refused; an unknown key is dropped. A reader that predated the key would ignore
|
|
160
|
+
it and charge every divergent sensor to the units that have it — the same
|
|
161
|
+
inversion, arrived at silently, which is the one way it could get worse.
|
|
125
162
|
|
|
126
163
|
`uri_suffix` is written **only when every contributing unit agrees**. Matching is
|
|
127
164
|
by name regardless; a baseline asserting one URI while the cohort reports several
|
|
128
165
|
would be asserting something no unit said, and advisory metadata that is wrong is
|
|
129
166
|
worse than absent.
|
|
130
167
|
|
|
168
|
+
**A baseline refuses to derive from fewer than 20 units** (`--floor N` to change
|
|
169
|
+
it, explicitly). A cohort of nine is an anecdote wearing a format key: at that size
|
|
170
|
+
one unlucky machine moves every ratio past any threshold worth setting.
|
|
171
|
+
|
|
172
|
+
## `fleet-sensor-baseline/fleet-baseline/1`
|
|
173
|
+
|
|
174
|
+
**Superseded. Refused, and deliberately not upgraded.**
|
|
175
|
+
|
|
176
|
+
A `/1` recorded only the sensors that cleared the presence threshold and silently
|
|
177
|
+
forgot every sensor the cohort disagreed about. `validate` refuses one and says
|
|
178
|
+
why; nothing reads it. There is no migration path and there cannot be one: the
|
|
179
|
+
information needed to judge was discarded at derivation, so a `/1` cannot be told
|
|
180
|
+
apart from a cohort that genuinely never had those sensors. **Derive again** — the
|
|
181
|
+
records it came from are still in the store, which is what an append-only store is
|
|
182
|
+
for.
|
|
183
|
+
|
|
184
|
+
## `bmc-sensor-audit/fleet-baseline/1` — written, not read
|
|
185
|
+
|
|
186
|
+
`baseline --for-referee PATH` writes the referee's declaration-source format, so a
|
|
187
|
+
cohort baseline can be used as the third source in `bmc-sensor-audit coverage`.
|
|
188
|
+
It is specified in that repository's `docs/declaration-sources.md`, not here, and
|
|
189
|
+
this build has no validator for it — the consumer is the checker.
|
|
190
|
+
|
|
191
|
+
Two properties of the conversion belong on this page because they are decisions
|
|
192
|
+
about what a format may lose:
|
|
193
|
+
|
|
194
|
+
- **It is a candidate.** `"reviewed": null`, so the referee refuses it until a
|
|
195
|
+
person adds their name and a date. The conversion is not the review.
|
|
196
|
+
- **The divergent band does not cross.** The referee's format has two states, not
|
|
197
|
+
three, because it judges one machine against a list of what that machine should
|
|
198
|
+
have. Divergent sensors are carried under `divergent_not_declared`, a key that
|
|
199
|
+
reader ignores by its own `/1` rule, and named on stderr at export. Declaring
|
|
200
|
+
them would rebuild the inversion one tool over; dropping them without a word
|
|
201
|
+
would rebuild `/1`.
|
|
202
|
+
|
|
131
203
|
---
|
|
132
204
|
|
|
133
205
|
## `fleet-sensor-baseline/summary/1`
|
|
@@ -174,6 +246,53 @@ satellites is fine — that is one machine with two surfaces.
|
|
|
174
246
|
|
|
175
247
|
---
|
|
176
248
|
|
|
249
|
+
## `fleet-sensor-baseline/targets/2`
|
|
250
|
+
|
|
251
|
+
The same rack list, and the only one that may declare a certificate pin.
|
|
252
|
+
|
|
253
|
+
```json
|
|
254
|
+
{
|
|
255
|
+
"format": "fleet-sensor-baseline/targets/2",
|
|
256
|
+
"targets": [
|
|
257
|
+
{"unit_key": "h-0042", "base_url": "https://192.0.2.1",
|
|
258
|
+
"pin_sha256": "AB:CD:...:EF", "password_env": "BMC_PASS_RACK17"}
|
|
259
|
+
]
|
|
260
|
+
}
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
**A pin is an expectation, not a credential.** A SHA-256 fingerprint of a public
|
|
264
|
+
certificate is public — anyone who can connect can compute it — so it belongs in
|
|
265
|
+
the file, where a change to it shows up in a review diff. A CA bundle is a path on
|
|
266
|
+
one operator's disk and is `collect --cafile PATH` instead.
|
|
267
|
+
|
|
268
|
+
**Version 2 exists for one reason: a declaration an older reader would IGNORE.**
|
|
269
|
+
`pin_sha256` says *require exactly this certificate*. A reader that does not know
|
|
270
|
+
the key would drop it and connect unpinned — a security expectation stated by an
|
|
271
|
+
operator and met with silence. An older build refuses an unknown format outright,
|
|
272
|
+
so bumping is what turns a silent downgrade into a refusal. `targets/1` stays
|
|
273
|
+
valid and stays the right choice for a rack list that pins nothing.
|
|
274
|
+
|
|
275
|
+
The validator refuses, in the file, before any connection is made:
|
|
276
|
+
|
|
277
|
+
- `pin_sha256` in a `targets/1`, naming `targets/2` as what to write instead;
|
|
278
|
+
- `pin_sha256` beside a `base_url` that is not `https` — nothing would verify the
|
|
279
|
+
connection. The referee refuses this too, from 0.1.4, but a rack list is
|
|
280
|
+
reviewed long before it is run, and catching it in the file is catching it where
|
|
281
|
+
somebody is looking;
|
|
282
|
+
- a fingerprint that is not 64 hex digits, colons optional;
|
|
283
|
+
- `pin_sha256` beside `insecure` — two answers to one question, and whichever won
|
|
284
|
+
would be a guess about which the operator meant.
|
|
285
|
+
|
|
286
|
+
**This section is here because `validate` could not check any of it.** `targets/2`
|
|
287
|
+
shipped in the format list and not in the dispatch table, so every `targets/2` file
|
|
288
|
+
was refused — with a message that named `targets/2` among the formats it had just
|
|
289
|
+
said it read. The validator had handled both versions from the day `/2` landed;
|
|
290
|
+
only the wiring was missing, and every test called the validator directly rather
|
|
291
|
+
than through `validate`. The format that exists to carry a security expectation was
|
|
292
|
+
the one the checker would not check.
|
|
293
|
+
|
|
294
|
+
---
|
|
295
|
+
|
|
177
296
|
## Content handles
|
|
178
297
|
|
|
179
298
|
`payload_digest` is `sha256:` and the hex digest of the **file bytes**, identical
|
|
@@ -193,7 +193,11 @@ than absorbed:
|
|
|
193
193
|
matches `\bsensor set unchanged\b`. That works and is the wrong kind of
|
|
194
194
|
contract: prose can be reworded without it reading as a breaking change, while
|
|
195
195
|
`walk/1`'s format string carries a written stability statement. Filed as
|
|
196
|
-
[#6](https://github.com/james-sheen/bmc-sensor-audit/issues/6), **
|
|
196
|
+
[#6](https://github.com/james-sheen/bmc-sensor-audit/issues/6), **closed in
|
|
197
|
+
`bmc-sensor-audit` 0.1.3 and consumed here** -- Section 5 below is the same
|
|
198
|
+
ask, written up. This bullet said *open* for two days after the fix shipped,
|
|
199
|
+
while the table above and Section 5 both said consumed: one fact recorded in
|
|
200
|
+
three places, and the copy nobody re-read was the stale one.
|
|
197
201
|
- A test assertion written expecting the old digest-dedup behaviour FAILED,
|
|
198
202
|
which is how the *store collapses a homogeneous fleet* claim was finally
|
|
199
203
|
measured and found false. See `store.py`.
|
|
@@ -70,6 +70,11 @@ dependencies = []
|
|
|
70
70
|
# this floor rather than merely old, and `preflight()` can refuse instead of
|
|
71
71
|
# saying it could not tell. That is the whole reason this floor moved: the check
|
|
72
72
|
# was written before the thing it checks existed.
|
|
73
|
+
#
|
|
74
|
+
# **The name is narrower than the contents, and renaming it would break every
|
|
75
|
+
# caller's install line.** `compare` needs the referee too -- it hands two
|
|
76
|
+
# stored walks to `regression` -- so this extra now gates two commands. Both
|
|
77
|
+
# were verified against every release the range admits: 0.2.0, 0.2.1, 0.2.2.
|
|
73
78
|
collect = ["bmc-sensor-audit>=0.2.0,<0.3"]
|
|
74
79
|
dev = ["pytest>=7"]
|
|
75
80
|
|
{fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/__init__.py
RENAMED
|
@@ -15,6 +15,6 @@ from __future__ import annotations
|
|
|
15
15
|
# read this to decide whether the README may name a tag at all: an unreleased
|
|
16
16
|
# tree that announces `v0.1.0` hands a reader a tag to check out that does not
|
|
17
17
|
# exist. Bump this at step 1 of a release, not at step 9.
|
|
18
|
-
__version__ = "0.2.
|
|
18
|
+
__version__ = "0.2.2"
|
|
19
19
|
|
|
20
20
|
__all__ = ["__version__"]
|
{fleet_sensor_baseline-0.2.0 → fleet_sensor_baseline-0.2.2}/src/fleet_sensor_baseline/baseline.py
RENAMED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
"""Deriving a `fleet-baseline/
|
|
1
|
+
"""Deriving a `fleet-baseline/2` -- an additional, labeled, downgraded declaration.
|
|
2
2
|
|
|
3
3
|
**Wherever a manufacturer declaration exists, it wins.** A fleet-derived
|
|
4
4
|
baseline is blind to an absence the whole cohort shares, which is the founding
|
|
@@ -181,7 +181,7 @@ def derive(present_by_unit: dict[str, set[str]],
|
|
|
181
181
|
absent_threshold: float = DEFAULT_ABSENT_THRESHOLD,
|
|
182
182
|
floor: int = DEFAULT_FLOOR,
|
|
183
183
|
window: tuple[str, str] | None = None) -> dict:
|
|
184
|
-
"""Build the `fleet-baseline/
|
|
184
|
+
"""Build the `fleet-baseline/2`. Raises `BaselineError` below the floor."""
|
|
185
185
|
total = len(present_by_unit)
|
|
186
186
|
if total == 0:
|
|
187
187
|
raise BaselineError(
|