kinema 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kinema/__init__.py +3 -0
- kinema/causes.json +112 -0
- kinema/checks/__init__.py +21 -0
- kinema/checks/actions.py +155 -0
- kinema/checks/base.py +188 -0
- kinema/checks/calibration.py +364 -0
- kinema/checks/deployment.py +283 -0
- kinema/checks/duplicates.py +276 -0
- kinema/checks/idle.py +351 -0
- kinema/checks/metadata.py +244 -0
- kinema/checks/outliers.py +267 -0
- kinema/checks/stats_drift.py +308 -0
- kinema/checks/timestamps.py +235 -0
- kinema/checks/visual.py +241 -0
- kinema/cli.py +166 -0
- kinema/dataset.py +909 -0
- kinema/diagnose.py +402 -0
- kinema/engine.py +57 -0
- kinema/fixtures/__init__.py +0 -0
- kinema/fixtures/synth.py +239 -0
- kinema/mcp_server.py +762 -0
- kinema/policies.json +46 -0
- kinema/report.py +416 -0
- kinema/similar.py +136 -0
- kinema/stats.py +267 -0
- kinema-0.1.0.dist-info/METADATA +264 -0
- kinema-0.1.0.dist-info/RECORD +31 -0
- kinema-0.1.0.dist-info/WHEEL +5 -0
- kinema-0.1.0.dist-info/entry_points.txt +2 -0
- kinema-0.1.0.dist-info/licenses/LICENSE +196 -0
- kinema-0.1.0.dist-info/top_level.txt +1 -0
kinema/__init__.py
ADDED
kinema/causes.json
ADDED
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 1,
|
|
3
|
+
"clusters": {
|
|
4
|
+
"normalization": {
|
|
5
|
+
"keywords": ["normaliz", "stats", "statistics", "wrong scale", "scale mismatch", "scaling", "scaled", "rescal", "quantile", "q01", "q99", "mean/std", "z-score"],
|
|
6
|
+
"checks": ["stats-drift", "nan-and-normalization", "deployment-readiness"]
|
|
7
|
+
},
|
|
8
|
+
"idle": {
|
|
9
|
+
"keywords": ["doesn't move", "does not move", "not moving", "stays still", "stands still", "no movement", "frozen", "freezes", "freeze", "won't move", "wont move", "barely moves", "stays static", "remains static", "idle", "does nothing"],
|
|
10
|
+
"checks": ["idle-segments", "stuck-actuator"]
|
|
11
|
+
}
|
|
12
|
+
},
|
|
13
|
+
"causes": [
|
|
14
|
+
{
|
|
15
|
+
"id": "stored-stats-drift",
|
|
16
|
+
"cluster": "normalization",
|
|
17
|
+
"title": "Stored normalization stats disagree with the data",
|
|
18
|
+
"indicates": {"check_id": "stats-drift", "min_severity": "WARN"},
|
|
19
|
+
"confidence": {"FAIL": "high", "WARN": "medium"},
|
|
20
|
+
"explanation": "A pipeline that normalizes from the dataset's stored stats trains and infers with these numbers. When they drift from the data, the policy trains on wrong-scaled inputs and fails with a clean loss curve - the openpi #906 pattern, the top community diagnosis for 'low loss, failed robot' on {policy}.",
|
|
21
|
+
"docs_section": "stats-drift",
|
|
22
|
+
"next_step": "Recompute meta/stats.json from the data (LeRobot aggregation, or openpi's compute_norm_stats.py for openpi training) and retrain; if the stats came from a converter, fix the converter."
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
"id": "missing-quantile-stats",
|
|
26
|
+
"cluster": "normalization",
|
|
27
|
+
"title": "The policy normalizes with q01/q99 but the stats have none",
|
|
28
|
+
"indicates": {"check_id": "deployment-readiness", "min_severity": "WARN", "message_contains": "q01/q99"},
|
|
29
|
+
"confidence": {"WARN": "medium"},
|
|
30
|
+
"explanation": "{policy} normalizes with q01/q99 quantiles. If your training pipeline normalizes from the dataset's stats, missing quantiles mean the run either fails on the missing keys or normalizes with quantiles computed elsewhere, by different code - inputs land in a different range than training assumed. Pipelines that compute their own norm stats at train time do not read these, and for them this finding does not explain the failure.",
|
|
31
|
+
"docs_section": "deployment-readiness",
|
|
32
|
+
"next_step": "Check where your training pipeline gets its norm stats. If from the dataset, recompute meta/stats.json with q01/q99 and retrain; if it computes its own at train time, this finding does not apply - look at the other causes."
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"id": "missing-stats",
|
|
36
|
+
"cluster": "normalization",
|
|
37
|
+
"title": "No normalization stats at all",
|
|
38
|
+
"indicates": {"check_id": "deployment-readiness", "min_severity": "FAIL", "message_contains": "no normalization stats"},
|
|
39
|
+
"confidence": {"FAIL": "high"},
|
|
40
|
+
"explanation": "There are no stats to normalize {policy}'s inputs or outputs with. Training may still run (some pipelines compute their own), but inference without the same stats produces garbage actions.",
|
|
41
|
+
"docs_section": "deployment-readiness",
|
|
42
|
+
"next_step": "Generate meta/stats.json for the dataset and make sure the training run and the deployment load the same stats."
|
|
43
|
+
},
|
|
44
|
+
{
|
|
45
|
+
"id": "zero-variance-dims",
|
|
46
|
+
"cluster": "normalization",
|
|
47
|
+
"title": "Constant or near-constant dims break normalization",
|
|
48
|
+
"indicates": {"check_id": "nan-and-normalization", "min_severity": "WARN", "message_contains": "std"},
|
|
49
|
+
"confidence": {"WARN": "medium"},
|
|
50
|
+
"explanation": "A zero-variance dim divides by ~0 during normalization (Inf/NaN batches) or amplifies sensor noise into a full-scale signal the policy wastes capacity fitting.",
|
|
51
|
+
"docs_section": "nan-and-normalization",
|
|
52
|
+
"next_step": "Drop constant dims from the features (and the policy's input dims), or fix the recording so those dims carry signal; then recompute stats."
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
"id": "nan-values",
|
|
56
|
+
"cluster": "normalization",
|
|
57
|
+
"title": "NaN or Inf values in the data",
|
|
58
|
+
"indicates": {"check_id": "nan-and-normalization", "min_severity": "FAIL", "message_contains": "NaN"},
|
|
59
|
+
"confidence": {"FAIL": "high"},
|
|
60
|
+
"explanation": "NaN/Inf frames poison the batch statistics and the loss; training can appear to converge on the surviving batches while the policy learns corrupted input statistics.",
|
|
61
|
+
"docs_section": "nan-and-normalization",
|
|
62
|
+
"next_step": "Find the source in recording or conversion, drop the affected episodes (or repair the frames), recompute stats, and retrain."
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"id": "chunk-too-short",
|
|
66
|
+
"cluster": "normalization",
|
|
67
|
+
"title": "Episodes shorter than the policy's action chunk",
|
|
68
|
+
"indicates": {"check_id": "deployment-readiness", "min_severity": "WARN", "message_contains": "action chunk"},
|
|
69
|
+
"confidence": {"WARN": "medium"},
|
|
70
|
+
"explanation": "{policy} predicts action chunks; episodes shorter than the chunk train on padded or truncated actions, teaching the policy boundary artifacts instead of task behavior.",
|
|
71
|
+
"docs_section": "deployment-readiness",
|
|
72
|
+
"next_step": "Drop or merge the short episodes, or train with a smaller action horizon (pass it back as a policy override here to re-check)."
|
|
73
|
+
},
|
|
74
|
+
{
|
|
75
|
+
"id": "shape-mismatch",
|
|
76
|
+
"cluster": "normalization",
|
|
77
|
+
"title": "info.json shapes disagree with the data",
|
|
78
|
+
"indicates": {"check_id": "deployment-readiness", "min_severity": "FAIL", "message_contains": "declares shape"},
|
|
79
|
+
"confidence": {"FAIL": "high"},
|
|
80
|
+
"explanation": "The declared feature shapes are what the policy builds its input/output layers from; a mismatch means the policy sees a different layout than the metadata promises - training against the metadata fails or silently truncates dims.",
|
|
81
|
+
"docs_section": "deployment-readiness",
|
|
82
|
+
"next_step": "Fix meta/info.json to the actual array shapes (or fix the conversion that wrote it) before training."
|
|
83
|
+
},
|
|
84
|
+
{
|
|
85
|
+
"id": "idle-heavy-episodes",
|
|
86
|
+
"cluster": "idle",
|
|
87
|
+
"title": "Long idle segments teach the policy to do nothing",
|
|
88
|
+
"indicates": {"check_id": "idle-segments", "min_severity": "WARN"},
|
|
89
|
+
"confidence": {"WARN": "medium"},
|
|
90
|
+
"explanation": "Episodes with long stretches where no joint moves teach {policy} that doing nothing is a good action - the classic 'robot stays still at inference' postmortem. Idle at the end of episodes is worse: it teaches stopping early.",
|
|
91
|
+
"docs_section": "idle-segments",
|
|
92
|
+
"next_step": "Trim the idle segments (or the worst episodes) and retrain; if the idling is an artifact of how demonstrations start/end, fix the recording flow."
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"id": "stuck-or-unused-joint",
|
|
96
|
+
"cluster": "idle",
|
|
97
|
+
"title": "A joint barely moves - or never moves",
|
|
98
|
+
"indicates": {"check_id": "stuck-actuator", "min_severity": "WARN"},
|
|
99
|
+
"confidence": {"WARN": "medium"},
|
|
100
|
+
"explanation": "A joint that never moves teaches {policy} to ignore that degree of freedom; a gripper that never actuates in pick/place episodes means the demonstrations never show the grasp the task needs.",
|
|
101
|
+
"docs_section": "stuck-actuator",
|
|
102
|
+
"next_step": "Check whether the joint/gripper is an unused DOF (fine - consider dropping the dim) or a recording/hardware failure (re-record); a commanded joint whose follower never moves points at a dead or blocked motor."
|
|
103
|
+
}
|
|
104
|
+
],
|
|
105
|
+
"blind_spots": [
|
|
106
|
+
"Unit conventions (degrees vs radians): stats computed on the same data always agree, so self-consistent wrong units are invisible to every check.",
|
|
107
|
+
"Joint-order mismatches between the dataset's layout and the policy's expected layout.",
|
|
108
|
+
"Absolute-vs-delta action space mismatches that only appear at inference (calibration-drift classifies those dims and sits them out by design).",
|
|
109
|
+
"Train-fps vs the deployment control-loop frequency (timestamp-grid checks internal consistency only).",
|
|
110
|
+
"Training pipelines that recompute normalization stats at train time with different code than LeRobot's aggregation (e.g. an openpi run over stats the dataset never ships)."
|
|
111
|
+
]
|
|
112
|
+
}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
"""Check registry. Importing this package registers all built-in checks."""
|
|
2
|
+
|
|
3
|
+
from kinema.checks.base import (
|
|
4
|
+
ALL_CHECKS,
|
|
5
|
+
Check,
|
|
6
|
+
CheckConfig,
|
|
7
|
+
CheckResult,
|
|
8
|
+
Finding,
|
|
9
|
+
Severity,
|
|
10
|
+
load_config_overrides,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"ALL_CHECKS",
|
|
15
|
+
"Check",
|
|
16
|
+
"CheckConfig",
|
|
17
|
+
"CheckResult",
|
|
18
|
+
"Finding",
|
|
19
|
+
"Severity",
|
|
20
|
+
"load_config_overrides",
|
|
21
|
+
]
|
kinema/checks/actions.py
ADDED
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Checks 7 & 8 - action saturation and NaN/normalization hazards."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import numpy as np
|
|
6
|
+
|
|
7
|
+
from kinema.checks.base import Check, CheckConfig, CheckResult, Finding, Severity
|
|
8
|
+
from kinema.dataset import ACTION_KEY, STATE_KEY
|
|
9
|
+
from kinema.stats import action_gripper_dims, toggle_dims, velocity_dims
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class ActionSaturationCheck(Check):
|
|
13
|
+
id = "action-saturation"
|
|
14
|
+
title = "Action saturation (commands clipped at the limits)"
|
|
15
|
+
|
|
16
|
+
# Clipping leaves a pile-up of commands at EXACTLY the limit value while
|
|
17
|
+
# the robot is moving. Merely sitting near an extreme is normal (rest
|
|
18
|
+
# poses, grippers fully open/shut), as are binary or constant dims.
|
|
19
|
+
|
|
20
|
+
def run(self, ctx, cfg: CheckConfig) -> CheckResult:
|
|
21
|
+
res = CheckResult(self.id, self.title)
|
|
22
|
+
action = ctx.action
|
|
23
|
+
if action is None:
|
|
24
|
+
return self.skip("action column missing")
|
|
25
|
+
n = action.shape[1]
|
|
26
|
+
named_grip, inferred_grip = action_gripper_dims(ctx)
|
|
27
|
+
grip = named_grip # inferred grippers are checked but only reported at INFO
|
|
28
|
+
# on/off commands beyond the gripper slots (odd layouts the slot prior
|
|
29
|
+
# misses) get the same downgrade: piling up at an extreme is their
|
|
30
|
+
# normal operation, not clipping
|
|
31
|
+
toggles = toggle_dims(ctx, exclude=named_grip | inferred_grip
|
|
32
|
+
| velocity_dims(ctx.action_names, n))
|
|
33
|
+
finite = np.isfinite(action).all(axis=1)
|
|
34
|
+
a = action[finite]
|
|
35
|
+
if len(a) < 20:
|
|
36
|
+
return self.skip("too few finite action frames")
|
|
37
|
+
lo, hi = a.min(axis=0), a.max(axis=0)
|
|
38
|
+
rng = hi - lo
|
|
39
|
+
tol = 1e-6 * np.maximum(rng, 1e-12)
|
|
40
|
+
|
|
41
|
+
# moving frames: some dim changed since the previous frame of the
|
|
42
|
+
# same episode (idle stretches are idle-segments' job)
|
|
43
|
+
eps = np.maximum(1e-8, cfg.idle_eps_frac * a.std(axis=0))
|
|
44
|
+
moving = np.zeros(len(action), dtype=bool)
|
|
45
|
+
for ep in ctx.episode_ids():
|
|
46
|
+
rows = ctx.rows_for(ep)
|
|
47
|
+
if len(rows) < 2:
|
|
48
|
+
continue
|
|
49
|
+
d = np.abs(np.diff(action[rows], axis=0)) >= eps
|
|
50
|
+
moving[rows[1:]] = d.any(axis=1)
|
|
51
|
+
moving = moving[finite]
|
|
52
|
+
|
|
53
|
+
checked = 0
|
|
54
|
+
for j in range(n):
|
|
55
|
+
if j in grip or rng[j] <= 1e-8 or len(np.unique(a[:, j])) <= 10:
|
|
56
|
+
continue # gripper, constant or discrete-valued dim
|
|
57
|
+
if moving.sum() < 20:
|
|
58
|
+
continue
|
|
59
|
+
checked += 1
|
|
60
|
+
at_hi = np.abs(a[:, j] - hi[j]) <= tol[j]
|
|
61
|
+
at_lo = np.abs(a[:, j] - lo[j]) <= tol[j]
|
|
62
|
+
f_hi = float((at_hi & moving).sum() / moving.sum())
|
|
63
|
+
f_lo = float((at_lo & moving).sum() / moving.sum())
|
|
64
|
+
f = max(f_hi, f_lo)
|
|
65
|
+
if f > cfg.saturation_warn_fraction:
|
|
66
|
+
side = "max" if f_hi >= f_lo else "min"
|
|
67
|
+
name = f" ({ctx.action_names[j]})" if ctx.action_names and len(ctx.action_names) == n else ""
|
|
68
|
+
if f > 0.5:
|
|
69
|
+
# most of the data AT the limit is a mode, not a clipped tail:
|
|
70
|
+
# an unnamed gripper / binary-like command or a rest state
|
|
71
|
+
res.findings.append(Finding(
|
|
72
|
+
self.id, Severity.INFO,
|
|
73
|
+
f"action dim {j}{name}: {f:.0%} of commands sit exactly at its {side} - "
|
|
74
|
+
"a gripper or on/off command rather than clipping?",
|
|
75
|
+
details={"dim": j, "saturated_fraction": f, "side": side},
|
|
76
|
+
))
|
|
77
|
+
continue
|
|
78
|
+
likely_grip = j in inferred_grip or j in toggles
|
|
79
|
+
note = (" [dim looks like an unnamed gripper, so not gated]" if j in inferred_grip
|
|
80
|
+
else " [two-level on/off command, so not gated]" if j in toggles else "")
|
|
81
|
+
res.findings.append(
|
|
82
|
+
Finding(
|
|
83
|
+
self.id,
|
|
84
|
+
Severity.INFO if likely_grip else Severity.WARN,
|
|
85
|
+
f"action dim {j}{name}: {f:.1%} of commands during motion are pinned at "
|
|
86
|
+
f"exactly its {side} ({(hi if side == 'max' else lo)[j]:.6g}) - clipped "
|
|
87
|
+
"commands the policy will imitate" + note,
|
|
88
|
+
details={"dim": j, "saturated_fraction": f, "side": side},
|
|
89
|
+
)
|
|
90
|
+
)
|
|
91
|
+
if not any(f.severity >= Severity.WARN for f in res.findings):
|
|
92
|
+
res.summary = (
|
|
93
|
+
f"no continuous action dim pinned at a limit above "
|
|
94
|
+
f"{cfg.saturation_warn_fraction:.0%} ({checked} dims checked)"
|
|
95
|
+
)
|
|
96
|
+
else:
|
|
97
|
+
res.summary = f"{len(res.findings)} dim(s) saturated"
|
|
98
|
+
return res
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
class NanAndNormalizationCheck(Check):
|
|
102
|
+
id = "nan-and-normalization"
|
|
103
|
+
title = "NaN / Inf and normalization hazards"
|
|
104
|
+
|
|
105
|
+
def run(self, ctx, cfg: CheckConfig) -> CheckResult:
|
|
106
|
+
res = CheckResult(self.id, self.title)
|
|
107
|
+
for feat, arr in ((STATE_KEY, ctx.state), (ACTION_KEY, ctx.action)):
|
|
108
|
+
if arr is None:
|
|
109
|
+
continue
|
|
110
|
+
n_nan = int(np.isnan(arr).sum())
|
|
111
|
+
n_inf = int(np.isinf(arr).sum())
|
|
112
|
+
if n_nan or n_inf:
|
|
113
|
+
bad_eps = sorted(
|
|
114
|
+
{
|
|
115
|
+
int(ctx.episode_index[r])
|
|
116
|
+
for r in np.nonzero(~np.isfinite(arr).all(axis=1))[0]
|
|
117
|
+
} # noqa: C401
|
|
118
|
+
)
|
|
119
|
+
res.findings.append(
|
|
120
|
+
Finding(
|
|
121
|
+
self.id,
|
|
122
|
+
Severity.FAIL,
|
|
123
|
+
f"{feat}: {n_nan} NaN and {n_inf} Inf values "
|
|
124
|
+
f"(episodes {bad_eps[:10]}{'...' if len(bad_eps) > 10 else ''}) - "
|
|
125
|
+
"one NaN poisons the loss for the whole batch",
|
|
126
|
+
details={"nan": n_nan, "inf": n_inf, "episodes": bad_eps},
|
|
127
|
+
)
|
|
128
|
+
)
|
|
129
|
+
std = arr.std(axis=0)
|
|
130
|
+
zero = np.nonzero(std == 0)[0]
|
|
131
|
+
tiny = np.nonzero((std > 0) & (std < 1e-6))[0]
|
|
132
|
+
for j in zero:
|
|
133
|
+
res.findings.append(
|
|
134
|
+
Finding(
|
|
135
|
+
self.id,
|
|
136
|
+
Severity.INFO,
|
|
137
|
+
f"{feat} dim {j}: constant (std=0) - carries no signal; if it ever "
|
|
138
|
+
"varies at deployment, mean/std normalization (std + 1e-8) blows "
|
|
139
|
+
"it up. Usually padding; consider dropping it",
|
|
140
|
+
details={"feature": feat, "dim": int(j)},
|
|
141
|
+
)
|
|
142
|
+
)
|
|
143
|
+
for j in tiny:
|
|
144
|
+
res.findings.append(
|
|
145
|
+
Finding(
|
|
146
|
+
self.id,
|
|
147
|
+
Severity.WARN,
|
|
148
|
+
f"{feat} dim {j}: std {std[j]:.2e} - near-zero variance amplifies "
|
|
149
|
+
"noise after normalization",
|
|
150
|
+
details={"feature": feat, "dim": int(j), "std": float(std[j])},
|
|
151
|
+
)
|
|
152
|
+
)
|
|
153
|
+
if not res.findings:
|
|
154
|
+
res.summary = "no NaN/Inf; all dims have usable variance"
|
|
155
|
+
return res
|
kinema/checks/base.py
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Shared check framework: severities, findings, config, registry."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from dataclasses import dataclass, field, fields
|
|
7
|
+
from enum import IntEnum
|
|
8
|
+
from typing import TYPE_CHECKING, Any
|
|
9
|
+
|
|
10
|
+
if TYPE_CHECKING: # pragma: no cover
|
|
11
|
+
from kinema.dataset import DatasetContext
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class Severity(IntEnum):
|
|
15
|
+
OK = 0
|
|
16
|
+
INFO = 1
|
|
17
|
+
WARN = 2
|
|
18
|
+
FAIL = 3
|
|
19
|
+
SKIP = 4 # check did not run; not a verdict on the data
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Finding:
|
|
24
|
+
check_id: str
|
|
25
|
+
severity: Severity
|
|
26
|
+
message: str
|
|
27
|
+
episode_index: int | None = None
|
|
28
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
@dataclass
|
|
32
|
+
class CheckResult:
|
|
33
|
+
check_id: str
|
|
34
|
+
title: str
|
|
35
|
+
findings: list[Finding] = field(default_factory=list)
|
|
36
|
+
summary: str = ""
|
|
37
|
+
skipped: bool = False
|
|
38
|
+
skip_reason: str = ""
|
|
39
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
40
|
+
|
|
41
|
+
@property
|
|
42
|
+
def verdict(self) -> Severity:
|
|
43
|
+
if self.skipped:
|
|
44
|
+
return Severity.SKIP
|
|
45
|
+
if not self.findings:
|
|
46
|
+
return Severity.OK
|
|
47
|
+
return max((f.severity for f in self.findings), default=Severity.OK)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class CheckConfig:
|
|
52
|
+
video: bool = False
|
|
53
|
+
policy: str | None = None
|
|
54
|
+
# per-run policies.json overrides (the MCP diagnosis layer passes these;
|
|
55
|
+
# not a threshold, so --config JSON files cannot set it)
|
|
56
|
+
policy_overrides: dict = field(default_factory=dict)
|
|
57
|
+
# stats-drift (mean in std devs, std as a ratio, min/max as range fraction)
|
|
58
|
+
stats_mean_warn_sigma: float = 0.10
|
|
59
|
+
stats_mean_fail_sigma: float = 0.50
|
|
60
|
+
stats_std_warn: float = 0.05
|
|
61
|
+
stats_std_fail: float = 0.25
|
|
62
|
+
stats_std_oversize_fail: float = 3.0 # stored action std >= 3x real: FAIL
|
|
63
|
+
stats_range_warn: float = 0.01
|
|
64
|
+
stats_range_fail: float = 0.10
|
|
65
|
+
# sampled loads without per-episode stats: gross-corruption backstop
|
|
66
|
+
stats_sample_std_ratio: float = 3.0
|
|
67
|
+
stats_sample_mean_sigma: float = 2.0
|
|
68
|
+
# ...and the sharper test above it: bootstrap the sampling distribution
|
|
69
|
+
# of the pooled stats over loaded episodes (the episode is the unit -
|
|
70
|
+
# frames within one are autocorrelated) and flag stored stats outside
|
|
71
|
+
# the (1 - alpha) interval
|
|
72
|
+
stats_sample_min_episodes: int = 8
|
|
73
|
+
stats_sample_bootstrap: int = 400
|
|
74
|
+
stats_sample_alpha: float = 0.001
|
|
75
|
+
# timestamp-grid
|
|
76
|
+
grid_warn_frac: float = 0.10 # fraction of the frame period
|
|
77
|
+
# calibration-drift
|
|
78
|
+
calib_warn_range_frac: float = 0.05
|
|
79
|
+
calib_fail_range_frac: float = 0.15
|
|
80
|
+
calib_min_corr: float = 0.90 # action must track state to be comparable
|
|
81
|
+
calib_scale_warn: float = 0.05 # offset drifts > 5% of range across the range
|
|
82
|
+
calib_scale_fail: float = 0.15
|
|
83
|
+
calib_scale_min_r2: float = 0.60 # ...and a line explains the offset
|
|
84
|
+
calib_scale_line_vs_step: float = 1.5 # ...and beats the best two-level step (backlash)
|
|
85
|
+
calib_gripper_warn: float = 0.15 # gripper offset at its free end
|
|
86
|
+
# delta commands: measured step vs commanded step (regression slope)
|
|
87
|
+
calib_delta_scale_warn: float = 0.5 # |slope - 1| at/above this gates
|
|
88
|
+
calib_delta_scale_info: float = 0.25 # milder disagreement: report only
|
|
89
|
+
calib_delta_min_r2: float = 0.5 # steps must track before a scale verdict
|
|
90
|
+
# idle / stuck
|
|
91
|
+
idle_eps_frac: float = 0.01 # fraction of per-joint action std
|
|
92
|
+
idle_min_seconds: float = 2.0
|
|
93
|
+
idle_warn_fraction: float = 0.30
|
|
94
|
+
idle_state_frac: float = 0.02 # state moves < 2% of its range over an idle run
|
|
95
|
+
stuck_range_frac: float = 0.005 # episode range < 0.5% of the dataset range
|
|
96
|
+
stuck_warn_share: float = 0.25 # stuck in <= 25% of episodes: anomalous
|
|
97
|
+
stuck_unused_share: float = 0.90 # stuck in >= 90%: unused DOF (INFO)
|
|
98
|
+
follow_min_command: float = 0.10 # commanded >= 10% of the joint's range...
|
|
99
|
+
follow_ratio: float = 0.10 # ...but the follower moved < 10% of that
|
|
100
|
+
# action saturation (frames pinned exactly at the limit while moving)
|
|
101
|
+
saturation_warn_fraction: float = 0.05
|
|
102
|
+
# outliers
|
|
103
|
+
outlier_iqr_k: float = 1.5
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
# runtime flags live on CheckConfig but are not thresholds: --video/--policy
|
|
107
|
+
# and the MCP layer's policy_overrides dict
|
|
108
|
+
_CONFIG_ONLY = ("video", "policy", "policy_overrides")
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
def load_config_overrides(path: str) -> dict[str, Any]:
|
|
112
|
+
"""Threshold overrides from a JSON file (`kinema check --config`).
|
|
113
|
+
|
|
114
|
+
Keys are CheckConfig field names; unknown keys, non-numeric values and
|
|
115
|
+
the runtime flags are rejected so a typo never silently tunes nothing.
|
|
116
|
+
The return is typed Any so `CheckConfig(**overrides)` type-checks -
|
|
117
|
+
the values are validated numeric above."""
|
|
118
|
+
with open(path) as f:
|
|
119
|
+
data = json.load(f)
|
|
120
|
+
known = {f.name for f in fields(CheckConfig)} - set(_CONFIG_ONLY)
|
|
121
|
+
if not isinstance(data, dict):
|
|
122
|
+
raise ValueError(f"{path}: expected a JSON object of threshold overrides")
|
|
123
|
+
unknown = sorted(set(data) - known)
|
|
124
|
+
if unknown:
|
|
125
|
+
raise ValueError(
|
|
126
|
+
f"{path}: unknown config keys: {', '.join(unknown)} "
|
|
127
|
+
f"(valid: {', '.join(sorted(known))})"
|
|
128
|
+
)
|
|
129
|
+
bad = sorted(
|
|
130
|
+
k for k, v in data.items() if isinstance(v, bool) or not isinstance(v, (int, float))
|
|
131
|
+
)
|
|
132
|
+
if bad:
|
|
133
|
+
raise ValueError(f"{path}: values must be numbers: {', '.join(bad)}")
|
|
134
|
+
return dict(data)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
class Check:
|
|
138
|
+
"""Base class. Subclasses set id/title and implement run()."""
|
|
139
|
+
|
|
140
|
+
id: str = ""
|
|
141
|
+
title: str = ""
|
|
142
|
+
requires_video: bool = False
|
|
143
|
+
|
|
144
|
+
def run(self, ctx: "DatasetContext", cfg: CheckConfig) -> CheckResult:
|
|
145
|
+
raise NotImplementedError
|
|
146
|
+
|
|
147
|
+
def skip(self, reason: str) -> CheckResult:
|
|
148
|
+
return CheckResult(self.id, self.title, skipped=True, skip_reason=reason)
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _registry() -> list[Check]:
|
|
152
|
+
from kinema.checks.actions import ActionSaturationCheck, NanAndNormalizationCheck
|
|
153
|
+
from kinema.checks.calibration import CalibrationDriftCheck
|
|
154
|
+
from kinema.checks.deployment import DeploymentReadinessCheck
|
|
155
|
+
from kinema.checks.duplicates import DuplicateCheck
|
|
156
|
+
from kinema.checks.idle import IdleSegmentsCheck, StuckActuatorCheck
|
|
157
|
+
from kinema.checks.metadata import (
|
|
158
|
+
CameraCoverageCheck,
|
|
159
|
+
EmbeddedImagesCheck,
|
|
160
|
+
RewardDoneCheck,
|
|
161
|
+
TaskStringsCheck,
|
|
162
|
+
)
|
|
163
|
+
from kinema.checks.outliers import OutlierEpisodesCheck
|
|
164
|
+
from kinema.checks.stats_drift import StatsDriftCheck
|
|
165
|
+
from kinema.checks.timestamps import DroppedFramesCheck, TimestampGridCheck
|
|
166
|
+
from kinema.checks.visual import VisualQualityCheck
|
|
167
|
+
|
|
168
|
+
return [
|
|
169
|
+
StatsDriftCheck(),
|
|
170
|
+
TimestampGridCheck(),
|
|
171
|
+
DroppedFramesCheck(),
|
|
172
|
+
CalibrationDriftCheck(),
|
|
173
|
+
IdleSegmentsCheck(),
|
|
174
|
+
StuckActuatorCheck(),
|
|
175
|
+
ActionSaturationCheck(),
|
|
176
|
+
NanAndNormalizationCheck(),
|
|
177
|
+
DuplicateCheck(),
|
|
178
|
+
OutlierEpisodesCheck(),
|
|
179
|
+
EmbeddedImagesCheck(),
|
|
180
|
+
CameraCoverageCheck(),
|
|
181
|
+
TaskStringsCheck(),
|
|
182
|
+
RewardDoneCheck(),
|
|
183
|
+
VisualQualityCheck(),
|
|
184
|
+
DeploymentReadinessCheck(),
|
|
185
|
+
]
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
ALL_CHECKS: list[Check] = _registry()
|