robolint 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- robolint/__init__.py +9 -0
- robolint/__main__.py +5 -0
- robolint/api.py +108 -0
- robolint/checks/__init__.py +53 -0
- robolint/checks/episodes.py +126 -0
- robolint/checks/meta.py +122 -0
- robolint/checks/signals.py +278 -0
- robolint/checks/timing.py +104 -0
- robolint/checks/video.py +211 -0
- robolint/cli.py +136 -0
- robolint/dataset.py +516 -0
- robolint/findings.py +90 -0
- robolint/report.py +262 -0
- robolint-0.1.0.dist-info/METADATA +150 -0
- robolint-0.1.0.dist-info/RECORD +18 -0
- robolint-0.1.0.dist-info/WHEEL +4 -0
- robolint-0.1.0.dist-info/entry_points.txt +2 -0
- robolint-0.1.0.dist-info/licenses/LICENSE +21 -0
robolint/__init__.py
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
"""robolint: quality checks for robot learning datasets (LeRobot format)."""
|
|
2
|
+
|
|
3
|
+
__version__ = "0.1.0"
|
|
4
|
+
|
|
5
|
+
from .api import Result, lint # noqa: E402
|
|
6
|
+
from .checks import Config # noqa: E402
|
|
7
|
+
from .findings import Finding, Severity # noqa: E402
|
|
8
|
+
|
|
9
|
+
__all__ = ["Config", "Finding", "Result", "Severity", "__version__", "lint"]
|
robolint/__main__.py
ADDED
robolint/api.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""Programmatic entry point: lint a dataset and get a structured result."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
from collections import Counter
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
from . import __version__
|
|
12
|
+
from .checks import Config, all_checks
|
|
13
|
+
from .dataset import Dataset, hub_blob_dirs, load_dataset, resolve_source
|
|
14
|
+
from .findings import Finding, Scores, Severity, score
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass
|
|
18
|
+
class Result:
|
|
19
|
+
source: str
|
|
20
|
+
dataset: Dataset
|
|
21
|
+
findings: list[Finding]
|
|
22
|
+
scores: Scores
|
|
23
|
+
videos_checked: bool
|
|
24
|
+
seconds: float
|
|
25
|
+
extra: dict[str, Any] = field(default_factory=dict)
|
|
26
|
+
|
|
27
|
+
@property
|
|
28
|
+
def dataset_findings(self) -> list[Finding]:
|
|
29
|
+
return [f for f in self.findings if f.episode is None]
|
|
30
|
+
|
|
31
|
+
def episode_findings(self, ep: int) -> list[Finding]:
|
|
32
|
+
return [f for f in self.findings if f.episode == ep]
|
|
33
|
+
|
|
34
|
+
def code_counts(self) -> list[tuple[str, Severity, int]]:
|
|
35
|
+
"""(code, worst severity, number of episodes or 1 for dataset-level) sorted by impact."""
|
|
36
|
+
eps: dict[str, set] = {}
|
|
37
|
+
sev: dict[str, Severity] = {}
|
|
38
|
+
for f in self.findings:
|
|
39
|
+
eps.setdefault(f.code, set()).add(f.episode)
|
|
40
|
+
sev[f.code] = max(sev.get(f.code, f.severity), f.severity)
|
|
41
|
+
rows = [(c, sev[c], len(e)) for c, e in eps.items()]
|
|
42
|
+
return sorted(rows, key=lambda r: (-int(r[1]), -r[2], r[0]))
|
|
43
|
+
|
|
44
|
+
def bad_episodes(self, min_severity: Severity = Severity.ERROR) -> list[int]:
|
|
45
|
+
return sorted({f.episode for f in self.findings if f.episode is not None and f.severity >= min_severity})
|
|
46
|
+
|
|
47
|
+
def to_dict(self) -> dict[str, Any]:
|
|
48
|
+
ds = self.dataset
|
|
49
|
+
sev = Counter(f.severity.label for f in self.findings)
|
|
50
|
+
return {
|
|
51
|
+
"tool": {"name": "robolint", "version": __version__},
|
|
52
|
+
"source": self.source,
|
|
53
|
+
"dataset": {
|
|
54
|
+
"codebase_version": ds.version,
|
|
55
|
+
"robot_type": ds.info.get("robot_type"),
|
|
56
|
+
"fps": ds.fps,
|
|
57
|
+
"episodes": len(ds.episodes),
|
|
58
|
+
"frames": ds.total_frames,
|
|
59
|
+
"cameras": ds.video_keys,
|
|
60
|
+
"tasks": len(ds.tasks),
|
|
61
|
+
},
|
|
62
|
+
"score": self.scores.dataset_score,
|
|
63
|
+
"grade": self.scores.grade,
|
|
64
|
+
"episodes": {
|
|
65
|
+
"clean": self.scores.clean_episodes,
|
|
66
|
+
"with_warnings": self.scores.episodes_with_warnings,
|
|
67
|
+
"with_errors": self.scores.episodes_with_errors,
|
|
68
|
+
},
|
|
69
|
+
"videos_checked": self.videos_checked,
|
|
70
|
+
"finding_counts": dict(sev),
|
|
71
|
+
"issues": [{"code": c, "severity": s.label, "affected": n} for c, s, n in self.code_counts()],
|
|
72
|
+
"episode_scores": {str(k): v for k, v in sorted(self.scores.episode_scores.items())},
|
|
73
|
+
"findings": [f.to_dict() for f in self.findings],
|
|
74
|
+
"seconds": round(self.seconds, 2),
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def lint(
|
|
79
|
+
source: str | Path,
|
|
80
|
+
*,
|
|
81
|
+
videos: bool = False,
|
|
82
|
+
revision: str | None = None,
|
|
83
|
+
cache_dir: str | None = None,
|
|
84
|
+
max_episodes: int | None = None,
|
|
85
|
+
config: Config | None = None,
|
|
86
|
+
) -> Result:
|
|
87
|
+
t0 = time.perf_counter()
|
|
88
|
+
cfg = config or Config()
|
|
89
|
+
# Only a Hub download gets the relaxed symlink rule; a local folder keeps the strict one.
|
|
90
|
+
from_hub = not Path(str(source)).expanduser().exists()
|
|
91
|
+
root = resolve_source(
|
|
92
|
+
str(source), include_videos=videos, revision=revision, cache_dir=cache_dir, max_episodes=max_episodes
|
|
93
|
+
)
|
|
94
|
+
hub_blobs = hub_blob_dirs(root) if from_hub else None
|
|
95
|
+
ds = load_dataset(root, source=str(source), max_episodes=max_episodes, hub_blobs=hub_blobs)
|
|
96
|
+
findings: list[Finding] = []
|
|
97
|
+
for check in all_checks(include_video=videos):
|
|
98
|
+
findings.extend(check(ds, cfg))
|
|
99
|
+
findings.sort(key=lambda f: (f.episode if f.episode is not None else -1, -int(f.severity), f.code))
|
|
100
|
+
scores = score(findings, [ep.index for ep in ds.episodes])
|
|
101
|
+
return Result(
|
|
102
|
+
source=str(source),
|
|
103
|
+
dataset=ds,
|
|
104
|
+
findings=findings,
|
|
105
|
+
scores=scores,
|
|
106
|
+
videos_checked=videos,
|
|
107
|
+
seconds=time.perf_counter() - t0,
|
|
108
|
+
)
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Check registry and shared configuration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable, Iterable
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
|
|
8
|
+
from ..dataset import Dataset
|
|
9
|
+
from ..findings import Finding
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class Config:
|
|
14
|
+
# Timing
|
|
15
|
+
gap_factor: float = 1.5 # dt above this many frame periods counts as dropped frames
|
|
16
|
+
jitter_ratio: float = 0.10 # std(dt) / period above this is jittery
|
|
17
|
+
drift_frames: float = 2.0 # end-to-end drift tolerance, in frame periods
|
|
18
|
+
# Signals (fractions of each dimension's dataset-wide range)
|
|
19
|
+
move_tol: float = 0.005 # change smaller than this counts as "not moving"
|
|
20
|
+
dead_tol: float = 0.01 # every dimension moves less than this -> dead episode
|
|
21
|
+
gripper_tol: float = 0.05 # gripper travel smaller than this -> never actuated
|
|
22
|
+
spike_quantile: float = 0.995 # robust scale for per-step action changes
|
|
23
|
+
spike_multiplier: float = 6.0
|
|
24
|
+
spike_min_range: float = 0.15 # a spike must also be at least this fraction of the range
|
|
25
|
+
idle_warn: float = 0.30 # idle lead-in + tail above this fraction of the episode
|
|
26
|
+
idle_info: float = 0.15
|
|
27
|
+
limit_seconds: float = 2.0 # pinned at a joint's extreme for this long
|
|
28
|
+
# Episodes
|
|
29
|
+
min_seconds: float = 1.0
|
|
30
|
+
short_ratio: float = 0.25 # shorter than this fraction of the median length
|
|
31
|
+
long_ratio: float = 3.0
|
|
32
|
+
duplicate_tol: float = 0.002 # mean abs difference (fraction of range) for duplicates
|
|
33
|
+
# Video
|
|
34
|
+
video_samples: int = 8
|
|
35
|
+
black_level: float = 8.0
|
|
36
|
+
white_level: float = 247.0
|
|
37
|
+
blank_std: float = 2.0
|
|
38
|
+
frozen_diff: float = 1.0 # minimum mean pixel change (0-255) before a feed counts as live
|
|
39
|
+
frozen_min_seconds: float = 2.0 # shorter episodes are too brief to judge
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
CheckFn = Callable[[Dataset, Config], Iterable[Finding]]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def all_checks(include_video: bool) -> list[CheckFn]:
|
|
46
|
+
from . import episodes, meta, signals, timing
|
|
47
|
+
|
|
48
|
+
checks: list[CheckFn] = [meta.check, timing.check, signals.check, episodes.check]
|
|
49
|
+
if include_video:
|
|
50
|
+
from . import video
|
|
51
|
+
|
|
52
|
+
checks.append(video.check)
|
|
53
|
+
return checks
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Episode-level checks: lengths, tasks, and duplicate demonstrations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ..dataset import Dataset
|
|
10
|
+
from ..findings import Finding, Severity
|
|
11
|
+
from . import Config
|
|
12
|
+
from .signals import dataset_stats
|
|
13
|
+
|
|
14
|
+
E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
|
|
15
|
+
RESAMPLE = 64
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
|
|
19
|
+
if not ds.episodes:
|
|
20
|
+
yield Finding("NO_EPISODES", E, "dataset contains no episodes")
|
|
21
|
+
return
|
|
22
|
+
fps = ds.fps or 30.0
|
|
23
|
+
lengths = np.array([ep.length for ep in ds.episodes])
|
|
24
|
+
median = float(np.median(lengths))
|
|
25
|
+
|
|
26
|
+
for ep in ds.episodes:
|
|
27
|
+
n = ep.length
|
|
28
|
+
if ep.length_meta is not None and ep.length_meta != n:
|
|
29
|
+
yield Finding(
|
|
30
|
+
"EPISODE_LENGTH_MISMATCH",
|
|
31
|
+
E,
|
|
32
|
+
f"metadata says {ep.length_meta} frames, data has {n}",
|
|
33
|
+
ep.index,
|
|
34
|
+
{"metadata": ep.length_meta, "data": n},
|
|
35
|
+
)
|
|
36
|
+
if 0 < n < cfg.min_seconds * fps:
|
|
37
|
+
yield Finding("EPISODE_TOO_SHORT", E, f"only {n} frames ({n / fps:.2f}s): likely a false start", ep.index)
|
|
38
|
+
elif len(lengths) >= 5 and n < cfg.short_ratio * median:
|
|
39
|
+
yield Finding(
|
|
40
|
+
"EPISODE_SHORT",
|
|
41
|
+
W,
|
|
42
|
+
f"{n} frames, under {cfg.short_ratio:.0%} of the median ({median:.0f}): possibly truncated",
|
|
43
|
+
ep.index,
|
|
44
|
+
)
|
|
45
|
+
if len(lengths) >= 5 and n > cfg.long_ratio * median and n - median > 5 * fps:
|
|
46
|
+
yield Finding(
|
|
47
|
+
"EPISODE_LONG",
|
|
48
|
+
I,
|
|
49
|
+
f"{n} frames, {n / median:.1f}x the median: recording may not have been stopped",
|
|
50
|
+
ep.index,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
ti = ep.frames.get("task_index")
|
|
54
|
+
if ti is not None and ds.tasks:
|
|
55
|
+
unknown = sorted({int(x) for x in np.unique(ti)} - set(ds.tasks))
|
|
56
|
+
if unknown:
|
|
57
|
+
yield Finding(
|
|
58
|
+
"TASK_INDEX_UNKNOWN", E, f"task_index values {unknown[:5]} have no task description", ep.index
|
|
59
|
+
)
|
|
60
|
+
if ep.tasks and not any(t.strip() for t in ep.tasks):
|
|
61
|
+
yield Finding("EPISODE_TASK_EMPTY", W, "episode's task description is empty", ep.index)
|
|
62
|
+
|
|
63
|
+
yield from _duplicates(ds, cfg)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _signature(a: np.ndarray, lo: np.ndarray, rng: np.ndarray) -> np.ndarray:
|
|
67
|
+
a = a.reshape(-1, 1) if a.ndim == 1 else a
|
|
68
|
+
t_old = np.linspace(0.0, 1.0, len(a))
|
|
69
|
+
t_new = np.linspace(0.0, 1.0, RESAMPLE)
|
|
70
|
+
res = np.stack([np.interp(t_new, t_old, a[:, j]) for j in range(a.shape[1])], axis=1)
|
|
71
|
+
return ((res - lo) / rng).ravel()
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _duplicates(ds: Dataset, cfg: Config) -> Iterator[Finding]:
|
|
75
|
+
st = dataset_stats(ds, "action", cfg)
|
|
76
|
+
if st is None:
|
|
77
|
+
return
|
|
78
|
+
act = st.active
|
|
79
|
+
if not act.any():
|
|
80
|
+
return
|
|
81
|
+
rng = np.where(act, st.rng, 1.0)
|
|
82
|
+
eps, sigs, lens = [], [], []
|
|
83
|
+
for ep in ds.episodes:
|
|
84
|
+
a = ep.frames.get("action")
|
|
85
|
+
if a is None or ep.length < 4:
|
|
86
|
+
continue
|
|
87
|
+
a2 = a.reshape(-1, 1) if a.ndim == 1 else a
|
|
88
|
+
if a2.shape[1] != len(rng) or not np.all(np.isfinite(a2)):
|
|
89
|
+
continue
|
|
90
|
+
if np.all((a2.max(0) - a2.min(0))[act] < cfg.dead_tol * st.rng[act]):
|
|
91
|
+
continue # dead episodes are reported elsewhere and would all look alike
|
|
92
|
+
eps.append(ep.index)
|
|
93
|
+
sigs.append(_signature(a2[:, act].astype(np.float64), st.lo[act], rng[act]))
|
|
94
|
+
lens.append(ep.length)
|
|
95
|
+
if len(sigs) < 2:
|
|
96
|
+
return
|
|
97
|
+
S = np.stack(sigs)
|
|
98
|
+
L = np.array(lens, dtype=float)
|
|
99
|
+
# mean|Si - Sj| >= |mean(Si) - mean(Sj)|, so only pairs whose signature means are within
|
|
100
|
+
# the tolerance can be duplicates. Sorting by mean makes this exact pruning, which keeps
|
|
101
|
+
# 100k-episode datasets fast.
|
|
102
|
+
M = S.mean(axis=1)
|
|
103
|
+
order = np.argsort(M, kind="stable")
|
|
104
|
+
M_sorted = M[order]
|
|
105
|
+
seen: set[int] = set()
|
|
106
|
+
for i in range(len(S)):
|
|
107
|
+
if eps[i] in seen:
|
|
108
|
+
continue
|
|
109
|
+
lo_pos = np.searchsorted(M_sorted, M[i] - cfg.duplicate_tol, side="left")
|
|
110
|
+
hi_pos = np.searchsorted(M_sorted, M[i] + cfg.duplicate_tol, side="right")
|
|
111
|
+
cand = order[lo_pos:hi_pos]
|
|
112
|
+
cand = np.sort(cand[cand > i])
|
|
113
|
+
cand = cand[np.abs(L[cand] - L[i]) <= 0.05 * L[i] + 1]
|
|
114
|
+
if not len(cand):
|
|
115
|
+
continue
|
|
116
|
+
dist = np.mean(np.abs(S[cand] - S[i]), axis=1)
|
|
117
|
+
for j in cand[dist < cfg.duplicate_tol]:
|
|
118
|
+
seen.add(eps[j])
|
|
119
|
+
yield Finding(
|
|
120
|
+
"DUPLICATE_EPISODE",
|
|
121
|
+
W,
|
|
122
|
+
f"actions are nearly identical to episode {eps[i]}: a duplicated recording, or a scripted demo "
|
|
123
|
+
"repeated with the same setup",
|
|
124
|
+
eps[j],
|
|
125
|
+
{"duplicate_of": eps[i], "mean_abs_diff": round(float(dist[list(cand).index(j)]), 5)},
|
|
126
|
+
)
|
robolint/checks/meta.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Dataset-level structure and metadata consistency."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterator
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from ..dataset import Dataset
|
|
10
|
+
from ..findings import Finding, Severity
|
|
11
|
+
from . import Config
|
|
12
|
+
|
|
13
|
+
E, W, I = Severity.ERROR, Severity.WARNING, Severity.INFO
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def check(ds: Dataset, cfg: Config) -> Iterator[Finding]:
|
|
17
|
+
info = ds.info
|
|
18
|
+
for msg in ds.load_warnings:
|
|
19
|
+
yield Finding("METADATA_INCOMPLETE", W, msg)
|
|
20
|
+
|
|
21
|
+
if ds.fps is None:
|
|
22
|
+
yield Finding("FPS_MISSING", E, "meta/info.json has no valid fps; timing cannot be verified")
|
|
23
|
+
|
|
24
|
+
major = ds.major_version
|
|
25
|
+
if major and major < 2:
|
|
26
|
+
yield Finding("OLD_FORMAT", W, f"codebase_version {ds.version} predates LeRobot v2; results may be incomplete")
|
|
27
|
+
|
|
28
|
+
if ds.partial:
|
|
29
|
+
yield Finding(
|
|
30
|
+
"PARTIAL_CHECK",
|
|
31
|
+
I,
|
|
32
|
+
f"quick mode: only the first {len(ds.episodes)} of {info.get('total_episodes', '?')} episodes were checked",
|
|
33
|
+
)
|
|
34
|
+
# Declared vs found counts (meaningless when only part of the dataset was loaded)
|
|
35
|
+
found_eps = len(ds.episodes)
|
|
36
|
+
declared_eps = info.get("total_episodes")
|
|
37
|
+
if ds.partial:
|
|
38
|
+
declared_eps = None
|
|
39
|
+
if isinstance(declared_eps, int) and declared_eps != found_eps:
|
|
40
|
+
yield Finding(
|
|
41
|
+
"EPISODE_COUNT_MISMATCH",
|
|
42
|
+
E,
|
|
43
|
+
f"info.json declares {declared_eps} episodes but the data contains {found_eps}",
|
|
44
|
+
details={"declared": declared_eps, "found": found_eps},
|
|
45
|
+
)
|
|
46
|
+
declared_frames = None if ds.partial else info.get("total_frames")
|
|
47
|
+
if isinstance(declared_frames, int) and declared_frames != ds.total_frames:
|
|
48
|
+
yield Finding(
|
|
49
|
+
"FRAME_COUNT_MISMATCH",
|
|
50
|
+
E,
|
|
51
|
+
f"info.json declares {declared_frames} frames but the data contains {ds.total_frames}",
|
|
52
|
+
details={"declared": declared_frames, "found": ds.total_frames},
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
# Episode indices should be 0..N-1
|
|
56
|
+
idx = sorted(ep.index for ep in ds.episodes)
|
|
57
|
+
if idx and idx != list(range(idx[0], idx[0] + len(idx))):
|
|
58
|
+
missing = sorted(set(range(idx[0], idx[-1] + 1)) - set(idx))
|
|
59
|
+
yield Finding(
|
|
60
|
+
"EPISODE_INDEX_GAPS",
|
|
61
|
+
W,
|
|
62
|
+
f"{len(missing)} episode indices missing between {idx[0]} and {idx[-1]}",
|
|
63
|
+
details={"missing": missing[:50]},
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
# Global "index" column should be a contiguous range across the dataset
|
|
67
|
+
globals_ = [ep.frames["index"] for ep in ds.episodes if "index" in ep.frames]
|
|
68
|
+
if globals_:
|
|
69
|
+
g = np.sort(np.concatenate(globals_).astype(np.int64))
|
|
70
|
+
dupes = int(np.sum(np.diff(g) == 0))
|
|
71
|
+
if dupes:
|
|
72
|
+
yield Finding("GLOBAL_INDEX_DUPLICATES", E, f"{dupes} duplicate values in the global 'index' column")
|
|
73
|
+
elif len(g) and (g[-1] - g[0] + 1) != len(g):
|
|
74
|
+
yield Finding("GLOBAL_INDEX_GAPS", W, "global 'index' column is not contiguous")
|
|
75
|
+
|
|
76
|
+
# Declared features must exist in the data
|
|
77
|
+
for key, spec in ds.features.items():
|
|
78
|
+
if not isinstance(spec, dict) or spec.get("dtype") in ("video", "image"):
|
|
79
|
+
continue
|
|
80
|
+
if key not in ds.data_columns:
|
|
81
|
+
yield Finding("FEATURE_MISSING", E, f"feature '{key}' is declared in info.json but absent from the data")
|
|
82
|
+
|
|
83
|
+
if "action" not in ds.data_columns:
|
|
84
|
+
yield Finding("NO_ACTIONS", E, "dataset has no 'action' column; it cannot be used for policy training")
|
|
85
|
+
if "timestamp" not in ds.data_columns:
|
|
86
|
+
yield Finding("NO_TIMESTAMPS", E, "dataset has no 'timestamp' column")
|
|
87
|
+
|
|
88
|
+
# Declared shapes should match
|
|
89
|
+
for key in ("action", "observation.state"):
|
|
90
|
+
spec = ds.features.get(key)
|
|
91
|
+
if not isinstance(spec, dict) or not ds.episodes or key not in ds.episodes[0].frames:
|
|
92
|
+
continue
|
|
93
|
+
shape = spec.get("shape")
|
|
94
|
+
arr = ds.episodes[0].frames[key]
|
|
95
|
+
d = arr.shape[1] if arr.ndim == 2 else 1
|
|
96
|
+
if isinstance(shape, list) and shape and isinstance(shape[0], int) and shape[0] != d:
|
|
97
|
+
yield Finding(
|
|
98
|
+
"SHAPE_MISMATCH",
|
|
99
|
+
E,
|
|
100
|
+
f"'{key}' declared with {shape[0]} dimensions but data has {d}",
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
# Tasks
|
|
104
|
+
if not ds.tasks:
|
|
105
|
+
yield Finding("TASKS_MISSING", W, "no task descriptions found (meta/tasks.parquet or tasks.jsonl)")
|
|
106
|
+
else:
|
|
107
|
+
empty = [i for i, t in ds.tasks.items() if not t.strip()]
|
|
108
|
+
if empty:
|
|
109
|
+
yield Finding(
|
|
110
|
+
"TASK_EMPTY", W, f"{len(empty)} task descriptions are empty", details={"task_indices": empty[:20]}
|
|
111
|
+
)
|
|
112
|
+
vague = [t for t in ds.tasks.values() if t.strip() and (len(t.strip()) < 8 or t.strip().lower() in VAGUE_TASKS)]
|
|
113
|
+
if vague:
|
|
114
|
+
yield Finding(
|
|
115
|
+
"TASK_VAGUE",
|
|
116
|
+
I,
|
|
117
|
+
f"{len(vague)} task descriptions are too vague to condition a language policy",
|
|
118
|
+
details={"examples": vague[:5]},
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
VAGUE_TASKS = {"task", "test", "demo", "default", "debug", "none", "null", "untitled", "do the task", "pick", "place"}
|