humancoin 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- humancoin/__init__.py +28 -0
- humancoin/audio.py +282 -0
- humancoin/checker.py +178 -0
- humancoin/dataset.py +371 -0
- humancoin/download.py +147 -0
- humancoin/index.py +127 -0
- humancoin/metadata.py +123 -0
- humancoin/parquet.py +179 -0
- humancoin/utils.py +236 -0
- humancoin/video.py +213 -0
- humancoin/visualization/__init__.py +1 -0
- humancoin/visualization/cli.py +32 -0
- humancoin/visualization/desktop.py +268 -0
- humancoin/visualization/web.py +630 -0
- humancoin/visualization/web_visualizer.html +1611 -0
- humancoin-0.1.1.dist-info/METADATA +866 -0
- humancoin-0.1.1.dist-info/RECORD +21 -0
- humancoin-0.1.1.dist-info/WHEEL +5 -0
- humancoin-0.1.1.dist-info/entry_points.txt +4 -0
- humancoin-0.1.1.dist-info/licenses/LICENSE +201 -0
- humancoin-0.1.1.dist-info/top_level.txt +1 -0
humancoin/__init__.py
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""humancoin — multi-frequency extension for LeRobot datasets."""
|
|
2
|
+
|
|
3
|
+
from .utils import CODEBASE_VERSION
|
|
4
|
+
__version__ = "0.1.1"
|
|
5
|
+
|
|
6
|
+
__all__ = [
|
|
7
|
+
"HumanCOINDataset",
|
|
8
|
+
"HumanCOINDatasetMetadata",
|
|
9
|
+
"AudioFeature",
|
|
10
|
+
"CODEBASE_VERSION",
|
|
11
|
+
"__version__",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def __getattr__(name):
|
|
16
|
+
"""Keep the download CLI usable without importing torch or LeRobot first."""
|
|
17
|
+
from importlib import import_module
|
|
18
|
+
|
|
19
|
+
modules = {
|
|
20
|
+
"HumanCOINDataset": ".dataset",
|
|
21
|
+
"HumanCOINDatasetMetadata": ".metadata",
|
|
22
|
+
"AudioFeature": ".audio",
|
|
23
|
+
}
|
|
24
|
+
if name not in modules:
|
|
25
|
+
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
|
26
|
+
value = getattr(import_module(modules[name], __name__), name)
|
|
27
|
+
globals()[name] = value
|
|
28
|
+
return value
|
humancoin/audio.py
ADDED
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
"""AudioFeature: one microphone — buffer chunks, write WAV, write timestamp parquet."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import wave
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
import numpy as np
|
|
9
|
+
import pyarrow as pa
|
|
10
|
+
import pyarrow.parquet as pq
|
|
11
|
+
import torch
|
|
12
|
+
|
|
13
|
+
from .utils import DEFAULT_AUDIO_PATH, DEFAULT_DATA_PATH
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class AudioFeature:
|
|
17
|
+
"""One microphone: buffer audio chunks per episode, save WAV + timestamp parquet.
|
|
18
|
+
|
|
19
|
+
Mirrors VideoFeature: each ``add`` records one chunk (its timestamp goes
|
|
20
|
+
into the parquet index, one row per chunk); ``save`` concatenates the
|
|
21
|
+
chunks into a single PCM16 WAV per episode. ``read`` resolves the chunk
|
|
22
|
+
nearest a query timestamp through the parquet index, then slices exactly
|
|
23
|
+
that chunk out of the WAV.
|
|
24
|
+
|
|
25
|
+
Samples are float32 in [-1, 1]. A chunk is ``(n_samples,)`` for mono or
|
|
26
|
+
``(n_samples, channels)`` for multichannel audio.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(self, key: str, spec: dict, root: Path, window_override=None):
|
|
30
|
+
self.key = key
|
|
31
|
+
self.spec = spec
|
|
32
|
+
self.root = root
|
|
33
|
+
self._window_override = window_override
|
|
34
|
+
self._sample_rate = int(spec.get("sample_rate", 16000))
|
|
35
|
+
self._ep_idx = 0
|
|
36
|
+
self._chunk_count = 0
|
|
37
|
+
self._chunks: list[np.ndarray] = []
|
|
38
|
+
self._timestamps: list[float] = []
|
|
39
|
+
self._ts_cache: dict[int, np.ndarray] = {}
|
|
40
|
+
self._offsets_cache: dict[int, np.ndarray] = {}
|
|
41
|
+
self._data_cache: dict[int, tuple[np.ndarray, np.ndarray]] = {}
|
|
42
|
+
|
|
43
|
+
# ── Add / Save / Read ──
|
|
44
|
+
|
|
45
|
+
def add(self, samples: np.ndarray | torch.Tensor, timestamp: float) -> None:
|
|
46
|
+
if isinstance(samples, torch.Tensor):
|
|
47
|
+
samples = samples.cpu().numpy()
|
|
48
|
+
samples = np.asarray(samples, dtype=np.float32)
|
|
49
|
+
if samples.ndim > 2:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f"audio chunk for '{self.key}' must be 1-D mono or 2-D "
|
|
52
|
+
f"(n_samples, channels), got shape {samples.shape}"
|
|
53
|
+
)
|
|
54
|
+
self._chunks.append(samples)
|
|
55
|
+
self._timestamps.append(timestamp)
|
|
56
|
+
self._chunk_count += 1
|
|
57
|
+
|
|
58
|
+
def save(self) -> None:
|
|
59
|
+
if not self._chunks:
|
|
60
|
+
return
|
|
61
|
+
chunks_size = self.spec.get("chunks_size", 1000)
|
|
62
|
+
ep_chunk = self._ep_idx // chunks_size
|
|
63
|
+
n_samples = np.array([len(c) for c in self._chunks], dtype=np.int64)
|
|
64
|
+
ts_arr = np.array(self._timestamps, dtype=np.float64)
|
|
65
|
+
|
|
66
|
+
# Timestamp parquet — one row per chunk, same file layout as video
|
|
67
|
+
table = pa.table({
|
|
68
|
+
"timestamp": pa.array(ts_arr, type=pa.float64()),
|
|
69
|
+
"episode_index": pa.array(
|
|
70
|
+
np.full(len(ts_arr), self._ep_idx, dtype=np.int64), type=pa.int64()
|
|
71
|
+
),
|
|
72
|
+
"num_samples": pa.array(n_samples, type=pa.int64()),
|
|
73
|
+
})
|
|
74
|
+
fpath = self.root / DEFAULT_DATA_PATH.format(
|
|
75
|
+
episode_chunk=ep_chunk, episode_index=self._ep_idx, feature_key=self.key,
|
|
76
|
+
)
|
|
77
|
+
fpath.parent.mkdir(parents=True, exist_ok=True)
|
|
78
|
+
pq.write_table(table, fpath, compression="snappy")
|
|
79
|
+
|
|
80
|
+
# WAV
|
|
81
|
+
wav_path = self.root / DEFAULT_AUDIO_PATH.format(
|
|
82
|
+
episode_chunk=ep_chunk, audio_key=self.key, episode_index=self._ep_idx,
|
|
83
|
+
)
|
|
84
|
+
wav_path.parent.mkdir(parents=True, exist_ok=True)
|
|
85
|
+
self._write_wav(wav_path)
|
|
86
|
+
|
|
87
|
+
def _write_wav(self, fpath: Path) -> None:
|
|
88
|
+
interleaved, channels = self._to_pcm16(np.concatenate(self._chunks, axis=0))
|
|
89
|
+
with wave.open(str(fpath), "wb") as wf:
|
|
90
|
+
wf.setnchannels(channels)
|
|
91
|
+
wf.setsampwidth(2) # PCM16
|
|
92
|
+
wf.setframerate(self._sample_rate)
|
|
93
|
+
wf.writeframes(interleaved.tobytes())
|
|
94
|
+
|
|
95
|
+
def _to_pcm16(self, samples: np.ndarray) -> tuple[np.ndarray, int]:
|
|
96
|
+
"""float32 [-1, 1] → interleaved int16; returns (flat array, n_channels)."""
|
|
97
|
+
channels = 1 if samples.ndim == 1 else samples.shape[1]
|
|
98
|
+
np.clip(samples, -1.0, 1.0, out=samples)
|
|
99
|
+
ints = (samples * 32767.0).astype("<i2")
|
|
100
|
+
return ints.reshape(-1), channels
|
|
101
|
+
|
|
102
|
+
def compute_stats(self, sample_count: int = 10) -> dict | None:
|
|
103
|
+
"""Amplitude stats in [-1, 1] over sampled chunks (per channel)."""
|
|
104
|
+
n = self._chunk_count
|
|
105
|
+
if n == 0:
|
|
106
|
+
return None
|
|
107
|
+
indices = list(range(0, n, max(1, n // sample_count)))[:sample_count]
|
|
108
|
+
arrays = [self._chunks[i] for i in indices if len(self._chunks[i]) > 0]
|
|
109
|
+
if not arrays:
|
|
110
|
+
return None
|
|
111
|
+
stacked = np.concatenate(
|
|
112
|
+
[a if a.ndim == 2 else a[:, None] for a in arrays], axis=0
|
|
113
|
+
) # [total sampled samples, C]; chunks need not have equal lengths
|
|
114
|
+
axes = 0 # reduce over samples — keep channel
|
|
115
|
+
total = sum(len(c) for c in self._chunks)
|
|
116
|
+
return {
|
|
117
|
+
"min": stacked.min(axis=axes),
|
|
118
|
+
"max": stacked.max(axis=axes),
|
|
119
|
+
"mean": stacked.mean(axis=axes),
|
|
120
|
+
"std": stacked.std(axis=axes),
|
|
121
|
+
"count": np.array([total]),
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
def next_episode(self):
|
|
125
|
+
self._ep_idx += 1
|
|
126
|
+
self._chunk_count = 0
|
|
127
|
+
self._chunks = []
|
|
128
|
+
self._timestamps = []
|
|
129
|
+
|
|
130
|
+
def read(self, ep_idx: int, timestamp: float, window=None) -> torch.Tensor:
|
|
131
|
+
"""Read audio around ``timestamp``.
|
|
132
|
+
|
|
133
|
+
Two modes, aligned with the video path:
|
|
134
|
+
|
|
135
|
+
- nearest (default): the chunk whose stored timestamp is nearest to
|
|
136
|
+
``timestamp``, returned whole — the atomic unit added via
|
|
137
|
+
``add_frame``, analogous to one video frame.
|
|
138
|
+
- window ``(start, end)``: every sample whose time lies in
|
|
139
|
+
``(timestamp + start, timestamp + end]``, sliced with sample
|
|
140
|
+
precision across chunk boundaries and concatenated.
|
|
141
|
+
|
|
142
|
+
Timing contract: a chunk's first sample sits at its parquet label
|
|
143
|
+
and samples advance at exactly ``1/sample_rate``; consecutive
|
|
144
|
+
chunks are back-to-back. Real dropouts (missing content) simply
|
|
145
|
+
shorten the returned segment — content is never stretched.
|
|
146
|
+
"""
|
|
147
|
+
window = window if window is not None else self._resolve_window()
|
|
148
|
+
if isinstance(window, (list, tuple)) and len(window) == 2:
|
|
149
|
+
return self._read_window(ep_idx, timestamp,
|
|
150
|
+
float(window[0]), float(window[1]))
|
|
151
|
+
ts = self._load_timestamps(ep_idx)
|
|
152
|
+
if len(ts) == 0:
|
|
153
|
+
return self._empty()
|
|
154
|
+
idx = int(np.argmin(np.abs(ts - timestamp)))
|
|
155
|
+
offsets = self._load_offsets(ep_idx)
|
|
156
|
+
return self._read_pcm(ep_idx, int(offsets[idx]),
|
|
157
|
+
int(offsets[idx + 1] - offsets[idx]))
|
|
158
|
+
|
|
159
|
+
def load(self, ep_idx: int) -> tuple[np.ndarray, np.ndarray]:
|
|
160
|
+
"""Whole episode as ``(per-sample timestamps, values)``.
|
|
161
|
+
|
|
162
|
+
Mirrors ``ParquetFeature.load`` so generic consumers (web visualizer,
|
|
163
|
+
alignment checker) can treat audio like any other signal: timestamps
|
|
164
|
+
are per sample (chunk label + sample offset / sample_rate), values are
|
|
165
|
+
the full-episode PCM shaped ``(N, channels)`` in [-1, 1].
|
|
166
|
+
"""
|
|
167
|
+
if ep_idx not in self._data_cache:
|
|
168
|
+
ts = self._load_timestamps(ep_idx)
|
|
169
|
+
offsets = self._load_offsets(ep_idx)
|
|
170
|
+
n = int(offsets[-1])
|
|
171
|
+
if len(ts) == 0 or n == 0:
|
|
172
|
+
shape = self.spec.get("shape", (0,))
|
|
173
|
+
channels = shape[1] if len(shape) > 1 else 1
|
|
174
|
+
self._data_cache[ep_idx] = (
|
|
175
|
+
np.array([], dtype=np.float64),
|
|
176
|
+
np.zeros((0, channels), dtype=np.float32),
|
|
177
|
+
)
|
|
178
|
+
else:
|
|
179
|
+
vals = self._read_pcm(ep_idx, 0, n).numpy()
|
|
180
|
+
if vals.ndim == 1:
|
|
181
|
+
vals = vals[:, None]
|
|
182
|
+
ns = np.diff(offsets)
|
|
183
|
+
pts = (np.repeat(ts, ns)
|
|
184
|
+
+ (np.arange(n, dtype=np.float64)
|
|
185
|
+
- np.repeat(offsets[:-1], ns)) / self._sample_rate)
|
|
186
|
+
self._data_cache[ep_idx] = (pts, vals)
|
|
187
|
+
return self._data_cache[ep_idx]
|
|
188
|
+
|
|
189
|
+
def _resolve_window(self):
|
|
190
|
+
return self._window_override or self.spec.get("window")
|
|
191
|
+
|
|
192
|
+
def _read_window(self, ep_idx: int, timestamp: float, w0: float, w1: float):
|
|
193
|
+
ts = self._load_timestamps(ep_idx)
|
|
194
|
+
offsets = self._load_offsets(ep_idx)
|
|
195
|
+
if len(ts) == 0 or w1 <= w0:
|
|
196
|
+
return self._empty()
|
|
197
|
+
rate = self._sample_rate
|
|
198
|
+
n = len(ts)
|
|
199
|
+
chunk_ns = np.diff(offsets) # samples per chunk
|
|
200
|
+
lo, hi = timestamp + w0, timestamp + w1
|
|
201
|
+
|
|
202
|
+
# chunks whose label-time span can intersect (lo, hi];
|
|
203
|
+
# the per-chunk clamps below reject the rest (window past either end)
|
|
204
|
+
i0 = int(np.clip(np.searchsorted(ts, lo, side="right") - 1, 0, n - 1))
|
|
205
|
+
i1 = int(np.clip(np.searchsorted(ts, hi, side="left"), 0, n - 1))
|
|
206
|
+
|
|
207
|
+
ranges: list[list[int]] = []
|
|
208
|
+
for i in range(i0, i1 + 1):
|
|
209
|
+
nc = int(chunk_ns[i])
|
|
210
|
+
# sample j of chunk i has time ts[i] + j/rate; keep (lo, hi]
|
|
211
|
+
j0 = 0 if lo <= ts[i] else int(np.floor((lo - ts[i]) * rate)) + 1
|
|
212
|
+
j1 = (nc if hi >= ts[i] + (nc - 1) / rate
|
|
213
|
+
else int(np.floor((hi - ts[i]) * rate)) + 1)
|
|
214
|
+
j0, j1 = max(0, j0), max(0, min(j1, nc))
|
|
215
|
+
if j1 > j0:
|
|
216
|
+
ranges.append([int(offsets[i]) + j0, int(offsets[i]) + j1])
|
|
217
|
+
|
|
218
|
+
# merge sample-adjacent ranges → usually one contiguous seek+read
|
|
219
|
+
merged: list[list[int]] = []
|
|
220
|
+
for s, e in ranges:
|
|
221
|
+
if merged and s <= merged[-1][1]:
|
|
222
|
+
merged[-1][1] = max(merged[-1][1], e)
|
|
223
|
+
else:
|
|
224
|
+
merged.append([s, e])
|
|
225
|
+
|
|
226
|
+
parts = [self._read_pcm(ep_idx, s, e - s) for s, e in merged]
|
|
227
|
+
if not parts:
|
|
228
|
+
return self._empty()
|
|
229
|
+
if len(parts) == 1:
|
|
230
|
+
return parts[0]
|
|
231
|
+
return torch.from_numpy(np.concatenate(
|
|
232
|
+
[p.numpy() for p in parts], axis=0))
|
|
233
|
+
|
|
234
|
+
def _read_pcm(self, ep_idx: int, start: int, length: int) -> torch.Tensor:
|
|
235
|
+
"""Read ``length`` samples at ``start`` from the episode WAV."""
|
|
236
|
+
if length <= 0:
|
|
237
|
+
return self._empty()
|
|
238
|
+
ep_chunk = ep_idx // self.spec.get("chunks_size", 1000)
|
|
239
|
+
wav_path = self.root / DEFAULT_AUDIO_PATH.format(
|
|
240
|
+
episode_chunk=ep_chunk, audio_key=self.key, episode_index=ep_idx,
|
|
241
|
+
)
|
|
242
|
+
with wave.open(str(wav_path), "rb") as wf:
|
|
243
|
+
channels = wf.getnchannels()
|
|
244
|
+
wf.setpos(start)
|
|
245
|
+
raw = wf.readframes(length)
|
|
246
|
+
samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
|
|
247
|
+
if channels > 1:
|
|
248
|
+
samples = samples.reshape(-1, channels)
|
|
249
|
+
return torch.from_numpy(samples)
|
|
250
|
+
|
|
251
|
+
def _empty(self) -> torch.Tensor:
|
|
252
|
+
shape = self.spec.get("shape", (0,))
|
|
253
|
+
return torch.zeros(shape if len(shape) <= 1 else (0, shape[1]),
|
|
254
|
+
dtype=torch.float32)
|
|
255
|
+
|
|
256
|
+
def _load_timestamps(self, ep_idx: int) -> np.ndarray:
|
|
257
|
+
self._load_index(ep_idx)
|
|
258
|
+
return self._ts_cache[ep_idx]
|
|
259
|
+
|
|
260
|
+
def _load_offsets(self, ep_idx: int) -> np.ndarray:
|
|
261
|
+
self._load_index(ep_idx)
|
|
262
|
+
return self._offsets_cache[ep_idx]
|
|
263
|
+
|
|
264
|
+
def _load_index(self, ep_idx: int) -> None:
|
|
265
|
+
if ep_idx in self._ts_cache:
|
|
266
|
+
return
|
|
267
|
+
ep_chunk = ep_idx // self.spec.get("chunks_size", 1000)
|
|
268
|
+
fpath = self.root / DEFAULT_DATA_PATH.format(
|
|
269
|
+
episode_chunk=ep_chunk, episode_index=ep_idx, feature_key=self.key,
|
|
270
|
+
)
|
|
271
|
+
if fpath.exists():
|
|
272
|
+
table = pq.read_table(fpath)
|
|
273
|
+
if len(table):
|
|
274
|
+
self._ts_cache[ep_idx] = table.column("timestamp").to_numpy()
|
|
275
|
+
cum = np.concatenate([
|
|
276
|
+
np.array([0], dtype=np.int64),
|
|
277
|
+
np.cumsum(table.column("num_samples").to_numpy(), dtype=np.int64),
|
|
278
|
+
])
|
|
279
|
+
self._offsets_cache[ep_idx] = cum
|
|
280
|
+
return
|
|
281
|
+
self._ts_cache[ep_idx] = np.array([], dtype=np.float64)
|
|
282
|
+
self._offsets_cache[ep_idx] = np.array([0], dtype=np.int64)
|
humancoin/checker.py
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""Post-save validation: video frame count and timestamp alignment."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
import numpy as np
|
|
8
|
+
|
|
9
|
+
from lerobot.datasets.utils import append_jsonlines
|
|
10
|
+
|
|
11
|
+
from .utils import DEFAULT_AUDIO_PATH, DEFAULT_DATA_PATH, DEFAULT_VIDEO_PATH
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _nearest_indices(ts: np.ndarray, master_ts: np.ndarray) -> np.ndarray:
|
|
15
|
+
"""Index in ``ts`` nearest to each master timestamp.
|
|
16
|
+
|
|
17
|
+
Vectorised searchsorted equivalent of ``np.argmin(np.abs(ts - mt))``
|
|
18
|
+
per frame, which rescans the whole axis for every master frame —
|
|
19
|
+
O(frames × readings) and dominant for high-rate streams. Timestamps
|
|
20
|
+
are recorded monotonically, as the readers already assume. Ties
|
|
21
|
+
resolve to the earlier sample, matching argmin's first-minimum.
|
|
22
|
+
With duplicated timestamps the picked index may differ from argmin's
|
|
23
|
+
(a later copy of the same value), but the timestamp value and diff —
|
|
24
|
+
all the report uses — are identical.
|
|
25
|
+
"""
|
|
26
|
+
if len(ts) == 1:
|
|
27
|
+
return np.zeros(len(master_ts), dtype=np.int64)
|
|
28
|
+
hi = np.clip(np.searchsorted(ts, master_ts, side="left"), 1, len(ts) - 1)
|
|
29
|
+
pick_right = np.abs(ts[hi] - master_ts) < np.abs(master_ts - ts[hi - 1])
|
|
30
|
+
return np.where(pick_right, hi, hi - 1)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class DatasetChecker:
|
|
34
|
+
"""Validates saved episodes: video integrity and timestamp alignment."""
|
|
35
|
+
|
|
36
|
+
def __init__(self, dataset):
|
|
37
|
+
self.ds = dataset
|
|
38
|
+
|
|
39
|
+
def check_video_frames(
|
|
40
|
+
self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
|
|
41
|
+
) -> None:
|
|
42
|
+
chunks_size = self.ds.meta.info.get("chunks_size", 1000)
|
|
43
|
+
ep_chunk = ep_idx // chunks_size
|
|
44
|
+
for cam_key in self.ds.meta.camera_keys:
|
|
45
|
+
video_path = self.ds.root / DEFAULT_VIDEO_PATH.format(
|
|
46
|
+
episode_chunk=ep_chunk, video_key=cam_key, episode_index=ep_idx
|
|
47
|
+
)
|
|
48
|
+
if not video_path.exists():
|
|
49
|
+
print(f"[VIDEO] episode {ep_idx}, '{cam_key}': MISSING")
|
|
50
|
+
continue
|
|
51
|
+
try:
|
|
52
|
+
import av
|
|
53
|
+
container = av.open(str(video_path))
|
|
54
|
+
video_stream = next(s for s in container.streams if s.type == "video")
|
|
55
|
+
nb_frames = video_stream.frames or 0
|
|
56
|
+
fps_stream = float(video_stream.average_rate or 0)
|
|
57
|
+
time_base = float(video_stream.time_base)
|
|
58
|
+
duration_s = float(video_stream.duration or 0) * time_base
|
|
59
|
+
container.close()
|
|
60
|
+
if nb_frames != episode_length:
|
|
61
|
+
print(f"[VIDEO] episode {ep_idx}, '{cam_key}': "
|
|
62
|
+
f"frame count {nb_frames} != {episode_length}")
|
|
63
|
+
else:
|
|
64
|
+
print(f"[VIDEO] episode {ep_idx}, '{cam_key}': "
|
|
65
|
+
f"OK ({nb_frames}f, {fps_stream:.1f}fps, "
|
|
66
|
+
f"duration {duration_s:.2f}s)")
|
|
67
|
+
except ImportError:
|
|
68
|
+
pass
|
|
69
|
+
except Exception as e:
|
|
70
|
+
print(f"[VIDEO] episode {ep_idx}, '{cam_key}': error — {e}")
|
|
71
|
+
|
|
72
|
+
def check_audio_frames(
|
|
73
|
+
self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
|
|
74
|
+
) -> None:
|
|
75
|
+
"""WAV integrity: chunk count vs index rows, sample count vs summed index."""
|
|
76
|
+
import wave
|
|
77
|
+
|
|
78
|
+
chunks_size = self.ds.meta.info.get("chunks_size", 1000)
|
|
79
|
+
ep_chunk = ep_idx // chunks_size
|
|
80
|
+
for audio_key in self._audio_keys():
|
|
81
|
+
wav_path = self.ds.root / DEFAULT_AUDIO_PATH.format(
|
|
82
|
+
episode_chunk=ep_chunk, audio_key=audio_key, episode_index=ep_idx
|
|
83
|
+
)
|
|
84
|
+
if not wav_path.exists():
|
|
85
|
+
print(f"[AUDIO] episode {ep_idx}, '{audio_key}': MISSING")
|
|
86
|
+
continue
|
|
87
|
+
index_path = self.ds.root / DEFAULT_DATA_PATH.format(
|
|
88
|
+
episode_chunk=ep_chunk, episode_index=ep_idx, feature_key=audio_key
|
|
89
|
+
)
|
|
90
|
+
try:
|
|
91
|
+
import pyarrow.parquet as pq
|
|
92
|
+
|
|
93
|
+
idx_table = pq.read_table(index_path)
|
|
94
|
+
n_rows = len(idx_table)
|
|
95
|
+
index_samples = int(
|
|
96
|
+
np.sum(idx_table.column("num_samples").to_numpy())
|
|
97
|
+
)
|
|
98
|
+
with wave.open(str(wav_path), "rb") as wf:
|
|
99
|
+
sample_rate = wf.getframerate()
|
|
100
|
+
channels = wf.getnchannels()
|
|
101
|
+
n_samples = wf.getnframes()
|
|
102
|
+
duration_s = n_samples / sample_rate
|
|
103
|
+
status = "OK" if n_samples == index_samples else (
|
|
104
|
+
f"sample count {n_samples} != index total {index_samples}"
|
|
105
|
+
)
|
|
106
|
+
print(f"[AUDIO] episode {ep_idx}, '{audio_key}': {status} "
|
|
107
|
+
f"({n_rows} chunks, {channels}ch, {sample_rate}Hz, "
|
|
108
|
+
f"{duration_s:.2f}s)")
|
|
109
|
+
except Exception as e:
|
|
110
|
+
print(f"[AUDIO] episode {ep_idx}, '{audio_key}': error — {e}")
|
|
111
|
+
|
|
112
|
+
def _audio_keys(self) -> list[str]:
|
|
113
|
+
return [
|
|
114
|
+
key for key, ft in self.ds.meta.features.items()
|
|
115
|
+
if ft.get("dtype") == "audio"
|
|
116
|
+
]
|
|
117
|
+
|
|
118
|
+
def check_episode_alignment(
|
|
119
|
+
self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
|
|
120
|
+
) -> None:
|
|
121
|
+
master_ts = np.array(
|
|
122
|
+
[r["timestamp"] for r in frame_records], dtype=np.float64
|
|
123
|
+
)
|
|
124
|
+
report_path = self.ds.root / "meta" / "alignment_check.jsonl"
|
|
125
|
+
|
|
126
|
+
for key in self.ds.meta.data_keys:
|
|
127
|
+
ft = self.ds.meta.features.get(key, {})
|
|
128
|
+
if ft.get("dtype") in ("video", "image"):
|
|
129
|
+
continue
|
|
130
|
+
tolerance = ft.get("tolerance_s", None)
|
|
131
|
+
if tolerance is None:
|
|
132
|
+
continue
|
|
133
|
+
f = self.ds._features.get(key)
|
|
134
|
+
if f is None:
|
|
135
|
+
continue
|
|
136
|
+
ts, _ = f.load(ep_idx)
|
|
137
|
+
if len(ts) == 0:
|
|
138
|
+
continue
|
|
139
|
+
|
|
140
|
+
idx = _nearest_indices(ts, master_ts)
|
|
141
|
+
diffs = np.abs(ts[idx] - master_ts)
|
|
142
|
+
violations = [
|
|
143
|
+
{
|
|
144
|
+
"frame": int(fi),
|
|
145
|
+
"t_master": round(float(master_ts[fi]), 6),
|
|
146
|
+
"t_nearest": round(float(ts[idx[fi]]), 6),
|
|
147
|
+
"diff": round(float(diffs[fi]), 6),
|
|
148
|
+
}
|
|
149
|
+
for fi in np.nonzero(diffs > tolerance)[0]
|
|
150
|
+
]
|
|
151
|
+
|
|
152
|
+
if violations:
|
|
153
|
+
report = {
|
|
154
|
+
"episode_index": ep_idx, "feature": key,
|
|
155
|
+
"fps": ft.get("fps", None),
|
|
156
|
+
"tolerance_s": tolerance,
|
|
157
|
+
"total_frames": episode_length,
|
|
158
|
+
"total_readings": len(ts),
|
|
159
|
+
"violations": len(violations),
|
|
160
|
+
"max_diff": round(float(diffs.max()), 6),
|
|
161
|
+
"details": violations,
|
|
162
|
+
}
|
|
163
|
+
report_path.parent.mkdir(parents=True, exist_ok=True)
|
|
164
|
+
append_jsonlines(report, report_path)
|
|
165
|
+
print(
|
|
166
|
+
f"[ALIGN] episode {ep_idx}, feature '{key}': "
|
|
167
|
+
f"{len(violations)}/{episode_length} frames exceed "
|
|
168
|
+
f"tolerance_s={tolerance}:"
|
|
169
|
+
)
|
|
170
|
+
for v in violations[:5]:
|
|
171
|
+
print(f" frame {v['frame']} (t={v['t_master']:.4f}) "
|
|
172
|
+
f"nearest at t={v['t_nearest']:.4f}, "
|
|
173
|
+
f"diff={v['diff']:.4f}s")
|
|
174
|
+
if len(violations) > 5:
|
|
175
|
+
print(f" ... ({len(violations) - 5} more)")
|
|
176
|
+
else:
|
|
177
|
+
print(f"[ALIGN] episode {ep_idx}, feature '{key}': "
|
|
178
|
+
f"OK ({episode_length} frames within tolerance_s={tolerance})")
|