humancoin 0.1.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
humancoin/__init__.py ADDED
@@ -0,0 +1,28 @@
1
+ """humancoin — multi-frequency extension for LeRobot datasets."""
2
+
3
+ from .utils import CODEBASE_VERSION
4
+ __version__ = "0.1.1"
5
+
6
+ __all__ = [
7
+ "HumanCOINDataset",
8
+ "HumanCOINDatasetMetadata",
9
+ "AudioFeature",
10
+ "CODEBASE_VERSION",
11
+ "__version__",
12
+ ]
13
+
14
+
15
+ def __getattr__(name):
16
+ """Keep the download CLI usable without importing torch or LeRobot first."""
17
+ from importlib import import_module
18
+
19
+ modules = {
20
+ "HumanCOINDataset": ".dataset",
21
+ "HumanCOINDatasetMetadata": ".metadata",
22
+ "AudioFeature": ".audio",
23
+ }
24
+ if name not in modules:
25
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
26
+ value = getattr(import_module(modules[name], __name__), name)
27
+ globals()[name] = value
28
+ return value
humancoin/audio.py ADDED
@@ -0,0 +1,282 @@
1
+ """AudioFeature: one microphone — buffer chunks, write WAV, write timestamp parquet."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import wave
6
+ from pathlib import Path
7
+
8
+ import numpy as np
9
+ import pyarrow as pa
10
+ import pyarrow.parquet as pq
11
+ import torch
12
+
13
+ from .utils import DEFAULT_AUDIO_PATH, DEFAULT_DATA_PATH
14
+
15
+
16
+ class AudioFeature:
17
+ """One microphone: buffer audio chunks per episode, save WAV + timestamp parquet.
18
+
19
+ Mirrors VideoFeature: each ``add`` records one chunk (its timestamp goes
20
+ into the parquet index, one row per chunk); ``save`` concatenates the
21
+ chunks into a single PCM16 WAV per episode. ``read`` resolves the chunk
22
+ nearest a query timestamp through the parquet index, then slices exactly
23
+ that chunk out of the WAV.
24
+
25
+ Samples are float32 in [-1, 1]. A chunk is ``(n_samples,)`` for mono or
26
+ ``(n_samples, channels)`` for multichannel audio.
27
+ """
28
+
29
+ def __init__(self, key: str, spec: dict, root: Path, window_override=None):
30
+ self.key = key
31
+ self.spec = spec
32
+ self.root = root
33
+ self._window_override = window_override
34
+ self._sample_rate = int(spec.get("sample_rate", 16000))
35
+ self._ep_idx = 0
36
+ self._chunk_count = 0
37
+ self._chunks: list[np.ndarray] = []
38
+ self._timestamps: list[float] = []
39
+ self._ts_cache: dict[int, np.ndarray] = {}
40
+ self._offsets_cache: dict[int, np.ndarray] = {}
41
+ self._data_cache: dict[int, tuple[np.ndarray, np.ndarray]] = {}
42
+
43
+ # ── Add / Save / Read ──
44
+
45
+ def add(self, samples: np.ndarray | torch.Tensor, timestamp: float) -> None:
46
+ if isinstance(samples, torch.Tensor):
47
+ samples = samples.cpu().numpy()
48
+ samples = np.asarray(samples, dtype=np.float32)
49
+ if samples.ndim > 2:
50
+ raise ValueError(
51
+ f"audio chunk for '{self.key}' must be 1-D mono or 2-D "
52
+ f"(n_samples, channels), got shape {samples.shape}"
53
+ )
54
+ self._chunks.append(samples)
55
+ self._timestamps.append(timestamp)
56
+ self._chunk_count += 1
57
+
58
+ def save(self) -> None:
59
+ if not self._chunks:
60
+ return
61
+ chunks_size = self.spec.get("chunks_size", 1000)
62
+ ep_chunk = self._ep_idx // chunks_size
63
+ n_samples = np.array([len(c) for c in self._chunks], dtype=np.int64)
64
+ ts_arr = np.array(self._timestamps, dtype=np.float64)
65
+
66
+ # Timestamp parquet — one row per chunk, same file layout as video
67
+ table = pa.table({
68
+ "timestamp": pa.array(ts_arr, type=pa.float64()),
69
+ "episode_index": pa.array(
70
+ np.full(len(ts_arr), self._ep_idx, dtype=np.int64), type=pa.int64()
71
+ ),
72
+ "num_samples": pa.array(n_samples, type=pa.int64()),
73
+ })
74
+ fpath = self.root / DEFAULT_DATA_PATH.format(
75
+ episode_chunk=ep_chunk, episode_index=self._ep_idx, feature_key=self.key,
76
+ )
77
+ fpath.parent.mkdir(parents=True, exist_ok=True)
78
+ pq.write_table(table, fpath, compression="snappy")
79
+
80
+ # WAV
81
+ wav_path = self.root / DEFAULT_AUDIO_PATH.format(
82
+ episode_chunk=ep_chunk, audio_key=self.key, episode_index=self._ep_idx,
83
+ )
84
+ wav_path.parent.mkdir(parents=True, exist_ok=True)
85
+ self._write_wav(wav_path)
86
+
87
+ def _write_wav(self, fpath: Path) -> None:
88
+ interleaved, channels = self._to_pcm16(np.concatenate(self._chunks, axis=0))
89
+ with wave.open(str(fpath), "wb") as wf:
90
+ wf.setnchannels(channels)
91
+ wf.setsampwidth(2) # PCM16
92
+ wf.setframerate(self._sample_rate)
93
+ wf.writeframes(interleaved.tobytes())
94
+
95
+ def _to_pcm16(self, samples: np.ndarray) -> tuple[np.ndarray, int]:
96
+ """float32 [-1, 1] → interleaved int16; returns (flat array, n_channels)."""
97
+ channels = 1 if samples.ndim == 1 else samples.shape[1]
98
+ np.clip(samples, -1.0, 1.0, out=samples)
99
+ ints = (samples * 32767.0).astype("<i2")
100
+ return ints.reshape(-1), channels
101
+
102
+ def compute_stats(self, sample_count: int = 10) -> dict | None:
103
+ """Amplitude stats in [-1, 1] over sampled chunks (per channel)."""
104
+ n = self._chunk_count
105
+ if n == 0:
106
+ return None
107
+ indices = list(range(0, n, max(1, n // sample_count)))[:sample_count]
108
+ arrays = [self._chunks[i] for i in indices if len(self._chunks[i]) > 0]
109
+ if not arrays:
110
+ return None
111
+ stacked = np.concatenate(
112
+ [a if a.ndim == 2 else a[:, None] for a in arrays], axis=0
113
+ ) # [total sampled samples, C]; chunks need not have equal lengths
114
+ axes = 0 # reduce over samples — keep channel
115
+ total = sum(len(c) for c in self._chunks)
116
+ return {
117
+ "min": stacked.min(axis=axes),
118
+ "max": stacked.max(axis=axes),
119
+ "mean": stacked.mean(axis=axes),
120
+ "std": stacked.std(axis=axes),
121
+ "count": np.array([total]),
122
+ }
123
+
124
+ def next_episode(self):
125
+ self._ep_idx += 1
126
+ self._chunk_count = 0
127
+ self._chunks = []
128
+ self._timestamps = []
129
+
130
+ def read(self, ep_idx: int, timestamp: float, window=None) -> torch.Tensor:
131
+ """Read audio around ``timestamp``.
132
+
133
+ Two modes, aligned with the video path:
134
+
135
+ - nearest (default): the chunk whose stored timestamp is nearest to
136
+ ``timestamp``, returned whole — the atomic unit added via
137
+ ``add_frame``, analogous to one video frame.
138
+ - window ``(start, end)``: every sample whose time lies in
139
+ ``(timestamp + start, timestamp + end]``, sliced with sample
140
+ precision across chunk boundaries and concatenated.
141
+
142
+ Timing contract: a chunk's first sample sits at its parquet label
143
+ and samples advance at exactly ``1/sample_rate``; consecutive
144
+ chunks are back-to-back. Real dropouts (missing content) simply
145
+ shorten the returned segment — content is never stretched.
146
+ """
147
+ window = window if window is not None else self._resolve_window()
148
+ if isinstance(window, (list, tuple)) and len(window) == 2:
149
+ return self._read_window(ep_idx, timestamp,
150
+ float(window[0]), float(window[1]))
151
+ ts = self._load_timestamps(ep_idx)
152
+ if len(ts) == 0:
153
+ return self._empty()
154
+ idx = int(np.argmin(np.abs(ts - timestamp)))
155
+ offsets = self._load_offsets(ep_idx)
156
+ return self._read_pcm(ep_idx, int(offsets[idx]),
157
+ int(offsets[idx + 1] - offsets[idx]))
158
+
159
+ def load(self, ep_idx: int) -> tuple[np.ndarray, np.ndarray]:
160
+ """Whole episode as ``(per-sample timestamps, values)``.
161
+
162
+ Mirrors ``ParquetFeature.load`` so generic consumers (web visualizer,
163
+ alignment checker) can treat audio like any other signal: timestamps
164
+ are per sample (chunk label + sample offset / sample_rate), values are
165
+ the full-episode PCM shaped ``(N, channels)`` in [-1, 1].
166
+ """
167
+ if ep_idx not in self._data_cache:
168
+ ts = self._load_timestamps(ep_idx)
169
+ offsets = self._load_offsets(ep_idx)
170
+ n = int(offsets[-1])
171
+ if len(ts) == 0 or n == 0:
172
+ shape = self.spec.get("shape", (0,))
173
+ channels = shape[1] if len(shape) > 1 else 1
174
+ self._data_cache[ep_idx] = (
175
+ np.array([], dtype=np.float64),
176
+ np.zeros((0, channels), dtype=np.float32),
177
+ )
178
+ else:
179
+ vals = self._read_pcm(ep_idx, 0, n).numpy()
180
+ if vals.ndim == 1:
181
+ vals = vals[:, None]
182
+ ns = np.diff(offsets)
183
+ pts = (np.repeat(ts, ns)
184
+ + (np.arange(n, dtype=np.float64)
185
+ - np.repeat(offsets[:-1], ns)) / self._sample_rate)
186
+ self._data_cache[ep_idx] = (pts, vals)
187
+ return self._data_cache[ep_idx]
188
+
189
+ def _resolve_window(self):
190
+ return self._window_override or self.spec.get("window")
191
+
192
+ def _read_window(self, ep_idx: int, timestamp: float, w0: float, w1: float):
193
+ ts = self._load_timestamps(ep_idx)
194
+ offsets = self._load_offsets(ep_idx)
195
+ if len(ts) == 0 or w1 <= w0:
196
+ return self._empty()
197
+ rate = self._sample_rate
198
+ n = len(ts)
199
+ chunk_ns = np.diff(offsets) # samples per chunk
200
+ lo, hi = timestamp + w0, timestamp + w1
201
+
202
+ # chunks whose label-time span can intersect (lo, hi];
203
+ # the per-chunk clamps below reject the rest (window past either end)
204
+ i0 = int(np.clip(np.searchsorted(ts, lo, side="right") - 1, 0, n - 1))
205
+ i1 = int(np.clip(np.searchsorted(ts, hi, side="left"), 0, n - 1))
206
+
207
+ ranges: list[list[int]] = []
208
+ for i in range(i0, i1 + 1):
209
+ nc = int(chunk_ns[i])
210
+ # sample j of chunk i has time ts[i] + j/rate; keep (lo, hi]
211
+ j0 = 0 if lo <= ts[i] else int(np.floor((lo - ts[i]) * rate)) + 1
212
+ j1 = (nc if hi >= ts[i] + (nc - 1) / rate
213
+ else int(np.floor((hi - ts[i]) * rate)) + 1)
214
+ j0, j1 = max(0, j0), max(0, min(j1, nc))
215
+ if j1 > j0:
216
+ ranges.append([int(offsets[i]) + j0, int(offsets[i]) + j1])
217
+
218
+ # merge sample-adjacent ranges → usually one contiguous seek+read
219
+ merged: list[list[int]] = []
220
+ for s, e in ranges:
221
+ if merged and s <= merged[-1][1]:
222
+ merged[-1][1] = max(merged[-1][1], e)
223
+ else:
224
+ merged.append([s, e])
225
+
226
+ parts = [self._read_pcm(ep_idx, s, e - s) for s, e in merged]
227
+ if not parts:
228
+ return self._empty()
229
+ if len(parts) == 1:
230
+ return parts[0]
231
+ return torch.from_numpy(np.concatenate(
232
+ [p.numpy() for p in parts], axis=0))
233
+
234
+ def _read_pcm(self, ep_idx: int, start: int, length: int) -> torch.Tensor:
235
+ """Read ``length`` samples at ``start`` from the episode WAV."""
236
+ if length <= 0:
237
+ return self._empty()
238
+ ep_chunk = ep_idx // self.spec.get("chunks_size", 1000)
239
+ wav_path = self.root / DEFAULT_AUDIO_PATH.format(
240
+ episode_chunk=ep_chunk, audio_key=self.key, episode_index=ep_idx,
241
+ )
242
+ with wave.open(str(wav_path), "rb") as wf:
243
+ channels = wf.getnchannels()
244
+ wf.setpos(start)
245
+ raw = wf.readframes(length)
246
+ samples = np.frombuffer(raw, dtype="<i2").astype(np.float32) / 32768.0
247
+ if channels > 1:
248
+ samples = samples.reshape(-1, channels)
249
+ return torch.from_numpy(samples)
250
+
251
+ def _empty(self) -> torch.Tensor:
252
+ shape = self.spec.get("shape", (0,))
253
+ return torch.zeros(shape if len(shape) <= 1 else (0, shape[1]),
254
+ dtype=torch.float32)
255
+
256
+ def _load_timestamps(self, ep_idx: int) -> np.ndarray:
257
+ self._load_index(ep_idx)
258
+ return self._ts_cache[ep_idx]
259
+
260
+ def _load_offsets(self, ep_idx: int) -> np.ndarray:
261
+ self._load_index(ep_idx)
262
+ return self._offsets_cache[ep_idx]
263
+
264
+ def _load_index(self, ep_idx: int) -> None:
265
+ if ep_idx in self._ts_cache:
266
+ return
267
+ ep_chunk = ep_idx // self.spec.get("chunks_size", 1000)
268
+ fpath = self.root / DEFAULT_DATA_PATH.format(
269
+ episode_chunk=ep_chunk, episode_index=ep_idx, feature_key=self.key,
270
+ )
271
+ if fpath.exists():
272
+ table = pq.read_table(fpath)
273
+ if len(table):
274
+ self._ts_cache[ep_idx] = table.column("timestamp").to_numpy()
275
+ cum = np.concatenate([
276
+ np.array([0], dtype=np.int64),
277
+ np.cumsum(table.column("num_samples").to_numpy(), dtype=np.int64),
278
+ ])
279
+ self._offsets_cache[ep_idx] = cum
280
+ return
281
+ self._ts_cache[ep_idx] = np.array([], dtype=np.float64)
282
+ self._offsets_cache[ep_idx] = np.array([0], dtype=np.int64)
humancoin/checker.py ADDED
@@ -0,0 +1,178 @@
1
+ """Post-save validation: video frame count and timestamp alignment."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Any
6
+
7
+ import numpy as np
8
+
9
+ from lerobot.datasets.utils import append_jsonlines
10
+
11
+ from .utils import DEFAULT_AUDIO_PATH, DEFAULT_DATA_PATH, DEFAULT_VIDEO_PATH
12
+
13
+
14
+ def _nearest_indices(ts: np.ndarray, master_ts: np.ndarray) -> np.ndarray:
15
+ """Index in ``ts`` nearest to each master timestamp.
16
+
17
+ Vectorised searchsorted equivalent of ``np.argmin(np.abs(ts - mt))``
18
+ per frame, which rescans the whole axis for every master frame —
19
+ O(frames × readings) and dominant for high-rate streams. Timestamps
20
+ are recorded monotonically, as the readers already assume. Ties
21
+ resolve to the earlier sample, matching argmin's first-minimum.
22
+ With duplicated timestamps the picked index may differ from argmin's
23
+ (a later copy of the same value), but the timestamp value and diff —
24
+ all the report uses — are identical.
25
+ """
26
+ if len(ts) == 1:
27
+ return np.zeros(len(master_ts), dtype=np.int64)
28
+ hi = np.clip(np.searchsorted(ts, master_ts, side="left"), 1, len(ts) - 1)
29
+ pick_right = np.abs(ts[hi] - master_ts) < np.abs(master_ts - ts[hi - 1])
30
+ return np.where(pick_right, hi, hi - 1)
31
+
32
+
33
+ class DatasetChecker:
34
+ """Validates saved episodes: video integrity and timestamp alignment."""
35
+
36
+ def __init__(self, dataset):
37
+ self.ds = dataset
38
+
39
+ def check_video_frames(
40
+ self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
41
+ ) -> None:
42
+ chunks_size = self.ds.meta.info.get("chunks_size", 1000)
43
+ ep_chunk = ep_idx // chunks_size
44
+ for cam_key in self.ds.meta.camera_keys:
45
+ video_path = self.ds.root / DEFAULT_VIDEO_PATH.format(
46
+ episode_chunk=ep_chunk, video_key=cam_key, episode_index=ep_idx
47
+ )
48
+ if not video_path.exists():
49
+ print(f"[VIDEO] episode {ep_idx}, '{cam_key}': MISSING")
50
+ continue
51
+ try:
52
+ import av
53
+ container = av.open(str(video_path))
54
+ video_stream = next(s for s in container.streams if s.type == "video")
55
+ nb_frames = video_stream.frames or 0
56
+ fps_stream = float(video_stream.average_rate or 0)
57
+ time_base = float(video_stream.time_base)
58
+ duration_s = float(video_stream.duration or 0) * time_base
59
+ container.close()
60
+ if nb_frames != episode_length:
61
+ print(f"[VIDEO] episode {ep_idx}, '{cam_key}': "
62
+ f"frame count {nb_frames} != {episode_length}")
63
+ else:
64
+ print(f"[VIDEO] episode {ep_idx}, '{cam_key}': "
65
+ f"OK ({nb_frames}f, {fps_stream:.1f}fps, "
66
+ f"duration {duration_s:.2f}s)")
67
+ except ImportError:
68
+ pass
69
+ except Exception as e:
70
+ print(f"[VIDEO] episode {ep_idx}, '{cam_key}': error — {e}")
71
+
72
+ def check_audio_frames(
73
+ self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
74
+ ) -> None:
75
+ """WAV integrity: chunk count vs index rows, sample count vs summed index."""
76
+ import wave
77
+
78
+ chunks_size = self.ds.meta.info.get("chunks_size", 1000)
79
+ ep_chunk = ep_idx // chunks_size
80
+ for audio_key in self._audio_keys():
81
+ wav_path = self.ds.root / DEFAULT_AUDIO_PATH.format(
82
+ episode_chunk=ep_chunk, audio_key=audio_key, episode_index=ep_idx
83
+ )
84
+ if not wav_path.exists():
85
+ print(f"[AUDIO] episode {ep_idx}, '{audio_key}': MISSING")
86
+ continue
87
+ index_path = self.ds.root / DEFAULT_DATA_PATH.format(
88
+ episode_chunk=ep_chunk, episode_index=ep_idx, feature_key=audio_key
89
+ )
90
+ try:
91
+ import pyarrow.parquet as pq
92
+
93
+ idx_table = pq.read_table(index_path)
94
+ n_rows = len(idx_table)
95
+ index_samples = int(
96
+ np.sum(idx_table.column("num_samples").to_numpy())
97
+ )
98
+ with wave.open(str(wav_path), "rb") as wf:
99
+ sample_rate = wf.getframerate()
100
+ channels = wf.getnchannels()
101
+ n_samples = wf.getnframes()
102
+ duration_s = n_samples / sample_rate
103
+ status = "OK" if n_samples == index_samples else (
104
+ f"sample count {n_samples} != index total {index_samples}"
105
+ )
106
+ print(f"[AUDIO] episode {ep_idx}, '{audio_key}': {status} "
107
+ f"({n_rows} chunks, {channels}ch, {sample_rate}Hz, "
108
+ f"{duration_s:.2f}s)")
109
+ except Exception as e:
110
+ print(f"[AUDIO] episode {ep_idx}, '{audio_key}': error — {e}")
111
+
112
+ def _audio_keys(self) -> list[str]:
113
+ return [
114
+ key for key, ft in self.ds.meta.features.items()
115
+ if ft.get("dtype") == "audio"
116
+ ]
117
+
118
+ def check_episode_alignment(
119
+ self, ep_idx: int, episode_length: int, frame_records: list[dict[str, Any]]
120
+ ) -> None:
121
+ master_ts = np.array(
122
+ [r["timestamp"] for r in frame_records], dtype=np.float64
123
+ )
124
+ report_path = self.ds.root / "meta" / "alignment_check.jsonl"
125
+
126
+ for key in self.ds.meta.data_keys:
127
+ ft = self.ds.meta.features.get(key, {})
128
+ if ft.get("dtype") in ("video", "image"):
129
+ continue
130
+ tolerance = ft.get("tolerance_s", None)
131
+ if tolerance is None:
132
+ continue
133
+ f = self.ds._features.get(key)
134
+ if f is None:
135
+ continue
136
+ ts, _ = f.load(ep_idx)
137
+ if len(ts) == 0:
138
+ continue
139
+
140
+ idx = _nearest_indices(ts, master_ts)
141
+ diffs = np.abs(ts[idx] - master_ts)
142
+ violations = [
143
+ {
144
+ "frame": int(fi),
145
+ "t_master": round(float(master_ts[fi]), 6),
146
+ "t_nearest": round(float(ts[idx[fi]]), 6),
147
+ "diff": round(float(diffs[fi]), 6),
148
+ }
149
+ for fi in np.nonzero(diffs > tolerance)[0]
150
+ ]
151
+
152
+ if violations:
153
+ report = {
154
+ "episode_index": ep_idx, "feature": key,
155
+ "fps": ft.get("fps", None),
156
+ "tolerance_s": tolerance,
157
+ "total_frames": episode_length,
158
+ "total_readings": len(ts),
159
+ "violations": len(violations),
160
+ "max_diff": round(float(diffs.max()), 6),
161
+ "details": violations,
162
+ }
163
+ report_path.parent.mkdir(parents=True, exist_ok=True)
164
+ append_jsonlines(report, report_path)
165
+ print(
166
+ f"[ALIGN] episode {ep_idx}, feature '{key}': "
167
+ f"{len(violations)}/{episode_length} frames exceed "
168
+ f"tolerance_s={tolerance}:"
169
+ )
170
+ for v in violations[:5]:
171
+ print(f" frame {v['frame']} (t={v['t_master']:.4f}) "
172
+ f"nearest at t={v['t_nearest']:.4f}, "
173
+ f"diff={v['diff']:.4f}s")
174
+ if len(violations) > 5:
175
+ print(f" ... ({len(violations) - 5} more)")
176
+ else:
177
+ print(f"[ALIGN] episode {ep_idx}, feature '{key}': "
178
+ f"OK ({episode_length} frames within tolerance_s={tolerance})")