tassello 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +16 -0
- package/README.md +40 -8
- package/dist/server/behavior_tree_playback.js +2 -2
- package/dist/server/behavior_trees.d.ts +5 -3
- package/dist/server/behavior_trees.js +33 -35
- package/dist/server/rosbag.d.ts +15 -11
- package/dist/server/rosbag.js +396 -144
- package/dist/server/rosbag_format.d.ts +28 -0
- package/dist/server/rosbag_format.js +366 -0
- package/dist/server/rosbag_process.d.ts +20 -0
- package/dist/server/rosbag_process.js +99 -0
- package/dist/server/rosbags.d.ts +42 -0
- package/dist/server/rosbags.js +444 -0
- package/dist/server/server.js +5 -2
- package/dist/server/session.d.ts +2 -0
- package/dist/server/session.js +11 -1
- package/dist/shared/protocol.d.ts +24 -2
- package/dist/shared/rosbag.d.ts +104 -0
- package/dist/shared/rosbag.js +35 -0
- package/dist/types/client/connection.d.ts +17 -2
- package/dist/types/client/sdk/index.d.ts +2 -1
- package/dist/types/shared/protocol.d.ts +24 -2
- package/dist/types/shared/rosbag.d.ts +104 -0
- package/dist/www/assets/icons.js +1 -1
- package/dist/www/assets/index.css +1 -1
- package/dist/www/assets/index.js +40 -40
- package/docs/plugins.md +3 -1
- package/package.json +1 -1
package/dist/server/rosbag.js
CHANGED
|
@@ -1,15 +1,26 @@
|
|
|
1
|
-
// Rosbags (rosbag2)
|
|
2
|
-
// the messages of some of
|
|
3
|
-
//
|
|
4
|
-
//
|
|
5
|
-
|
|
6
|
-
|
|
1
|
+
// Rosbags (rosbag2) as files: what the Rosbags widget lists of a folder's
|
|
2
|
+
// bags (rosbags.ts), and the messages of some of a bag's topics, which the
|
|
3
|
+
// behavior tree inspectors play back. Either storage is read (MCAP, Jazzy's
|
|
4
|
+
// default, or SQLite, Humble's), compressed by rosbag2 or not (zstd, of each
|
|
5
|
+
// file or of each message); messages come serialized as recorded. Recording
|
|
6
|
+
// and playing them is ros2 bag's (rosbag_process.ts).
|
|
7
|
+
import { createReadStream, createWriteStream, promises as fs } from 'node:fs';
|
|
7
8
|
import os from 'node:os';
|
|
8
9
|
import path from 'node:path';
|
|
10
|
+
import { pipeline } from 'node:stream/promises';
|
|
9
11
|
import zlib from 'node:zlib';
|
|
10
|
-
import {
|
|
11
|
-
|
|
12
|
-
|
|
12
|
+
import { parseMetadata } from './rosbag_format.js';
|
|
13
|
+
import { errorMessage } from './util.js';
|
|
14
|
+
// The files of a bag, compressed by rosbag2 (.zstd) or not
|
|
15
|
+
const BAG_FILE = /\.(mcap|db3)(\.zstd)?$/i;
|
|
16
|
+
const COMPRESSED = /\.zstd$/i;
|
|
17
|
+
const METADATA = 'metadata.yaml';
|
|
18
|
+
// SQLite bags without metadata are counted only up to this size: node:sqlite
|
|
19
|
+
// blocks the server while it reads
|
|
20
|
+
const SQLITE_SCAN_BYTES = 64 * 1024 * 1024;
|
|
21
|
+
const natural = (a, b) => a.localeCompare(b, undefined, { numeric: true });
|
|
22
|
+
// An absolute path from /… or ~/…
|
|
23
|
+
export function bagPath(input, home = os.homedir()) {
|
|
13
24
|
const trimmed = input.trim();
|
|
14
25
|
const expanded = trimmed === '~' || trimmed.startsWith('~/') ? path.join(home, trimmed.slice(1)) : trimmed;
|
|
15
26
|
if (!path.isAbsolute(expanded)) {
|
|
@@ -18,33 +29,63 @@ function absolute(input, home) {
|
|
|
18
29
|
return path.resolve(expanded);
|
|
19
30
|
}
|
|
20
31
|
const nsToMs = (ns) => Number(ns / 1000n) / 1000;
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
32
|
+
async function readMetadata(folder) {
|
|
33
|
+
const text = await fs.readFile(path.join(folder, METADATA), 'utf8').catch(() => null);
|
|
34
|
+
return text === null ? null : parseMetadata(text);
|
|
35
|
+
}
|
|
36
|
+
// A bag: its folder (its files those its metadata names, else those it has,
|
|
37
|
+
// split bags' in order), or one of its files
|
|
38
|
+
export async function openBag(input, home = os.homedir()) {
|
|
39
|
+
const location = bagPath(input, home);
|
|
24
40
|
const stat = await fs.stat(location).catch(() => null);
|
|
25
41
|
if (!stat) {
|
|
26
42
|
throw new Error(`No rosbag at ${location}`);
|
|
27
43
|
}
|
|
28
|
-
if (stat.isDirectory()) {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
if (files.length === 0) {
|
|
32
|
-
const compressed = names.some((name) => /\.(mcap|db3)\.\w+$/i.test(name));
|
|
33
|
-
throw new Error(`No rosbag in ${location}: no .mcap nor .db3 file${compressed ? ' (compressed files are not read)' : ''}`);
|
|
44
|
+
if (!stat.isDirectory()) {
|
|
45
|
+
if (!BAG_FILE.test(location)) {
|
|
46
|
+
throw new Error(`Not a rosbag: ${location} (a bag’s folder, an .mcap or .db3 file)`);
|
|
34
47
|
}
|
|
35
|
-
return files
|
|
48
|
+
return { files: [location], messageCompression: '' };
|
|
36
49
|
}
|
|
37
|
-
|
|
38
|
-
|
|
50
|
+
let metadata = null;
|
|
51
|
+
try {
|
|
52
|
+
metadata = await readMetadata(location);
|
|
39
53
|
}
|
|
40
|
-
|
|
54
|
+
catch {
|
|
55
|
+
// Its files are read without it
|
|
56
|
+
}
|
|
57
|
+
const names = await fs.readdir(location);
|
|
58
|
+
let files = metadata?.files.filter((name) => names.includes(name)) ?? [];
|
|
59
|
+
if (files.length === 0) {
|
|
60
|
+
const found = names.filter((name) => BAG_FILE.test(name)).sort(natural);
|
|
61
|
+
// A compressed bag played leaves its files uncompressed beside them
|
|
62
|
+
const plain = found.filter((name) => !COMPRESSED.test(name));
|
|
63
|
+
files = plain.length ? plain : found;
|
|
64
|
+
}
|
|
65
|
+
if (files.length === 0) {
|
|
66
|
+
throw new Error(`No rosbag in ${location}: no .mcap nor .db3 file`);
|
|
67
|
+
}
|
|
68
|
+
return {
|
|
69
|
+
files: files.map((name) => path.join(location, name)),
|
|
70
|
+
messageCompression: metadata?.compression?.mode === 'message' ? metadata.compression.format : '',
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
// The files of a bag: see openBag
|
|
74
|
+
export async function bagFiles(input, home = os.homedir()) {
|
|
75
|
+
return (await openBag(input, home)).files;
|
|
41
76
|
}
|
|
42
77
|
// The messages of the topics accepted, by topic, in the order recorded
|
|
43
|
-
export async function readBag(
|
|
78
|
+
export async function readBag(bag, accept) {
|
|
79
|
+
const { files, messageCompression } = Array.isArray(bag) ? { files: bag, messageCompression: '' } : bag;
|
|
44
80
|
const topics = new Map();
|
|
45
81
|
for (const file of files) {
|
|
46
|
-
const read =
|
|
82
|
+
const read = await readFile(file, accept);
|
|
47
83
|
for (const [name, found] of read) {
|
|
84
|
+
if (messageCompression) {
|
|
85
|
+
for (const message of found.messages) {
|
|
86
|
+
message.data = decompress(messageCompression, Buffer.from(message.data));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
48
89
|
const known = topics.get(name);
|
|
49
90
|
if (known) {
|
|
50
91
|
known.messages = known.messages.concat(found.messages);
|
|
@@ -59,6 +100,235 @@ export async function readBag(files, accept) {
|
|
|
59
100
|
}
|
|
60
101
|
return topics;
|
|
61
102
|
}
|
|
103
|
+
function readFile(file, accept) {
|
|
104
|
+
if (COMPRESSED.test(file)) {
|
|
105
|
+
return withDecompressed(file, (plain) => readFile(plain, accept));
|
|
106
|
+
}
|
|
107
|
+
return /\.mcap$/i.test(file) ? readMcap(file, accept) : readSqlite(file, accept);
|
|
108
|
+
}
|
|
109
|
+
// A file rosbag2 compressed whole, decompressed into a folder of its own for
|
|
110
|
+
// the time of use
|
|
111
|
+
async function withDecompressed(file, use) {
|
|
112
|
+
if (typeof zlib.createZstdDecompress !== 'function') {
|
|
113
|
+
throw new Error('Reading zstd compressed rosbags needs Node.js 22.15 or newer');
|
|
114
|
+
}
|
|
115
|
+
const folder = await fs.mkdtemp(path.join(os.tmpdir(), 'tassello-bag-'));
|
|
116
|
+
try {
|
|
117
|
+
const plain = path.join(folder, path.basename(file).replace(COMPRESSED, ''));
|
|
118
|
+
await pipeline(createReadStream(file), zlib.createZstdDecompress(), createWriteStream(plain)).catch((e) => {
|
|
119
|
+
throw new Error(`Cannot decompress ${file}: ${errorMessage(e)}`);
|
|
120
|
+
});
|
|
121
|
+
return await use(plain);
|
|
122
|
+
}
|
|
123
|
+
finally {
|
|
124
|
+
await fs.rm(folder, { recursive: true, force: true });
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
async function fileSummary(file, size) {
|
|
128
|
+
if (COMPRESSED.test(file)) {
|
|
129
|
+
return null;
|
|
130
|
+
}
|
|
131
|
+
return /\.mcap$/i.test(file) ? mcapSummary(file) : sqliteSummary(file, size);
|
|
132
|
+
}
|
|
133
|
+
// A bag's folder, or one of its files, as listed: its metadata, else its
|
|
134
|
+
// files' summaries. Throws if it is not a bag.
|
|
135
|
+
export async function bagInfo(input, home = os.homedir()) {
|
|
136
|
+
const location = bagPath(input, home);
|
|
137
|
+
const stat = await fs.stat(location).catch(() => null);
|
|
138
|
+
if (!stat) {
|
|
139
|
+
throw new Error(`No rosbag at ${location}`);
|
|
140
|
+
}
|
|
141
|
+
const info = {
|
|
142
|
+
path: location,
|
|
143
|
+
name: path.basename(location),
|
|
144
|
+
size: 0,
|
|
145
|
+
modified: stat.mtimeMs,
|
|
146
|
+
storage: '',
|
|
147
|
+
start: null,
|
|
148
|
+
duration: null,
|
|
149
|
+
messages: null,
|
|
150
|
+
topics: [],
|
|
151
|
+
files: [],
|
|
152
|
+
compression: null,
|
|
153
|
+
recording: false,
|
|
154
|
+
problem: null,
|
|
155
|
+
};
|
|
156
|
+
let metadata = null;
|
|
157
|
+
let names;
|
|
158
|
+
if (stat.isDirectory()) {
|
|
159
|
+
try {
|
|
160
|
+
metadata = await readMetadata(location);
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
info.problem = `Its ${METADATA} cannot be read: ${errorMessage(e)}`;
|
|
164
|
+
}
|
|
165
|
+
const entries = await fs.readdir(location);
|
|
166
|
+
names = metadata?.files.filter((name) => entries.includes(name)) ?? [];
|
|
167
|
+
if (names.length === 0) {
|
|
168
|
+
names = entries.filter((name) => BAG_FILE.test(name)).sort(natural);
|
|
169
|
+
}
|
|
170
|
+
if (!metadata && names.length === 0) {
|
|
171
|
+
throw new Error(`No rosbag in ${location}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
else if (BAG_FILE.test(location)) {
|
|
175
|
+
names = [path.basename(location)];
|
|
176
|
+
}
|
|
177
|
+
else {
|
|
178
|
+
throw new Error(`Not a rosbag: ${location}`);
|
|
179
|
+
}
|
|
180
|
+
const folder = stat.isDirectory() ? location : path.dirname(location);
|
|
181
|
+
for (const name of names) {
|
|
182
|
+
const file = await fs.stat(path.join(folder, name)).catch(() => null);
|
|
183
|
+
if (file) {
|
|
184
|
+
info.files.push({ name, size: file.size });
|
|
185
|
+
info.size += file.size;
|
|
186
|
+
info.modified = Math.max(info.modified, file.mtimeMs);
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
if (metadata) {
|
|
190
|
+
Object.assign(info, {
|
|
191
|
+
storage: metadata.storage,
|
|
192
|
+
start: metadata.start,
|
|
193
|
+
duration: metadata.duration,
|
|
194
|
+
messages: metadata.messages,
|
|
195
|
+
topics: metadata.topics,
|
|
196
|
+
compression: metadata.compression,
|
|
197
|
+
});
|
|
198
|
+
if (info.files.length < names.length) {
|
|
199
|
+
info.problem = `${names.length - info.files.length} of its files missing`;
|
|
200
|
+
}
|
|
201
|
+
return info;
|
|
202
|
+
}
|
|
203
|
+
// Without metadata: what its files say of themselves
|
|
204
|
+
const topics = new Map();
|
|
205
|
+
let start = Infinity;
|
|
206
|
+
let end = -Infinity;
|
|
207
|
+
let known = true;
|
|
208
|
+
for (const { name, size } of info.files) {
|
|
209
|
+
let summary = null;
|
|
210
|
+
try {
|
|
211
|
+
summary = await fileSummary(path.join(folder, name), size);
|
|
212
|
+
}
|
|
213
|
+
catch {
|
|
214
|
+
// As if it said nothing
|
|
215
|
+
}
|
|
216
|
+
if (!summary) {
|
|
217
|
+
known = false;
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
info.storage ||= summary.storage;
|
|
221
|
+
for (const topic of summary.topics) {
|
|
222
|
+
const seen = topics.get(topic.name);
|
|
223
|
+
if (!seen) {
|
|
224
|
+
topics.set(topic.name, { ...topic });
|
|
225
|
+
}
|
|
226
|
+
else if (seen.count !== null) {
|
|
227
|
+
seen.count = topic.count === null ? null : seen.count + topic.count;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
if (summary.start === null || summary.end === null) {
|
|
231
|
+
known = false;
|
|
232
|
+
}
|
|
233
|
+
else {
|
|
234
|
+
start = Math.min(start, summary.start);
|
|
235
|
+
end = Math.max(end, summary.end);
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
info.storage ||= /\.mcap/i.test(names[0] ?? '') ? 'mcap' : 'sqlite3';
|
|
239
|
+
info.topics = [...topics.values()].sort((a, b) => a.name.localeCompare(b.name));
|
|
240
|
+
if (known && Number.isFinite(start)) {
|
|
241
|
+
info.start = start;
|
|
242
|
+
info.duration = end - start;
|
|
243
|
+
info.messages = info.topics.every((topic) => topic.count !== null)
|
|
244
|
+
? info.topics.reduce((sum, topic) => sum + (topic.count ?? 0), 0)
|
|
245
|
+
: null;
|
|
246
|
+
}
|
|
247
|
+
if (stat.isDirectory() && !info.problem) {
|
|
248
|
+
info.problem = `No ${METADATA}: being recorded, or its recording did not end (ros2 bag reindex makes it)`;
|
|
249
|
+
}
|
|
250
|
+
return info;
|
|
251
|
+
}
|
|
252
|
+
// The version of a bag as files: its files' names, sizes and times
|
|
253
|
+
async function bagVersion(location, directory) {
|
|
254
|
+
const names = directory ? await fs.readdir(location) : [''];
|
|
255
|
+
const parts = await Promise.all(names.map(async (name) => {
|
|
256
|
+
const stat = await fs.stat(path.join(location, name)).catch(() => null);
|
|
257
|
+
return stat ? `${name}:${stat.size}:${stat.mtimeMs}` : name;
|
|
258
|
+
}));
|
|
259
|
+
return parts.sort().join('|');
|
|
260
|
+
}
|
|
261
|
+
// The bags of a folder (its folders that are bags, and its lone bag files),
|
|
262
|
+
// the newest first. A folder not there yet has none. Bags read before are
|
|
263
|
+
// read again only if their files changed.
|
|
264
|
+
export async function listBags(folder, cache = new Map(), home = os.homedir()) {
|
|
265
|
+
const location = bagPath(folder, home);
|
|
266
|
+
const entries = await fs.readdir(location, { withFileTypes: true }).catch((e) => {
|
|
267
|
+
if (e.code === 'ENOENT') {
|
|
268
|
+
return [];
|
|
269
|
+
}
|
|
270
|
+
throw new Error(`Cannot read ${location}: ${e.code ?? errorMessage(e)}`);
|
|
271
|
+
});
|
|
272
|
+
const bags = [];
|
|
273
|
+
const seen = new Set();
|
|
274
|
+
for (const entry of entries) {
|
|
275
|
+
if (entry.name.startsWith('.')) {
|
|
276
|
+
continue;
|
|
277
|
+
}
|
|
278
|
+
const full = path.join(location, entry.name);
|
|
279
|
+
let directory = entry.isDirectory();
|
|
280
|
+
if (entry.isSymbolicLink()) {
|
|
281
|
+
directory = (await fs.stat(full).catch(() => null))?.isDirectory() ?? false;
|
|
282
|
+
}
|
|
283
|
+
if (directory) {
|
|
284
|
+
const inside = await fs.readdir(full).catch(() => []);
|
|
285
|
+
if (!inside.some((name) => name === METADATA || BAG_FILE.test(name))) {
|
|
286
|
+
continue;
|
|
287
|
+
}
|
|
288
|
+
}
|
|
289
|
+
else if (!/\.(mcap|db3)$/i.test(entry.name)) {
|
|
290
|
+
continue;
|
|
291
|
+
}
|
|
292
|
+
seen.add(full);
|
|
293
|
+
const version = await bagVersion(full, directory);
|
|
294
|
+
const cached = cache.get(full);
|
|
295
|
+
if (cached?.version === version) {
|
|
296
|
+
bags.push(cached.info);
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
let info;
|
|
300
|
+
try {
|
|
301
|
+
info = await bagInfo(full, home);
|
|
302
|
+
}
|
|
303
|
+
catch {
|
|
304
|
+
// Gone while read, or not a bag after all
|
|
305
|
+
continue;
|
|
306
|
+
}
|
|
307
|
+
cache.set(full, { version, info });
|
|
308
|
+
bags.push(info);
|
|
309
|
+
}
|
|
310
|
+
for (const key of cache.keys()) {
|
|
311
|
+
if (path.dirname(key) === location && !seen.has(key)) {
|
|
312
|
+
cache.delete(key);
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
return bags.sort((a, b) => (b.start ?? b.modified) - (a.start ?? a.modified));
|
|
316
|
+
}
|
|
317
|
+
// The folder a recording makes: a new one, absolute or ~/…, its parent made
|
|
318
|
+
export async function newBagFolder(input, home = os.homedir()) {
|
|
319
|
+
const folder = bagPath(input.trim().replace(/(.)\/+$/, '$1'), home);
|
|
320
|
+
if (await fs.stat(folder).catch(() => null)) {
|
|
321
|
+
throw new Error(`${folder} exists already: a recording makes a new folder`);
|
|
322
|
+
}
|
|
323
|
+
await fs.mkdir(path.dirname(folder), { recursive: true });
|
|
324
|
+
return folder;
|
|
325
|
+
}
|
|
326
|
+
// Of a folder's files, bytes
|
|
327
|
+
export async function folderSize(folder) {
|
|
328
|
+
const names = await fs.readdir(folder).catch(() => []);
|
|
329
|
+
const sizes = await Promise.all(names.map((name) => fs.stat(path.join(folder, name)).then((s) => (s.isFile() ? s.size : 0), () => 0)));
|
|
330
|
+
return sizes.reduce((sum, size) => sum + size, 0);
|
|
331
|
+
}
|
|
62
332
|
// ── MCAP ─────────────────────────────────────────────────────────────────────
|
|
63
333
|
// Records: opcode (u8), length (u64), content; little endian, strings and
|
|
64
334
|
// maps prefixed with their u32 byte length (https://mcap.dev/spec)
|
|
@@ -69,6 +339,7 @@ const CHANNEL = 0x04;
|
|
|
69
339
|
const MESSAGE = 0x05;
|
|
70
340
|
const CHUNK = 0x06;
|
|
71
341
|
const CHUNK_INDEX = 0x08;
|
|
342
|
+
const STATISTICS = 0x0b;
|
|
72
343
|
const DATA_END = 0x0f;
|
|
73
344
|
// Opcode, length, summary start, summary offset start, summary CRC
|
|
74
345
|
const FOOTER_BYTES = 1 + 8 + 8 + 8 + 4;
|
|
@@ -113,32 +384,32 @@ class Cursor {
|
|
|
113
384
|
return this.bytes(this.u32()).toString('utf8');
|
|
114
385
|
}
|
|
115
386
|
}
|
|
387
|
+
// MCAP's chunks, and rosbag2's messages
|
|
116
388
|
function decompress(compression, data) {
|
|
117
389
|
if (!compression) {
|
|
118
390
|
return data;
|
|
119
391
|
}
|
|
120
392
|
if (compression === 'zstd') {
|
|
121
393
|
if (typeof zlib.zstdDecompressSync !== 'function') {
|
|
122
|
-
throw new Error('Reading zstd compressed
|
|
394
|
+
throw new Error('Reading zstd compressed rosbags needs Node.js 22.15 or newer');
|
|
123
395
|
}
|
|
124
396
|
return zlib.zstdDecompressSync(data);
|
|
125
397
|
}
|
|
126
|
-
throw new Error(`
|
|
398
|
+
throw new Error(`Rosbags compressed with ${compression} are not read: record with zstd, or without compression`);
|
|
127
399
|
}
|
|
128
|
-
// What the records of a file say: its schemas and channels,
|
|
129
|
-
// of
|
|
400
|
+
// What the records of a file say: its schemas and channels, its statistics,
|
|
401
|
+
// and the messages of the channels accepted
|
|
130
402
|
class McapRecords {
|
|
131
403
|
topics = new Map();
|
|
404
|
+
// Every channel, and those whose messages are read
|
|
132
405
|
channels = new Map();
|
|
406
|
+
accepted = new Set();
|
|
407
|
+
statistics = null;
|
|
133
408
|
schemas = new Map();
|
|
134
409
|
accept;
|
|
135
410
|
constructor(accept) {
|
|
136
411
|
this.accept = accept;
|
|
137
412
|
}
|
|
138
|
-
// The channels whose messages are read
|
|
139
|
-
wanted() {
|
|
140
|
-
return new Set([...this.channels].flatMap(([id, topic]) => (topic ? [id] : [])));
|
|
141
|
-
}
|
|
142
413
|
record(opcode, content) {
|
|
143
414
|
const cursor = new Cursor(content);
|
|
144
415
|
switch (opcode) {
|
|
@@ -150,12 +421,16 @@ class McapRecords {
|
|
|
150
421
|
const schema = cursor.u16();
|
|
151
422
|
const topic = { name: cursor.string(), type: this.schemas.get(schema) ?? '' };
|
|
152
423
|
if (!this.channels.has(id)) {
|
|
153
|
-
this.channels.set(id,
|
|
424
|
+
this.channels.set(id, topic);
|
|
425
|
+
if (this.accept(topic)) {
|
|
426
|
+
this.accepted.add(id);
|
|
427
|
+
}
|
|
154
428
|
}
|
|
155
429
|
break;
|
|
156
430
|
}
|
|
157
431
|
case MESSAGE: {
|
|
158
|
-
const
|
|
432
|
+
const id = cursor.u16();
|
|
433
|
+
const topic = this.accepted.has(id) ? this.channels.get(id) : undefined;
|
|
159
434
|
if (!topic) {
|
|
160
435
|
break;
|
|
161
436
|
}
|
|
@@ -179,6 +454,20 @@ class McapRecords {
|
|
|
179
454
|
this.records(decompress(compression, cursor.bytes(cursor.size())));
|
|
180
455
|
break;
|
|
181
456
|
}
|
|
457
|
+
case STATISTICS: {
|
|
458
|
+
cursor.u64();
|
|
459
|
+
cursor.u16();
|
|
460
|
+
cursor.offset += 4 * 4;
|
|
461
|
+
const start = nsToMs(cursor.u64());
|
|
462
|
+
const end = nsToMs(cursor.u64());
|
|
463
|
+
const map = new Cursor(cursor.bytes(cursor.u32()));
|
|
464
|
+
const counts = new Map();
|
|
465
|
+
while (map.offset < map.data.length) {
|
|
466
|
+
counts.set(map.u16(), Number(map.u64()));
|
|
467
|
+
}
|
|
468
|
+
this.statistics = { start, end, counts };
|
|
469
|
+
break;
|
|
470
|
+
}
|
|
182
471
|
}
|
|
183
472
|
}
|
|
184
473
|
// A sequence of records, e.g. a chunk's
|
|
@@ -190,7 +479,8 @@ class McapRecords {
|
|
|
190
479
|
}
|
|
191
480
|
}
|
|
192
481
|
}
|
|
193
|
-
|
|
482
|
+
// Opens an MCAP file for use: throws if it is not one
|
|
483
|
+
async function withMcap(file, use) {
|
|
194
484
|
const handle = await fs.open(file, 'r').catch((e) => {
|
|
195
485
|
throw new Error(`Cannot read ${file}: ${e.code ?? errorMessage(e)}`);
|
|
196
486
|
});
|
|
@@ -205,20 +495,41 @@ async function readMcap(file, accept) {
|
|
|
205
495
|
if (!MCAP_MAGIC.every((byte, i) => head[i] === byte)) {
|
|
206
496
|
throw new Error(`${file} is not an MCAP file`);
|
|
207
497
|
}
|
|
208
|
-
|
|
209
|
-
if (!(await readIndexed(records, read, size))) {
|
|
210
|
-
await readSequentially(records, read, size);
|
|
211
|
-
}
|
|
212
|
-
return records.topics;
|
|
498
|
+
return await use(read, size);
|
|
213
499
|
}
|
|
214
500
|
finally {
|
|
215
501
|
await handle.close();
|
|
216
502
|
}
|
|
217
503
|
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
504
|
+
async function readMcap(file, accept) {
|
|
505
|
+
return withMcap(file, async (read, size) => {
|
|
506
|
+
const records = new McapRecords(accept);
|
|
507
|
+
if (!(await readIndexed(records, read, size, true))) {
|
|
508
|
+
await readSequentially(records, read, size);
|
|
509
|
+
}
|
|
510
|
+
return records.topics;
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
// Its channels and statistics, from its summary
|
|
514
|
+
async function mcapSummary(file) {
|
|
515
|
+
return withMcap(file, async (read, size) => {
|
|
516
|
+
const records = new McapRecords(() => false);
|
|
517
|
+
await readIndexed(records, read, size, false);
|
|
518
|
+
const { statistics } = records;
|
|
519
|
+
if (records.channels.size === 0 && !statistics) {
|
|
520
|
+
return null;
|
|
521
|
+
}
|
|
522
|
+
const topics = [...records.channels].map(([id, topic]) => ({ ...topic, count: statistics?.counts.get(id) ?? (statistics ? 0 : null) }));
|
|
523
|
+
// Statistics of a file without messages say 0 for both
|
|
524
|
+
const timed = statistics && statistics.end >= statistics.start && statistics.end > 0;
|
|
525
|
+
return { storage: 'mcap', topics, start: timed ? statistics.start : null, end: timed ? statistics.end : null };
|
|
526
|
+
});
|
|
527
|
+
}
|
|
528
|
+
// From the summary at the end of the file: the channels, the statistics, and
|
|
529
|
+
// (chunks) where the chunks with messages of those accepted are, which alone
|
|
530
|
+
// are read. false if the file has no summary (unfinished, or written without
|
|
531
|
+
// chunks).
|
|
532
|
+
async function readIndexed(records, read, size, chunks) {
|
|
222
533
|
const footerAt = size - 8 - FOOTER_BYTES;
|
|
223
534
|
if (footerAt < 8) {
|
|
224
535
|
return false;
|
|
@@ -232,7 +543,7 @@ async function readIndexed(records, read, size) {
|
|
|
232
543
|
if (summaryAt === 0 || summaryAt >= footerAt) {
|
|
233
544
|
return false;
|
|
234
545
|
}
|
|
235
|
-
const
|
|
546
|
+
const indexes = [];
|
|
236
547
|
const summary = new Cursor(await read(summaryAt, footerAt - summaryAt));
|
|
237
548
|
while (summary.offset + 9 <= summary.data.length) {
|
|
238
549
|
const opcode = summary.u8();
|
|
@@ -252,14 +563,16 @@ async function readIndexed(records, read, size) {
|
|
|
252
563
|
channels.push(map.u16());
|
|
253
564
|
map.u64();
|
|
254
565
|
}
|
|
255
|
-
|
|
566
|
+
indexes.push({ at, length, start, channels });
|
|
567
|
+
}
|
|
568
|
+
if (!chunks) {
|
|
569
|
+
return true;
|
|
256
570
|
}
|
|
257
|
-
if (
|
|
571
|
+
if (indexes.length === 0) {
|
|
258
572
|
return false;
|
|
259
573
|
}
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
for (const chunk of chunks.filter((c) => c.channels.some((id) => wanted.has(id)))) {
|
|
574
|
+
indexes.sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0));
|
|
575
|
+
for (const chunk of indexes.filter((c) => c.channels.some((id) => records.accepted.has(id)))) {
|
|
263
576
|
const record = new Cursor(await read(chunk.at, chunk.length));
|
|
264
577
|
const opcode = record.u8();
|
|
265
578
|
records.record(opcode, record.bytes(record.size()));
|
|
@@ -288,7 +601,7 @@ async function readSequentially(records, read, size) {
|
|
|
288
601
|
}
|
|
289
602
|
}
|
|
290
603
|
// ── SQLite ───────────────────────────────────────────────────────────────────
|
|
291
|
-
async function
|
|
604
|
+
async function withSqlite(file, use) {
|
|
292
605
|
let sqlite;
|
|
293
606
|
try {
|
|
294
607
|
sqlite = await import('node:sqlite');
|
|
@@ -304,6 +617,17 @@ async function readSqlite(file, accept) {
|
|
|
304
617
|
throw new Error(`Cannot read ${file}: ${errorMessage(e)}`);
|
|
305
618
|
}
|
|
306
619
|
try {
|
|
620
|
+
return use(db);
|
|
621
|
+
}
|
|
622
|
+
catch (e) {
|
|
623
|
+
throw new Error(`Cannot read ${file}: ${errorMessage(e)}`);
|
|
624
|
+
}
|
|
625
|
+
finally {
|
|
626
|
+
db.close();
|
|
627
|
+
}
|
|
628
|
+
}
|
|
629
|
+
async function readSqlite(file, accept) {
|
|
630
|
+
return withSqlite(file, (db) => {
|
|
307
631
|
const topics = new Map();
|
|
308
632
|
const rows = db.prepare('SELECT id, name, type FROM topics').all();
|
|
309
633
|
const select = db.prepare('SELECT timestamp, data FROM messages WHERE topic_id = ? ORDER BY timestamp');
|
|
@@ -320,99 +644,27 @@ async function readSqlite(file, accept) {
|
|
|
320
644
|
}
|
|
321
645
|
}
|
|
322
646
|
return topics;
|
|
323
|
-
}
|
|
324
|
-
catch (e) {
|
|
325
|
-
throw new Error(`Cannot read ${file}: ${errorMessage(e)}`);
|
|
326
|
-
}
|
|
327
|
-
finally {
|
|
328
|
-
db.close();
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
// ── Recording ────────────────────────────────────────────────────────────────
|
|
332
|
-
// Waited for ros2 bag record to say it records; quiet, it is taken as doing so
|
|
333
|
-
const START_TIMEOUT_MS = 8000;
|
|
334
|
-
// Then it is killed: the bag may lack its metadata
|
|
335
|
-
const STOP_TIMEOUT_MS = 10_000;
|
|
336
|
-
const READY = /Subscribed to topic|Listening for topics|Recording\.\.\./;
|
|
337
|
-
const OUTPUT_LIMIT = 4000;
|
|
338
|
-
// The folder a recording makes: a new one, absolute or ~/…, its parent made
|
|
339
|
-
export async function newBagFolder(input, home = os.homedir()) {
|
|
340
|
-
const folder = absolute(input.trim().replace(/(.)\/+$/, '$1'), home);
|
|
341
|
-
if (await fs.stat(folder).catch(() => null)) {
|
|
342
|
-
throw new Error(`${folder} exists already: a recording makes a new folder`);
|
|
343
|
-
}
|
|
344
|
-
await fs.mkdir(path.dirname(folder), { recursive: true });
|
|
345
|
-
return folder;
|
|
647
|
+
});
|
|
346
648
|
}
|
|
347
|
-
//
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
output = '';
|
|
354
|
-
stopping = false;
|
|
355
|
-
constructor(child, folder) {
|
|
356
|
-
this.child = child;
|
|
357
|
-
this.folder = folder;
|
|
358
|
-
}
|
|
359
|
-
// Resolves once it records, rejects with why it could not
|
|
360
|
-
static start(folder, topics) {
|
|
361
|
-
return new Promise((resolve, reject) => {
|
|
362
|
-
const child = spawn('ros2', ['bag', 'record', '-o', folder, ...topics], {
|
|
363
|
-
cwd: os.homedir(),
|
|
364
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
365
|
-
});
|
|
366
|
-
const recorder = new BagRecorder(child, folder);
|
|
367
|
-
let settled = false;
|
|
368
|
-
const ready = () => {
|
|
369
|
-
if (!settled) {
|
|
370
|
-
settled = true;
|
|
371
|
-
clearTimeout(timer);
|
|
372
|
-
resolve(recorder);
|
|
373
|
-
}
|
|
374
|
-
};
|
|
375
|
-
const timer = setTimeout(ready, START_TIMEOUT_MS);
|
|
376
|
-
const onOutput = (chunk) => {
|
|
377
|
-
recorder.output = (recorder.output + stripAnsi(chunk.toString())).slice(-OUTPUT_LIMIT);
|
|
378
|
-
if (READY.test(recorder.output)) {
|
|
379
|
-
ready();
|
|
380
|
-
}
|
|
381
|
-
};
|
|
382
|
-
child.stdout?.on('data', onOutput);
|
|
383
|
-
child.stderr?.on('data', onOutput);
|
|
384
|
-
child.on('error', (e) => {
|
|
385
|
-
recorder.output = e.code === 'ENOENT' ? 'ros2 not found: is ROS sourced?' : errorMessage(e);
|
|
386
|
-
if (!settled) {
|
|
387
|
-
settled = true;
|
|
388
|
-
clearTimeout(timer);
|
|
389
|
-
reject(new Error(recorder.output));
|
|
390
|
-
}
|
|
391
|
-
});
|
|
392
|
-
child.on('close', (code, signal) => {
|
|
393
|
-
clearTimeout(timer);
|
|
394
|
-
const lines = recorder.output.trim().split('\n').slice(-2).join(' ').trim();
|
|
395
|
-
const reason = lines || `ros2 bag record exited (${signal ?? code})`;
|
|
396
|
-
if (!settled) {
|
|
397
|
-
settled = true;
|
|
398
|
-
reject(new Error(reason));
|
|
399
|
-
}
|
|
400
|
-
else if (!recorder.stopping) {
|
|
401
|
-
recorder.onExit?.(reason);
|
|
402
|
-
}
|
|
403
|
-
});
|
|
404
|
-
});
|
|
405
|
-
}
|
|
406
|
-
// As Ctrl+C does, which completes the bag
|
|
407
|
-
async stop() {
|
|
408
|
-
this.stopping = true;
|
|
409
|
-
if (this.child.exitCode !== null || this.child.signalCode !== null) {
|
|
410
|
-
return;
|
|
649
|
+
// Its topics; their counts and its time span if small enough to count
|
|
650
|
+
async function sqliteSummary(file, size) {
|
|
651
|
+
return withSqlite(file, (db) => {
|
|
652
|
+
const rows = db.prepare('SELECT id, name, type FROM topics').all();
|
|
653
|
+
if (size > SQLITE_SCAN_BYTES) {
|
|
654
|
+
return { storage: 'sqlite3', topics: rows.map((row) => ({ name: row.name, type: row.type, count: null })), start: null, end: null };
|
|
411
655
|
}
|
|
412
|
-
const
|
|
413
|
-
|
|
414
|
-
const
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
656
|
+
const select = db.prepare('SELECT topic_id, COUNT(*) AS count, MIN(timestamp) AS first, MAX(timestamp) AS last FROM messages GROUP BY topic_id');
|
|
657
|
+
select.setReadBigInts(true);
|
|
658
|
+
const counts = new Map();
|
|
659
|
+
for (const row of select.all()) {
|
|
660
|
+
counts.set(Number(row.topic_id), row);
|
|
661
|
+
}
|
|
662
|
+
const spans = [...counts.values()];
|
|
663
|
+
return {
|
|
664
|
+
storage: 'sqlite3',
|
|
665
|
+
topics: rows.map((row) => ({ name: row.name, type: row.type, count: Number(counts.get(row.id)?.count ?? 0n) })),
|
|
666
|
+
start: spans.length ? Math.min(...spans.map((s) => nsToMs(s.first))) : null,
|
|
667
|
+
end: spans.length ? Math.max(...spans.map((s) => nsToMs(s.last))) : null,
|
|
668
|
+
};
|
|
669
|
+
});
|
|
418
670
|
}
|