small-mcap 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- small_mcap-0.1.0/PKG-INFO +308 -0
- small_mcap-0.1.0/README.md +271 -0
- small_mcap-0.1.0/pyproject.toml +55 -0
- small_mcap-0.1.0/src/small_mcap/__init__.py +130 -0
- small_mcap-0.1.0/src/small_mcap/exceptions.py +81 -0
- small_mcap-0.1.0/src/small_mcap/json_decoder.py +52 -0
- small_mcap-0.1.0/src/small_mcap/py.typed +0 -0
- small_mcap-0.1.0/src/small_mcap/reader.py +843 -0
- small_mcap-0.1.0/src/small_mcap/rebuild.py +423 -0
- small_mcap-0.1.0/src/small_mcap/records.py +1030 -0
- small_mcap-0.1.0/src/small_mcap/remapper.py +162 -0
- small_mcap-0.1.0/src/small_mcap/well_known.py +28 -0
- small_mcap-0.1.0/src/small_mcap/writer.py +752 -0
|
@@ -0,0 +1,308 @@
|
|
|
1
|
+
Metadata-Version: 2.3
|
|
2
|
+
Name: small-mcap
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lightweight Python library for reading and writing MCAP files
|
|
5
|
+
Keywords: mcap,robotics,ros,ros2,serialization,logging
|
|
6
|
+
Author: Marko Bausch
|
|
7
|
+
License: GPL-3.0
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: GNU General Public License v3 (GPLv3)
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering
|
|
19
|
+
Classifier: Topic :: System :: Logging
|
|
20
|
+
Classifier: Typing :: Typed
|
|
21
|
+
Requires-Dist: small-mcap[zstd] ; extra == 'compression'
|
|
22
|
+
Requires-Dist: small-mcap[lz4] ; extra == 'compression'
|
|
23
|
+
Requires-Dist: mcap>=1.0.0 ; extra == 'dev'
|
|
24
|
+
Requires-Dist: rosbags>=0.11.0 ; extra == 'dev'
|
|
25
|
+
Requires-Dist: pybag-sdk>=0.6.0 ; extra == 'dev'
|
|
26
|
+
Requires-Dist: lz4>=4.3.3 ; extra == 'lz4'
|
|
27
|
+
Requires-Dist: zstandard>=0.23.0 ; extra == 'zstd'
|
|
28
|
+
Requires-Python: >=3.10
|
|
29
|
+
Project-URL: Homepage, https://github.com/mrkbac/robotic-tools
|
|
30
|
+
Project-URL: Issues, https://github.com/mrkbac/robotic-tools/issues
|
|
31
|
+
Project-URL: Repository, https://github.com/mrkbac/robotic-tools
|
|
32
|
+
Provides-Extra: compression
|
|
33
|
+
Provides-Extra: dev
|
|
34
|
+
Provides-Extra: lz4
|
|
35
|
+
Provides-Extra: zstd
|
|
36
|
+
Description-Content-Type: text/markdown
|
|
37
|
+
|
|
38
|
+
# small-mcap
|
|
39
|
+
|
|
40
|
+
Lightweight Python library for reading and writing MCAP files.
|
|
41
|
+
|
|
42
|
+
## Installation
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
uv add small-mcap
|
|
46
|
+
|
|
47
|
+
# With compression support
|
|
48
|
+
uv add small-mcap[compression] # ZSTD + LZ4
|
|
49
|
+
uv add small-mcap[zstd] # ZSTD only
|
|
50
|
+
uv add small-mcap[lz4] # LZ4 only
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
## Reader
|
|
54
|
+
|
|
55
|
+
### Basic read
|
|
56
|
+
|
|
57
|
+
```python
|
|
58
|
+
from small_mcap import read_message
|
|
59
|
+
|
|
60
|
+
with open("input.mcap", "rb") as f:
|
|
61
|
+
for schema, channel, message in read_message(f):
|
|
62
|
+
print(f"{channel.topic}: {message.data}")
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Read multiple inputs
|
|
66
|
+
|
|
67
|
+
```python
|
|
68
|
+
from small_mcap import read_message
|
|
69
|
+
|
|
70
|
+
with open("recording1.mcap", "rb") as f1, \
|
|
71
|
+
open("recording2.mcap", "rb") as f2, \
|
|
72
|
+
open("recording3.mcap", "rb") as f3:
|
|
73
|
+
for schema, channel, message in read_message([f1, f2, f3]):
|
|
74
|
+
print(f"{channel.topic}: {message.log_time}")
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
### Read with topic filtering
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from small_mcap import read_message, include_topics
|
|
81
|
+
|
|
82
|
+
with open("input.mcap", "rb") as f:
|
|
83
|
+
topics = ["/camera/image", "/lidar/points"]
|
|
84
|
+
for schema, channel, message in read_message(f, should_include=include_topics(topics)):
|
|
85
|
+
print(f"{channel.topic}: {len(message.data)} bytes")
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
### Read with time range
|
|
89
|
+
|
|
90
|
+
```python
|
|
91
|
+
from small_mcap import read_message
|
|
92
|
+
|
|
93
|
+
with open("input.mcap", "rb") as f:
|
|
94
|
+
start = 1000000000 # nanoseconds
|
|
95
|
+
end = 2000000000
|
|
96
|
+
for schema, channel, message in read_message(f, start_time_ns=start, end_time_ns=end):
|
|
97
|
+
print(f"{channel.topic} at {message.log_time}")
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
### Read decoded messages
|
|
101
|
+
|
|
102
|
+
```python
|
|
103
|
+
from small_mcap import read_message_decoded
|
|
104
|
+
import json
|
|
105
|
+
|
|
106
|
+
class JsonDecoderFactory:
|
|
107
|
+
def decoder_for(self, schema):
|
|
108
|
+
if schema.encoding == "json":
|
|
109
|
+
return lambda data: json.loads(data)
|
|
110
|
+
return None
|
|
111
|
+
|
|
112
|
+
with open("input.mcap", "rb") as f:
|
|
113
|
+
for msg in read_message_decoded(f, decoder_factories=[JsonDecoderFactory()]):
|
|
114
|
+
print(f"{msg.channel.topic}: {msg.decoded_message}")
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
### Read summary/metadata
|
|
118
|
+
|
|
119
|
+
```python
|
|
120
|
+
from small_mcap import get_summary, get_header
|
|
121
|
+
|
|
122
|
+
with open("input.mcap", "rb") as f:
|
|
123
|
+
summary = get_summary(f)
|
|
124
|
+
print(f"Messages: {summary.statistics.message_count}")
|
|
125
|
+
print(f"Duration: {summary.statistics.message_start_time} - {summary.statistics.message_end_time}")
|
|
126
|
+
|
|
127
|
+
for channel in summary.channels.values():
|
|
128
|
+
print(f" {channel.topic}: {channel.message_encoding}")
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
## Writer
|
|
132
|
+
|
|
133
|
+
### Basic write
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from small_mcap import McapWriter
|
|
137
|
+
|
|
138
|
+
with open("output.mcap", "wb") as f:
|
|
139
|
+
writer = McapWriter(f)
|
|
140
|
+
writer.start(profile="", library="my-app")
|
|
141
|
+
|
|
142
|
+
# Add schema
|
|
143
|
+
schema_id = writer.add_schema("MySchema", "json", b'{"type": "object"}')
|
|
144
|
+
|
|
145
|
+
# Add channel
|
|
146
|
+
channel_id = writer.add_channel("/my/topic", "json", schema_id=schema_id)
|
|
147
|
+
|
|
148
|
+
# Add messages
|
|
149
|
+
for i in range(100):
|
|
150
|
+
writer.add_message(
|
|
151
|
+
channel_id,
|
|
152
|
+
log_time=i * 1000000, # nanoseconds
|
|
153
|
+
data=b'{"value": 42}',
|
|
154
|
+
publish_time=i * 1000000
|
|
155
|
+
)
|
|
156
|
+
|
|
157
|
+
writer.finish()
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
### Write with compression
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
from small_mcap import McapWriter, CompressionType
|
|
164
|
+
|
|
165
|
+
with open("output.mcap", "wb") as f:
|
|
166
|
+
writer = McapWriter(
|
|
167
|
+
f,
|
|
168
|
+
compression=CompressionType.ZSTD,
|
|
169
|
+
chunk_size=1024 * 1024 # 1MB chunks
|
|
170
|
+
)
|
|
171
|
+
writer.start(profile="", library="my-app")
|
|
172
|
+
|
|
173
|
+
schema_id = writer.add_schema("MySchema", "json", b"{}")
|
|
174
|
+
channel_id = writer.add_channel("/topic", "json", schema_id=schema_id)
|
|
175
|
+
|
|
176
|
+
for i in range(1000):
|
|
177
|
+
writer.add_message(channel_id, log_time=i*1000, data=b"data", publish_time=i*1000)
|
|
178
|
+
|
|
179
|
+
writer.finish()
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
### Write with encoder factory
|
|
183
|
+
|
|
184
|
+
```python
|
|
185
|
+
from small_mcap import McapWriter, EncoderFactory
|
|
186
|
+
import json
|
|
187
|
+
|
|
188
|
+
class JsonEncoder(EncoderFactory):
|
|
189
|
+
def get_schema_encoding(self, schema_name):
|
|
190
|
+
return "json", b'{"type": "object"}'
|
|
191
|
+
|
|
192
|
+
def get_channel_encoding(self, topic):
|
|
193
|
+
return "json"
|
|
194
|
+
|
|
195
|
+
def encode(self, topic, msg):
|
|
196
|
+
return json.dumps(msg).encode()
|
|
197
|
+
|
|
198
|
+
with open("output.mcap", "wb") as f:
|
|
199
|
+
writer = McapWriter(f)
|
|
200
|
+
writer.start(profile="", library="my-app")
|
|
201
|
+
|
|
202
|
+
encoder = JsonEncoder()
|
|
203
|
+
|
|
204
|
+
# Encoder automatically registers schemas and channels
|
|
205
|
+
for i in range(100):
|
|
206
|
+
msg = {"timestamp": i, "value": i * 2}
|
|
207
|
+
writer.add_message_encoded("/sensor/data", i * 1000, msg, encoder, publish_time=i * 1000)
|
|
208
|
+
|
|
209
|
+
writer.finish()
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
## Features
|
|
213
|
+
|
|
214
|
+
- Zero dependencies for core functionality
|
|
215
|
+
- Optional compression support (ZSTD, LZ4)
|
|
216
|
+
- Lazy chunk loading for efficient memory usage
|
|
217
|
+
- Topic and time-range filtering
|
|
218
|
+
- Automatic schema/channel registration
|
|
219
|
+
- CRC validation
|
|
220
|
+
- Fast summary/metadata access
|
|
221
|
+
|
|
222
|
+
## Performance
|
|
223
|
+
|
|
224
|
+
`small-mcap` is optimized for high-performance MCAP file reading with zero-copy operations and lazy chunk loading:
|
|
225
|
+
|
|
226
|
+
**Key Optimizations:**
|
|
227
|
+
|
|
228
|
+
- **Zero-copy memory access**: Uses `memoryview` to avoid unnecessary data copies
|
|
229
|
+
- **Lazy chunk loading**: Only decompresses chunks when needed
|
|
230
|
+
- **Binary search**: Efficient time-range filtering using chunk indexes
|
|
231
|
+
- **Heap-based merging**: Optimal multi-file reading with automatic ID remapping
|
|
232
|
+
|
|
233
|
+
**Comparison with other libraries:**
|
|
234
|
+
|
|
235
|
+
| Feature | small-mcap | mcap (official) | rosbags | pybag |
|
|
236
|
+
| -------------------- | ---------- | --------------- | -------- | -------- |
|
|
237
|
+
| Performance | Fastest | Fast | Fast | Moderate |
|
|
238
|
+
| Zero dependencies | Yes | No | No | No |
|
|
239
|
+
| Non-seekable streams | Yes | Yes | No | No |
|
|
240
|
+
| Multi-file reading | Yes | No | Yes | Yes |
|
|
241
|
+
| ROS1 support | No | No | Yes | No |
|
|
242
|
+
| SQLite3 backend | No | No | Yes | No |
|
|
243
|
+
|
|
244
|
+
## Benchmarks
|
|
245
|
+
|
|
246
|
+
Benchmark results comparing small-mcap against mcap (official), rosbags, and pybag libraries using a nuScenes dataset (30,900 messages, 19.15s duration, 560 zstd chunks).
|
|
247
|
+
|
|
248
|
+
### Full File Read (Seekable)
|
|
249
|
+
|
|
250
|
+
```txt
|
|
251
|
+
----------------------------------------------------------------------------------------- benchmark 'full-seekable': 4 tests -----------------------------------------------------------------------------------------
|
|
252
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
253
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
254
|
+
test_benchmark_read[full-seekable-small_mcap] 442.7987 (1.0) 455.6817 (1.0) 448.6568 (1.0) 4.7916 (1.0) 448.9002 (1.0) 6.6608 (1.0) 2;0 2.2288 (1.0) 5 1
|
|
255
|
+
test_benchmark_read[full-seekable-rosbags] 502.2698 (1.13) 523.9009 (1.15) 510.3689 (1.14) 8.6200 (1.80) 506.1880 (1.13) 11.7877 (1.77) 1;0 1.9594 (0.88) 5 1
|
|
256
|
+
test_benchmark_read[full-seekable-pybag] 559.9649 (1.26) 596.3682 (1.31) 578.5393 (1.29) 13.1715 (2.75) 581.2666 (1.29) 15.2660 (2.29) 2;0 1.7285 (0.78) 5 1
|
|
257
|
+
test_benchmark_read[full-seekable-mcap] 574.9254 (1.30) 614.3929 (1.35) 594.9063 (1.33) 15.2217 (3.18) 593.9697 (1.32) 21.7823 (3.27) 2;0 1.6809 (0.75) 5 1
|
|
258
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
### Full File Read (Non-seekable Stream)
|
|
262
|
+
|
|
263
|
+
```txt
|
|
264
|
+
----------------------------------------------------------------------------------------- benchmark 'full-nonseekable': 2 tests ------------------------------------------------------------------------------------------
|
|
265
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
266
|
+
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
267
|
+
test_benchmark_read[full-nonseekable-small_mcap] 423.7839 (1.0) 454.8048 (1.0) 433.9779 (1.0) 12.7063 (1.0) 428.9654 (1.0) 14.9051 (1.0) 1;0 2.3043 (1.0) 5 1
|
|
268
|
+
test_benchmark_read[full-nonseekable-mcap] 595.2403 (1.40) 639.9618 (1.41) 616.2232 (1.42) 19.9259 (1.57) 607.5073 (1.42) 34.9823 (2.35) 2;0 1.6228 (0.70) 5 1
|
|
269
|
+
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
Note: rosbags and pybag require seekable streams and are skipped for non-seekable tests.
|
|
273
|
+
|
|
274
|
+
### Time-Range Filtered Read (Seekable)
|
|
275
|
+
|
|
276
|
+
```txt
|
|
277
|
+
---------------------------------------------------------------------------------------- benchmark 'time-seekable': 4 tests ----------------------------------------------------------------------------------------
|
|
278
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
279
|
+
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
280
|
+
test_benchmark_read[time-seekable-small_mcap] 119.7578 (1.0) 128.1060 (1.0) 123.7998 (1.0) 2.5372 (1.0) 123.7509 (1.0) 2.7058 (1.0) 3;0 8.0776 (1.0) 9 1
|
|
281
|
+
test_benchmark_read[time-seekable-pybag] 140.4680 (1.17) 159.2642 (1.24) 146.3533 (1.18) 6.4415 (2.54) 143.5709 (1.16) 6.2359 (2.30) 1;1 6.8328 (0.85) 7 1
|
|
282
|
+
test_benchmark_read[time-seekable-mcap] 146.3309 (1.22) 155.6906 (1.22) 150.6005 (1.22) 3.9600 (1.56) 150.2616 (1.21) 7.3775 (2.73) 2;0 6.6401 (0.82) 7 1
|
|
283
|
+
test_benchmark_read[time-seekable-rosbags] 509.0745 (4.25) 521.1191 (4.07) 512.3522 (4.14) 4.9971 (1.97) 510.4790 (4.13) 4.5978 (1.70) 1;1 1.9518 (0.24) 5 1
|
|
284
|
+
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
### Topic-Filtered Read (Seekable)
|
|
288
|
+
|
|
289
|
+
```txt
|
|
290
|
+
----------------------------------------------------------------------------------------- benchmark 'topic-seekable': 4 tests -----------------------------------------------------------------------------------------
|
|
291
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
292
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
293
|
+
test_benchmark_read[topic-seekable-small_mcap] 442.4237 (1.0) 454.9705 (1.0) 446.8667 (1.0) 4.9801 (1.05) 445.4813 (1.0) 6.3691 (1.0) 1;0 2.2378 (1.0) 5 1
|
|
294
|
+
test_benchmark_read[topic-seekable-rosbags] 502.9512 (1.14) 514.8851 (1.13) 508.7413 (1.14) 4.7252 (1.0) 507.7358 (1.14) 7.3330 (1.15) 2;0 1.9656 (0.88) 5 1
|
|
295
|
+
test_benchmark_read[topic-seekable-pybag] 507.1222 (1.15) 536.2468 (1.18) 520.2659 (1.16) 12.1789 (2.58) 517.0470 (1.16) 20.4743 (3.21) 2;0 1.9221 (0.86) 5 1
|
|
296
|
+
test_benchmark_read[topic-seekable-mcap] 548.7598 (1.24) 560.8708 (1.23) 554.8000 (1.24) 5.2638 (1.11) 554.2846 (1.24) 9.4890 (1.49) 2;0 1.8025 (0.81) 5 1
|
|
297
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
298
|
+
```
|
|
299
|
+
|
|
300
|
+
**Summary:**
|
|
301
|
+
|
|
302
|
+
- **1.13-1.42x faster** than mcap (official) across all scenarios
|
|
303
|
+
- **1.14-4.14x faster** than rosbags (especially for time-range filtering)
|
|
304
|
+
- **1.16-1.29x faster** than pybag for seekable streams
|
|
305
|
+
|
|
306
|
+
## Links
|
|
307
|
+
|
|
308
|
+
- [MCAP Specification](https://mcap.dev/)
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# small-mcap
|
|
2
|
+
|
|
3
|
+
Lightweight Python library for reading and writing MCAP files.
|
|
4
|
+
|
|
5
|
+
## Installation
|
|
6
|
+
|
|
7
|
+
```bash
|
|
8
|
+
uv add small-mcap
|
|
9
|
+
|
|
10
|
+
# With compression support
|
|
11
|
+
uv add small-mcap[compression] # ZSTD + LZ4
|
|
12
|
+
uv add small-mcap[zstd] # ZSTD only
|
|
13
|
+
uv add small-mcap[lz4] # LZ4 only
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
## Reader
|
|
17
|
+
|
|
18
|
+
### Basic read
|
|
19
|
+
|
|
20
|
+
```python
|
|
21
|
+
from small_mcap import read_message
|
|
22
|
+
|
|
23
|
+
with open("input.mcap", "rb") as f:
|
|
24
|
+
for schema, channel, message in read_message(f):
|
|
25
|
+
print(f"{channel.topic}: {message.data}")
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
### Read multiple inputs
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from small_mcap import read_message
|
|
32
|
+
|
|
33
|
+
with open("recording1.mcap", "rb") as f1, \
|
|
34
|
+
open("recording2.mcap", "rb") as f2, \
|
|
35
|
+
open("recording3.mcap", "rb") as f3:
|
|
36
|
+
for schema, channel, message in read_message([f1, f2, f3]):
|
|
37
|
+
print(f"{channel.topic}: {message.log_time}")
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
### Read with topic filtering
|
|
41
|
+
|
|
42
|
+
```python
|
|
43
|
+
from small_mcap import read_message, include_topics
|
|
44
|
+
|
|
45
|
+
with open("input.mcap", "rb") as f:
|
|
46
|
+
topics = ["/camera/image", "/lidar/points"]
|
|
47
|
+
for schema, channel, message in read_message(f, should_include=include_topics(topics)):
|
|
48
|
+
print(f"{channel.topic}: {len(message.data)} bytes")
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
### Read with time range
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from small_mcap import read_message
|
|
55
|
+
|
|
56
|
+
with open("input.mcap", "rb") as f:
|
|
57
|
+
start = 1000000000 # nanoseconds
|
|
58
|
+
end = 2000000000
|
|
59
|
+
for schema, channel, message in read_message(f, start_time_ns=start, end_time_ns=end):
|
|
60
|
+
print(f"{channel.topic} at {message.log_time}")
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
### Read decoded messages
|
|
64
|
+
|
|
65
|
+
```python
|
|
66
|
+
from small_mcap import read_message_decoded
|
|
67
|
+
import json
|
|
68
|
+
|
|
69
|
+
class JsonDecoderFactory:
|
|
70
|
+
def decoder_for(self, schema):
|
|
71
|
+
if schema.encoding == "json":
|
|
72
|
+
return lambda data: json.loads(data)
|
|
73
|
+
return None
|
|
74
|
+
|
|
75
|
+
with open("input.mcap", "rb") as f:
|
|
76
|
+
for msg in read_message_decoded(f, decoder_factories=[JsonDecoderFactory()]):
|
|
77
|
+
print(f"{msg.channel.topic}: {msg.decoded_message}")
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
### Read summary/metadata
|
|
81
|
+
|
|
82
|
+
```python
|
|
83
|
+
from small_mcap import get_summary, get_header
|
|
84
|
+
|
|
85
|
+
with open("input.mcap", "rb") as f:
|
|
86
|
+
summary = get_summary(f)
|
|
87
|
+
print(f"Messages: {summary.statistics.message_count}")
|
|
88
|
+
print(f"Duration: {summary.statistics.message_start_time} - {summary.statistics.message_end_time}")
|
|
89
|
+
|
|
90
|
+
for channel in summary.channels.values():
|
|
91
|
+
print(f" {channel.topic}: {channel.message_encoding}")
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
## Writer
|
|
95
|
+
|
|
96
|
+
### Basic write
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
from small_mcap import McapWriter
|
|
100
|
+
|
|
101
|
+
with open("output.mcap", "wb") as f:
|
|
102
|
+
writer = McapWriter(f)
|
|
103
|
+
writer.start(profile="", library="my-app")
|
|
104
|
+
|
|
105
|
+
# Add schema
|
|
106
|
+
schema_id = writer.add_schema("MySchema", "json", b'{"type": "object"}')
|
|
107
|
+
|
|
108
|
+
# Add channel
|
|
109
|
+
channel_id = writer.add_channel("/my/topic", "json", schema_id=schema_id)
|
|
110
|
+
|
|
111
|
+
# Add messages
|
|
112
|
+
for i in range(100):
|
|
113
|
+
writer.add_message(
|
|
114
|
+
channel_id,
|
|
115
|
+
log_time=i * 1000000, # nanoseconds
|
|
116
|
+
data=b'{"value": 42}',
|
|
117
|
+
publish_time=i * 1000000
|
|
118
|
+
)
|
|
119
|
+
|
|
120
|
+
writer.finish()
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
### Write with compression
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
from small_mcap import McapWriter, CompressionType
|
|
127
|
+
|
|
128
|
+
with open("output.mcap", "wb") as f:
|
|
129
|
+
writer = McapWriter(
|
|
130
|
+
f,
|
|
131
|
+
compression=CompressionType.ZSTD,
|
|
132
|
+
chunk_size=1024 * 1024 # 1MB chunks
|
|
133
|
+
)
|
|
134
|
+
writer.start(profile="", library="my-app")
|
|
135
|
+
|
|
136
|
+
schema_id = writer.add_schema("MySchema", "json", b"{}")
|
|
137
|
+
channel_id = writer.add_channel("/topic", "json", schema_id=schema_id)
|
|
138
|
+
|
|
139
|
+
for i in range(1000):
|
|
140
|
+
writer.add_message(channel_id, log_time=i*1000, data=b"data", publish_time=i*1000)
|
|
141
|
+
|
|
142
|
+
writer.finish()
|
|
143
|
+
```
|
|
144
|
+
|
|
145
|
+
### Write with encoder factory
|
|
146
|
+
|
|
147
|
+
```python
|
|
148
|
+
from small_mcap import McapWriter, EncoderFactory
|
|
149
|
+
import json
|
|
150
|
+
|
|
151
|
+
class JsonEncoder(EncoderFactory):
|
|
152
|
+
def get_schema_encoding(self, schema_name):
|
|
153
|
+
return "json", b'{"type": "object"}'
|
|
154
|
+
|
|
155
|
+
def get_channel_encoding(self, topic):
|
|
156
|
+
return "json"
|
|
157
|
+
|
|
158
|
+
def encode(self, topic, msg):
|
|
159
|
+
return json.dumps(msg).encode()
|
|
160
|
+
|
|
161
|
+
with open("output.mcap", "wb") as f:
|
|
162
|
+
writer = McapWriter(f)
|
|
163
|
+
writer.start(profile="", library="my-app")
|
|
164
|
+
|
|
165
|
+
encoder = JsonEncoder()
|
|
166
|
+
|
|
167
|
+
# Encoder automatically registers schemas and channels
|
|
168
|
+
for i in range(100):
|
|
169
|
+
msg = {"timestamp": i, "value": i * 2}
|
|
170
|
+
writer.add_message_encoded("/sensor/data", i * 1000, msg, encoder, publish_time=i * 1000)
|
|
171
|
+
|
|
172
|
+
writer.finish()
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
## Features
|
|
176
|
+
|
|
177
|
+
- Zero dependencies for core functionality
|
|
178
|
+
- Optional compression support (ZSTD, LZ4)
|
|
179
|
+
- Lazy chunk loading for efficient memory usage
|
|
180
|
+
- Topic and time-range filtering
|
|
181
|
+
- Automatic schema/channel registration
|
|
182
|
+
- CRC validation
|
|
183
|
+
- Fast summary/metadata access
|
|
184
|
+
|
|
185
|
+
## Performance
|
|
186
|
+
|
|
187
|
+
`small-mcap` is optimized for high-performance MCAP file reading with zero-copy operations and lazy chunk loading:
|
|
188
|
+
|
|
189
|
+
**Key Optimizations:**
|
|
190
|
+
|
|
191
|
+
- **Zero-copy memory access**: Uses `memoryview` to avoid unnecessary data copies
|
|
192
|
+
- **Lazy chunk loading**: Only decompresses chunks when needed
|
|
193
|
+
- **Binary search**: Efficient time-range filtering using chunk indexes
|
|
194
|
+
- **Heap-based merging**: Optimal multi-file reading with automatic ID remapping
|
|
195
|
+
|
|
196
|
+
**Comparison with other libraries:**
|
|
197
|
+
|
|
198
|
+
| Feature | small-mcap | mcap (official) | rosbags | pybag |
|
|
199
|
+
| -------------------- | ---------- | --------------- | -------- | -------- |
|
|
200
|
+
| Performance | Fastest | Fast | Fast | Moderate |
|
|
201
|
+
| Zero dependencies | Yes | No | No | No |
|
|
202
|
+
| Non-seekable streams | Yes | Yes | No | No |
|
|
203
|
+
| Multi-file reading | Yes | No | Yes | Yes |
|
|
204
|
+
| ROS1 support | No | No | Yes | No |
|
|
205
|
+
| SQLite3 backend | No | No | Yes | No |
|
|
206
|
+
|
|
207
|
+
## Benchmarks
|
|
208
|
+
|
|
209
|
+
Benchmark results comparing small-mcap against mcap (official), rosbags, and pybag libraries using a nuScenes dataset (30,900 messages, 19.15s duration, 560 zstd chunks).
|
|
210
|
+
|
|
211
|
+
### Full File Read (Seekable)
|
|
212
|
+
|
|
213
|
+
```txt
|
|
214
|
+
----------------------------------------------------------------------------------------- benchmark 'full-seekable': 4 tests -----------------------------------------------------------------------------------------
|
|
215
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
216
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
217
|
+
test_benchmark_read[full-seekable-small_mcap] 442.7987 (1.0) 455.6817 (1.0) 448.6568 (1.0) 4.7916 (1.0) 448.9002 (1.0) 6.6608 (1.0) 2;0 2.2288 (1.0) 5 1
|
|
218
|
+
test_benchmark_read[full-seekable-rosbags] 502.2698 (1.13) 523.9009 (1.15) 510.3689 (1.14) 8.6200 (1.80) 506.1880 (1.13) 11.7877 (1.77) 1;0 1.9594 (0.88) 5 1
|
|
219
|
+
test_benchmark_read[full-seekable-pybag] 559.9649 (1.26) 596.3682 (1.31) 578.5393 (1.29) 13.1715 (2.75) 581.2666 (1.29) 15.2660 (2.29) 2;0 1.7285 (0.78) 5 1
|
|
220
|
+
test_benchmark_read[full-seekable-mcap] 574.9254 (1.30) 614.3929 (1.35) 594.9063 (1.33) 15.2217 (3.18) 593.9697 (1.32) 21.7823 (3.27) 2;0 1.6809 (0.75) 5 1
|
|
221
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
### Full File Read (Non-seekable Stream)
|
|
225
|
+
|
|
226
|
+
```txt
|
|
227
|
+
----------------------------------------------------------------------------------------- benchmark 'full-nonseekable': 2 tests ------------------------------------------------------------------------------------------
|
|
228
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
229
|
+
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
230
|
+
test_benchmark_read[full-nonseekable-small_mcap] 423.7839 (1.0) 454.8048 (1.0) 433.9779 (1.0) 12.7063 (1.0) 428.9654 (1.0) 14.9051 (1.0) 1;0 2.3043 (1.0) 5 1
|
|
231
|
+
test_benchmark_read[full-nonseekable-mcap] 595.2403 (1.40) 639.9618 (1.41) 616.2232 (1.42) 19.9259 (1.57) 607.5073 (1.42) 34.9823 (2.35) 2;0 1.6228 (0.70) 5 1
|
|
232
|
+
---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
233
|
+
```
|
|
234
|
+
|
|
235
|
+
Note: rosbags and pybag require seekable streams and are skipped for non-seekable tests.
|
|
236
|
+
|
|
237
|
+
### Time-Range Filtered Read (Seekable)
|
|
238
|
+
|
|
239
|
+
```txt
|
|
240
|
+
---------------------------------------------------------------------------------------- benchmark 'time-seekable': 4 tests ----------------------------------------------------------------------------------------
|
|
241
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
242
|
+
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
243
|
+
test_benchmark_read[time-seekable-small_mcap] 119.7578 (1.0) 128.1060 (1.0) 123.7998 (1.0) 2.5372 (1.0) 123.7509 (1.0) 2.7058 (1.0) 3;0 8.0776 (1.0) 9 1
|
|
244
|
+
test_benchmark_read[time-seekable-pybag] 140.4680 (1.17) 159.2642 (1.24) 146.3533 (1.18) 6.4415 (2.54) 143.5709 (1.16) 6.2359 (2.30) 1;1 6.8328 (0.85) 7 1
|
|
245
|
+
test_benchmark_read[time-seekable-mcap] 146.3309 (1.22) 155.6906 (1.22) 150.6005 (1.22) 3.9600 (1.56) 150.2616 (1.21) 7.3775 (2.73) 2;0 6.6401 (0.82) 7 1
|
|
246
|
+
test_benchmark_read[time-seekable-rosbags] 509.0745 (4.25) 521.1191 (4.07) 512.3522 (4.14) 4.9971 (1.97) 510.4790 (4.13) 4.5978 (1.70) 1;1 1.9518 (0.24) 5 1
|
|
247
|
+
--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
248
|
+
```
|
|
249
|
+
|
|
250
|
+
### Topic-Filtered Read (Seekable)
|
|
251
|
+
|
|
252
|
+
```txt
|
|
253
|
+
----------------------------------------------------------------------------------------- benchmark 'topic-seekable': 4 tests -----------------------------------------------------------------------------------------
|
|
254
|
+
Name (time in ms) Min Max Mean StdDev Median IQR Outliers OPS Rounds Iterations
|
|
255
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
256
|
+
test_benchmark_read[topic-seekable-small_mcap] 442.4237 (1.0) 454.9705 (1.0) 446.8667 (1.0) 4.9801 (1.05) 445.4813 (1.0) 6.3691 (1.0) 1;0 2.2378 (1.0) 5 1
|
|
257
|
+
test_benchmark_read[topic-seekable-rosbags] 502.9512 (1.14) 514.8851 (1.13) 508.7413 (1.14) 4.7252 (1.0) 507.7358 (1.14) 7.3330 (1.15) 2;0 1.9656 (0.88) 5 1
|
|
258
|
+
test_benchmark_read[topic-seekable-pybag] 507.1222 (1.15) 536.2468 (1.18) 520.2659 (1.16) 12.1789 (2.58) 517.0470 (1.16) 20.4743 (3.21) 2;0 1.9221 (0.86) 5 1
|
|
259
|
+
test_benchmark_read[topic-seekable-mcap] 548.7598 (1.24) 560.8708 (1.23) 554.8000 (1.24) 5.2638 (1.11) 554.2846 (1.24) 9.4890 (1.49) 2;0 1.8025 (0.81) 5 1
|
|
260
|
+
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
|
|
261
|
+
```
|
|
262
|
+
|
|
263
|
+
**Summary:**
|
|
264
|
+
|
|
265
|
+
- **1.13-1.42x faster** than mcap (official) across all scenarios
|
|
266
|
+
- **1.14-4.14x faster** than rosbags (especially for time-range filtering)
|
|
267
|
+
- **1.16-1.29x faster** than pybag for seekable streams
|
|
268
|
+
|
|
269
|
+
## Links
|
|
270
|
+
|
|
271
|
+
- [MCAP Specification](https://mcap.dev/)
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "small-mcap"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Lightweight Python library for reading and writing MCAP files"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
license = {text = "GPL-3.0"}
|
|
8
|
+
authors = [
|
|
9
|
+
{name = "Marko Bausch"}
|
|
10
|
+
]
|
|
11
|
+
keywords = ["mcap", "robotics", "ros", "ros2", "serialization", "logging"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Development Status :: 4 - Beta",
|
|
14
|
+
"Intended Audience :: Developers",
|
|
15
|
+
"Intended Audience :: Science/Research",
|
|
16
|
+
"License :: OSI Approved :: GNU General Public License v3 (GPLv3)",
|
|
17
|
+
"Operating System :: OS Independent",
|
|
18
|
+
"Programming Language :: Python :: 3",
|
|
19
|
+
"Programming Language :: Python :: 3.10",
|
|
20
|
+
"Programming Language :: Python :: 3.11",
|
|
21
|
+
"Programming Language :: Python :: 3.12",
|
|
22
|
+
"Programming Language :: Python :: 3.13",
|
|
23
|
+
"Topic :: Scientific/Engineering",
|
|
24
|
+
"Topic :: System :: Logging",
|
|
25
|
+
"Typing :: Typed",
|
|
26
|
+
]
|
|
27
|
+
dependencies = []
|
|
28
|
+
|
|
29
|
+
[project.urls]
|
|
30
|
+
Homepage = "https://github.com/mrkbac/robotic-tools"
|
|
31
|
+
Repository = "https://github.com/mrkbac/robotic-tools"
|
|
32
|
+
Issues = "https://github.com/mrkbac/robotic-tools/issues"
|
|
33
|
+
|
|
34
|
+
[project.optional-dependencies]
|
|
35
|
+
zstd = ["zstandard>=0.23.0"]
|
|
36
|
+
lz4 = ["lz4>=4.3.3"]
|
|
37
|
+
compression = ["small-mcap[zstd]", "small-mcap[lz4]"]
|
|
38
|
+
dev = [
|
|
39
|
+
"mcap>=1.0.0",
|
|
40
|
+
"rosbags>=0.11.0",
|
|
41
|
+
"pybag-sdk>=0.6.0",
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
[build-system]
|
|
45
|
+
requires = ["uv_build>=0.8.9,<0.9.0"]
|
|
46
|
+
build-backend = "uv_build"
|
|
47
|
+
|
|
48
|
+
[[tool.mypy.overrides]]
|
|
49
|
+
module = "lz4.frame"
|
|
50
|
+
ignore_missing_imports = true
|
|
51
|
+
|
|
52
|
+
[dependency-groups]
|
|
53
|
+
dev = [
|
|
54
|
+
"pytest-mock>=3.15.1",
|
|
55
|
+
]
|