tmsgpack 0.1.1__tar.gz → 0.1.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/COPYING +1 -1
- {tmsgpack-0.1.1/tmsgpack.egg-info → tmsgpack-0.1.2}/PKG-INFO +59 -210
- tmsgpack-0.1.2/README.md +278 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/contexts_for_tests.py +1 -1
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_buffer.py +6 -6
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_case.py +5 -23
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_except.py +18 -7
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_format.py +6 -11
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_limits.py +9 -9
- tmsgpack-0.1.2/test/test_memoryview.py +72 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_newspec.py +9 -11
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_pack.py +8 -28
- tmsgpack-0.1.2/test/test_roundtrip.py +51 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_seq.py +1 -1
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_sequnpack.py +3 -3
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_trace_from_x.py +7 -24
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_typed_objects.py +2 -2
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_unpack.py +3 -3
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/__init__.py +31 -57
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_ctmsgpack.cpp +1620 -1864
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_packer.pyx +33 -47
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_unpacker.pyx +20 -42
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/buff_converter.h +1 -1
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/fallback.py +59 -76
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/pack.h +0 -22
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/pack_template.h +4 -7
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack.h +21 -39
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack_define.h +4 -4
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack_template.h +13 -13
- {tmsgpack-0.1.1 → tmsgpack-0.1.2/tmsgpack.egg-info}/PKG-INFO +59 -210
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/SOURCES.txt +1 -1
- tmsgpack-0.1.1/README.md +0 -429
- tmsgpack-0.1.1/test/test_memoryview.py +0 -100
- tmsgpack-0.1.1/tmsgpack/ext.py +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/MANIFEST.in +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/pyproject.toml +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/setup.cfg +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/setup.py +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_subtype.py +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_ctmsgpack.pyx +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/exceptions.py +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/sysdep.h +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/dependency_links.txt +0 -0
- {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/top_level.txt +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: tmsgpack
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.2
|
|
4
4
|
Summary: Typed MessagePack serializer (similar but incompatible with msgpack)
|
|
5
5
|
Home-page: https://github.com/Yaakov-Belch/tmsgpack
|
|
6
6
|
Author: Inada Naoki, Yaakov Belch
|
|
@@ -24,8 +24,10 @@ Dynamic: license-file
|
|
|
24
24
|
|
|
25
25
|
# tmsgpack: Typed MessagePack-inspired pack/unpack component
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
See also: [FORMAT.md](FORMAT.md) and [DEVELOP.md](DEVELOP.md)
|
|
28
|
+
|
|
29
|
+
The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
|
|
30
|
+
tuples/lists) with an `object_type` property.
|
|
29
31
|
|
|
30
32
|
Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
|
|
31
33
|
solution. It is a composable component that helps you to build end-to-end communication
|
|
@@ -47,7 +49,10 @@ This python package makes a minimal (backwards-incompatible) modification to the
|
|
|
47
49
|
msgpack format to make all this elegantly possible. This package is based on
|
|
48
50
|
`msgpack v1.0.5`.
|
|
49
51
|
|
|
50
|
-
##
|
|
52
|
+
## Installation
|
|
53
|
+
```bash
|
|
54
|
+
pip install tmsgpack
|
|
55
|
+
```
|
|
51
56
|
|
|
52
57
|
## Usage
|
|
53
58
|
Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
|
|
@@ -93,7 +98,7 @@ class MinimalPackCtrl:
|
|
|
93
98
|
class MinimalUnpackCtrl:
|
|
94
99
|
def from_dict(self, ctype, dct):
|
|
95
100
|
raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
|
|
96
|
-
def
|
|
101
|
+
def from_tuple(self, ctype, lst):
|
|
97
102
|
raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
|
|
98
103
|
options: UnpackConfig
|
|
99
104
|
|
|
@@ -106,15 +111,15 @@ minimal_unpack_ctrl = uctrl()
|
|
|
106
111
|
|
|
107
112
|
## The API and configuration
|
|
108
113
|
As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
|
|
109
|
-
object provides the methods `from_dict` and `
|
|
114
|
+
object provides the methods `from_dict` and `from_tuple`:
|
|
110
115
|
```python
|
|
111
116
|
as_dict, data_type, data = pack_ctrl.from(obj)
|
|
112
117
|
|
|
113
118
|
# When `as_dict` is true, then `data` should be a dictionary.
|
|
114
|
-
# When `as_dict` is false, then `data` should be a list.
|
|
119
|
+
# When `as_dict` is false, then `data` should be a tuple or a list.
|
|
115
120
|
|
|
116
121
|
unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
|
|
117
|
-
unpacked = unpack_ctrl.
|
|
122
|
+
unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
|
|
118
123
|
```
|
|
119
124
|
|
|
120
125
|
## PackConfig configuration objects for pack_ctrl
|
|
@@ -123,35 +128,24 @@ unpacked = unpack_ctrl.from_list(data_type, data) # used when as_dict is false.
|
|
|
123
128
|
from tmsgpack import PackConfig
|
|
124
129
|
|
|
125
130
|
config = PackConfig(
|
|
126
|
-
|
|
127
|
-
tuple_as_list=True, strict_types=False,
|
|
128
|
-
unicode_errors='strict', sort_keys=False,
|
|
131
|
+
p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
|
|
129
132
|
)
|
|
130
133
|
"""
|
|
131
134
|
Config object for pack_ctrl.options
|
|
132
135
|
|
|
133
|
-
:param bool
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
:
|
|
137
|
-
Use bin type introduced in tmsgpack spec 2.0 for bytes.
|
|
138
|
-
It also enables str8 type for unicode. (default: True)
|
|
139
|
-
|
|
140
|
-
:param bool tuple_as_list:
|
|
141
|
-
If true, tuples are serialized as lists. (default: True)
|
|
142
|
-
Otherwise, tuples are passed to pack_ctrl.from_obj(?).
|
|
136
|
+
:param bool p_shortcuts:
|
|
137
|
+
If true, pack dicts, tuples, lists without calling from_obj(...).
|
|
138
|
+
Object types: {...} => None; (...) => None; [...] => False
|
|
139
|
+
(default: True)
|
|
143
140
|
|
|
144
|
-
:param bool
|
|
145
|
-
|
|
146
|
-
Derived classes are distinct and passed to pack_ctrl.from_obj(?).
|
|
147
|
-
Dicts, lists and tuples are not affected by strict_types.
|
|
141
|
+
:param bool sort_keys:
|
|
142
|
+
Sort output dictionaries by key. (default: False)
|
|
148
143
|
|
|
149
|
-
:param
|
|
150
|
-
|
|
151
|
-
DO NOT USE THIS!! This option is kept for very specific usage.
|
|
144
|
+
:param bool p_str_keys:
|
|
145
|
+
If true, accept only strings as dict keys. (default: False)
|
|
152
146
|
|
|
153
|
-
:param bool
|
|
154
|
-
|
|
147
|
+
:param bool use_single_float:
|
|
148
|
+
Use single precision float type for float. (default: False)
|
|
155
149
|
"""
|
|
156
150
|
```
|
|
157
151
|
## UnpackConfig configuration objects for unpack_ctrl
|
|
@@ -160,55 +154,43 @@ Config object for pack_ctrl.options
|
|
|
160
154
|
from tmsgpack import UnpackConfig
|
|
161
155
|
|
|
162
156
|
config = UnpackConfig(
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
unicode_errors='strict', max_buffer_size=0,
|
|
157
|
+
u_shortcuts=True, u_str_keys=False,
|
|
158
|
+
read_size=16*1024, max_buffer_size=0,
|
|
166
159
|
max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
|
|
167
160
|
)
|
|
168
161
|
"""
|
|
169
162
|
Config object for unpack_ctrl.options
|
|
170
163
|
|
|
171
|
-
:param
|
|
172
|
-
|
|
164
|
+
:param bool u_shortcuts:
|
|
165
|
+
If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
|
|
166
|
+
Object types: None => {...}/(...), False => [...]
|
|
167
|
+
(default: True)
|
|
173
168
|
|
|
174
|
-
:param bool
|
|
175
|
-
|
|
169
|
+
:param bool u_str_keys:
|
|
170
|
+
If true, accept only strings as dict keys. (default: False)
|
|
176
171
|
|
|
177
|
-
:param
|
|
178
|
-
|
|
179
|
-
Otherwise, unpack to Python str by decoding with UTF-8 encoding (default: False).
|
|
172
|
+
:param int read_size:
|
|
173
|
+
Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
|
|
180
174
|
|
|
181
|
-
:param
|
|
182
|
-
|
|
175
|
+
:param int max_buffer_size:
|
|
176
|
+
(default: 100*1024*1024 (100MiB))
|
|
177
|
+
Limits size of data waiting unpacked. 0 means 2**32-1.
|
|
178
|
+
Raises `BufferFull` exception when it is insufficient.
|
|
179
|
+
You should set this parameter when unpacking data from untrusted source.
|
|
183
180
|
|
|
184
|
-
:param
|
|
185
|
-
|
|
186
|
-
Otherwise, as dicts (default: False).
|
|
181
|
+
:param int max_str_len:
|
|
182
|
+
Limits max length of str. (default: max_buffer_size)
|
|
187
183
|
|
|
188
|
-
:param
|
|
189
|
-
|
|
190
|
-
This option should be used only when you have tmsgpack data which
|
|
191
|
-
contains invalid UTF-8 string.
|
|
184
|
+
:param int max_bin_len:
|
|
185
|
+
Limits max length of bin. (default: max_buffer_size)
|
|
192
186
|
|
|
193
|
-
:param int
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
Raises `BufferFull` exception when it is insufficient.
|
|
197
|
-
You should set this parameter when unpacking data from untrusted source.
|
|
187
|
+
:param int max_list_len:
|
|
188
|
+
Limits max length of list.
|
|
189
|
+
(default: max_buffer_size)
|
|
198
190
|
|
|
199
|
-
:param int
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
:param int max_bin_len:
|
|
203
|
-
Limits max length of bin. (default: max_buffer_size)
|
|
204
|
-
|
|
205
|
-
:param int max_list_len:
|
|
206
|
-
Limits max length of list.
|
|
207
|
-
(default: max_buffer_size)
|
|
208
|
-
|
|
209
|
-
:param int max_dict_len:
|
|
210
|
-
Limits max length of dict.
|
|
211
|
-
(default: max_buffer_size//2)
|
|
191
|
+
:param int max_dict_len:
|
|
192
|
+
Limits max length of dict.
|
|
193
|
+
(default: max_buffer_size//2)
|
|
212
194
|
"""
|
|
213
195
|
```
|
|
214
196
|
## Packing and Unpacking dataclass objects
|
|
@@ -248,7 +230,7 @@ class TypedPackCtrl:
|
|
|
248
230
|
class TypedUnpackCtrl:
|
|
249
231
|
constructors: Dict[str, callable]
|
|
250
232
|
def from_dict(self, ctype, data): return self.constructors[ctype](**data)
|
|
251
|
-
def
|
|
233
|
+
def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
|
|
252
234
|
options: UnpackConfig
|
|
253
235
|
|
|
254
236
|
def unpack(self, packed):
|
|
@@ -287,33 +269,20 @@ class Bar:
|
|
|
287
269
|
x: int = 1
|
|
288
270
|
y: str = 'Y'
|
|
289
271
|
as_list = True
|
|
290
|
-
|
|
291
|
-
@dataclass
|
|
292
|
-
class Add:
|
|
293
|
-
x: int = 10
|
|
294
|
-
y: int = 20
|
|
295
|
-
|
|
296
|
-
class Expr:
|
|
297
|
-
@staticmethod
|
|
298
|
-
def Add(x:int, y:int): return x+y
|
|
299
|
-
@staticmethod
|
|
300
|
-
def tuple(*args): return args
|
|
301
272
|
```
|
|
302
273
|
Notes:
|
|
303
274
|
* Class Bar has a class property `as_list=True`. It will be packed compactly as
|
|
304
275
|
a parameter value list.
|
|
305
|
-
* The function (static method) `Expr.Add` has the same name as the class constructor
|
|
306
|
-
`Add`. We will soon exploit this...
|
|
307
276
|
|
|
308
277
|
Here is a simple test runner to be used for several tests:
|
|
309
278
|
```python
|
|
310
279
|
def run(input, expected=None):
|
|
311
280
|
if expected is None: expected = input
|
|
312
281
|
|
|
313
|
-
constructors = [Foo, Bar
|
|
282
|
+
constructors = [Foo, Bar]
|
|
314
283
|
|
|
315
|
-
pack_ctrl
|
|
316
|
-
unpack_ctrl
|
|
284
|
+
pack_ctrl = pc()
|
|
285
|
+
unpack_ctrl = uc(constructors)
|
|
317
286
|
|
|
318
287
|
packed = pack_ctrl.pack(input)
|
|
319
288
|
output = unpack_ctrl.unpack(packed)
|
|
@@ -321,133 +290,13 @@ def run(input, expected=None):
|
|
|
321
290
|
assert output == expected
|
|
322
291
|
```
|
|
323
292
|
|
|
324
|
-
|
|
325
|
-
correctly:
|
|
293
|
+
The `Foo()` and `Bar()` objects are packed and unpacked correctly:
|
|
326
294
|
```python
|
|
327
295
|
def test_typed_foobar():
|
|
328
|
-
run(Foo())
|
|
329
|
-
run(Bar())
|
|
330
|
-
run((1,2,3)) # Tuples are encoded as a typed list with object_type=
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
The second test shows that we can encode an object tree into an
|
|
334
|
-
binary expression buffer. When unpacking this expression buffer, the expression
|
|
335
|
-
is evaluated.
|
|
336
|
-
```python
|
|
337
|
-
def test_simple_expression():
|
|
338
|
-
run(Add(Add(1,2), Add(2,3)), 8) # Unpacking is expression evaluation.
|
|
339
|
-
```
|
|
340
|
-
Variables (parameters) can be implemented using a dictionary inside the `unpack_ctrl`
|
|
341
|
-
object.
|
|
342
|
-
|
|
343
|
-
## Development: Environment and testing
|
|
344
|
-
|
|
345
|
-
```
|
|
346
|
-
# Clone the repository
|
|
347
|
-
cd ~/git/
|
|
348
|
-
# git clone https://github.com/Yaakov-Belch/tmsgpack.git # You need permissions for that.
|
|
349
|
-
git clone https://github.com/Yaakov-Belch/tmsgpack.git
|
|
350
|
-
|
|
351
|
-
# Create a virtual environment in your project directory
|
|
352
|
-
cd ~/git/tmsgpack
|
|
353
|
-
python -m venv venv
|
|
354
|
-
|
|
355
|
-
# Activate the virtual environment
|
|
356
|
-
source venv/bin/activate
|
|
357
|
-
|
|
358
|
-
# Upgrade pip (recommended)
|
|
359
|
-
pip install --upgrade pip
|
|
360
|
-
|
|
361
|
-
# Install the required dependencies
|
|
362
|
-
pip install -r requirements.txt
|
|
363
|
-
|
|
364
|
-
# Now you can run the tests
|
|
365
|
-
make test
|
|
366
|
-
|
|
367
|
-
# Run one test
|
|
368
|
-
pytest -v test/test_typed_objects.py
|
|
369
|
-
```
|
|
370
|
-
|
|
371
|
-
## Build, test, and publish to PyPI
|
|
372
|
-
Follow the steps above in "Development": Clone this repository, create and activate
|
|
373
|
-
a virtual environment, upgrade pip, install required dependencies; make test.
|
|
374
|
-
```bash
|
|
375
|
-
cd ~/git/tmsgpack
|
|
376
|
-
source venv/bin/activate
|
|
377
|
-
|
|
378
|
-
pip install build
|
|
379
|
-
rm -rf dist/ # remove old builds
|
|
380
|
-
python -m build # puts results in dist/
|
|
381
|
-
|
|
382
|
-
# Test the new package -- in a fresh test environment:
|
|
383
|
-
|
|
384
|
-
# Create a fresh test environment
|
|
385
|
-
deactivate
|
|
386
|
-
python -m venv venv-test
|
|
387
|
-
source venv-test/bin/activate
|
|
388
|
-
pip install --upgrade pip
|
|
389
|
-
pip install dist/tmsgpack-*.whl # use the latest whl built.
|
|
390
|
-
|
|
391
|
-
pip install pytest
|
|
392
|
-
|
|
393
|
-
# These tests uses the source files:
|
|
394
|
-
pytest -v test/
|
|
395
|
-
TMSGPACK_PUREPYTHON=1 pytest -v test
|
|
396
|
-
python -c "import tmsgpack; print(tmsgpack.__file__)"
|
|
397
|
-
|
|
398
|
-
# These tests use the installed files:
|
|
399
|
-
cd test
|
|
400
|
-
pytest -v .
|
|
401
|
-
TMSGPACK_PUREPYTHON=1 pytest -v .
|
|
402
|
-
python -c "import tmsgpack; print(tmsgpack.__file__)"
|
|
403
|
-
cd ..
|
|
404
|
-
deactivate
|
|
405
|
-
rm -rf venv-test
|
|
406
|
-
```
|
|
407
|
-
Upload to PyPi:
|
|
408
|
-
```bash
|
|
409
|
-
pip install twine
|
|
410
|
-
|
|
411
|
-
# Register at pypi.org if you haven't
|
|
412
|
-
# Connecting to upload.pypi.org
|
|
413
|
-
# twine upload dist/* # This does not work: Platform incompatible.
|
|
414
|
-
twine upload dist/tmsgpack-*.tar.gz
|
|
296
|
+
run(Foo()) # Encoded as a typed dict
|
|
297
|
+
run(Bar()) # Encoded as a typed list
|
|
298
|
+
run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
|
|
299
|
+
run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
|
|
300
|
+
run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
|
|
415
301
|
```
|
|
416
302
|
|
|
417
|
-
Test in a separate virtual environment
|
|
418
|
-
```bash
|
|
419
|
-
deactivate
|
|
420
|
-
rm -rf ~/test-tmsgpack
|
|
421
|
-
mkdir -p ~/test-tmsgpack
|
|
422
|
-
cd ~/test-tmsgpack
|
|
423
|
-
python -m venv venv-test
|
|
424
|
-
source venv-test/bin/activate
|
|
425
|
-
pip install --upgrade pip
|
|
426
|
-
pip install tmsgpack pytest
|
|
427
|
-
python -c "import tmsgpack; print('Version:', tmsgpack.__version__)"
|
|
428
|
-
pytest -v ~/git/tmsgpack/test/
|
|
429
|
-
TMSGPACK_PUREPYTHON=1 pytest -v ~/git/tmsgpack/test/
|
|
430
|
-
```
|
|
431
|
-
|
|
432
|
-
## The tmsgpack format (version 0.1.0)
|
|
433
|
-
The msgpack format defines two types of containers: maps and arrays (dicts and lists).
|
|
434
|
-
They are encoded by a `container_header` that identifies the container type and the
|
|
435
|
-
number of key-value pairs or array-elements that follow after the `container_header`:
|
|
436
|
-
```
|
|
437
|
-
dict_container_header(3) key1 value1 key2 value2 key3 value3
|
|
438
|
-
array_container_header(3) element1 element2 element3
|
|
439
|
-
```
|
|
440
|
-
The tmsgpack format uses the same rules -- but adds an `object_type` entry right
|
|
441
|
-
after every `container_header`:
|
|
442
|
-
```
|
|
443
|
-
dict_container_header(3) object_type key1 value1 key2 value2 key3 value3
|
|
444
|
-
array_container_header(3) object_type element1 element2 element3
|
|
445
|
-
```
|
|
446
|
-
|
|
447
|
-
## Future extensibility
|
|
448
|
-
In msgpack, possible `ExtType` values for the first `data_header` byte declare
|
|
449
|
-
this data element as an msgpack extension. The tmsgpack format does not use this
|
|
450
|
-
extension mechanism -- and these eight values are available for future extensions.
|
|
451
|
-
|
|
452
|
-
The value `0xC1` is never used by the original msgpack specification. It is also
|
|
453
|
-
available for future extensions of the tmsgpack format.
|
tmsgpack-0.1.2/README.md
ADDED
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
# tmsgpack: Typed MessagePack-inspired pack/unpack component
|
|
2
|
+
|
|
3
|
+
See also: [FORMAT.md](FORMAT.md) and [DEVELOP.md](DEVELOP.md)
|
|
4
|
+
|
|
5
|
+
The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
|
|
6
|
+
tuples/lists) with an `object_type` property.
|
|
7
|
+
|
|
8
|
+
Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
|
|
9
|
+
solution. It is a composable component that helps you to build end-to-end communication
|
|
10
|
+
solutions.
|
|
11
|
+
|
|
12
|
+
Your system solution design will make decisions on:
|
|
13
|
+
* What objects are serializable and what objects are not.
|
|
14
|
+
* What code to use (and, maybe, dynamically load) to instantiate serialized objects.
|
|
15
|
+
* How to represent objects that are unpacked but not supposed to 'live' in this process.
|
|
16
|
+
* How to share dynamic data between different packs/unpacks.
|
|
17
|
+
* How to asynchronously load and integrate shared data from different sources.
|
|
18
|
+
* How to map typed object meaning between different programming languages.
|
|
19
|
+
* Whether and how to convert persisted "old" data to current, new semantics (schemas).
|
|
20
|
+
* How much to attach explicit meaning and predictable schemas to your object types.
|
|
21
|
+
* Whether or not to use the 'expression execution' capabilities of tmsgpack.
|
|
22
|
+
* etc.
|
|
23
|
+
|
|
24
|
+
This python package makes a minimal (backwards-incompatible) modification to the
|
|
25
|
+
msgpack format to make all this elegantly possible. This package is based on
|
|
26
|
+
`msgpack v1.0.5`.
|
|
27
|
+
|
|
28
|
+
## Installation
|
|
29
|
+
```bash
|
|
30
|
+
pip install tmsgpack
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
## Usage
|
|
34
|
+
Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
|
|
35
|
+
below for details how to create them):
|
|
36
|
+
```python
|
|
37
|
+
from tmsgpack import packb, unpackb
|
|
38
|
+
packed = packb(data, pack_ctrl=pack_ctrl)
|
|
39
|
+
unpacked = unpackb(packed, unpack_ctrl=unpack_ctrl)
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
## Streaming unpacking
|
|
43
|
+
For multiple uses, you can use packer and unpacker objects:
|
|
44
|
+
```python
|
|
45
|
+
from tmsgpack import Packer
|
|
46
|
+
packer = Packer(pack_ctrl=pack_ctrl)
|
|
47
|
+
|
|
48
|
+
packed = packer.pack(data) # Send these packages via a socket...
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
from tmsgpack import Packer, Unpacker
|
|
52
|
+
|
|
53
|
+
unpacker = Unpacker(unpack_ctrl=unpack_ctrl)
|
|
54
|
+
while buf := sock.recv(1024**2):
|
|
55
|
+
unpacker.feed(buf)
|
|
56
|
+
for o in unpacker:
|
|
57
|
+
process(o)
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Minimal pack_ctrl and unpack_ctrl objects
|
|
61
|
+
Minimal controllers allow only JSON-like objects and raise errors when you ask for more
|
|
62
|
+
(below, we show examples for more useful controllers):
|
|
63
|
+
```python
|
|
64
|
+
from tmsgpack import PackConfig, UnpackConfig
|
|
65
|
+
from dataclasses import dataclass
|
|
66
|
+
|
|
67
|
+
@dataclass
|
|
68
|
+
class MinimalPackCtrl:
|
|
69
|
+
def from_obj(self, obj):
|
|
70
|
+
raise TypeError(f'Cannot serialize {type(obj)} object.')
|
|
71
|
+
options: PackConfig
|
|
72
|
+
|
|
73
|
+
@dataclass
|
|
74
|
+
class MinimalUnpackCtrl:
|
|
75
|
+
def from_dict(self, ctype, dct):
|
|
76
|
+
raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
|
|
77
|
+
def from_tuple(self, ctype, lst):
|
|
78
|
+
raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
|
|
79
|
+
options: UnpackConfig
|
|
80
|
+
|
|
81
|
+
def pctrl(**kwargs): return MinimalPackCtrl(options=PackConfig(**kwargs))
|
|
82
|
+
def uctrl(**kwargs): return MinimalUnpackCtrl(options=UnpackConfig(**kwargs))
|
|
83
|
+
|
|
84
|
+
minimal_pack_ctrl = pctrl()
|
|
85
|
+
minimal_unpack_ctrl = uctrl()
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
## The API and configuration
|
|
89
|
+
As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
|
|
90
|
+
object provides the methods `from_dict` and `from_tuple`:
|
|
91
|
+
```python
|
|
92
|
+
as_dict, data_type, data = pack_ctrl.from(obj)
|
|
93
|
+
|
|
94
|
+
# When `as_dict` is true, then `data` should be a dictionary.
|
|
95
|
+
# When `as_dict` is false, then `data` should be a tuple or a list.
|
|
96
|
+
|
|
97
|
+
unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
|
|
98
|
+
unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
## PackConfig configuration objects for pack_ctrl
|
|
102
|
+
`PackConfig` objects provide the following options:
|
|
103
|
+
```python
|
|
104
|
+
from tmsgpack import PackConfig
|
|
105
|
+
|
|
106
|
+
config = PackConfig(
|
|
107
|
+
p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
|
|
108
|
+
)
|
|
109
|
+
"""
|
|
110
|
+
Config object for pack_ctrl.options
|
|
111
|
+
|
|
112
|
+
:param bool p_shortcuts:
|
|
113
|
+
If true, pack dicts, tuples, lists without calling from_obj(...).
|
|
114
|
+
Object types: {...} => None; (...) => None; [...] => False
|
|
115
|
+
(default: True)
|
|
116
|
+
|
|
117
|
+
:param bool sort_keys:
|
|
118
|
+
Sort output dictionaries by key. (default: False)
|
|
119
|
+
|
|
120
|
+
:param bool p_str_keys:
|
|
121
|
+
If true, accept only strings as dict keys. (default: False)
|
|
122
|
+
|
|
123
|
+
:param bool use_single_float:
|
|
124
|
+
Use single precision float type for float. (default: False)
|
|
125
|
+
"""
|
|
126
|
+
```
|
|
127
|
+
## UnpackConfig configuration objects for unpack_ctrl
|
|
128
|
+
`UnpackConfig` objects provide the following options:
|
|
129
|
+
```python
|
|
130
|
+
from tmsgpack import UnpackConfig
|
|
131
|
+
|
|
132
|
+
config = UnpackConfig(
|
|
133
|
+
u_shortcuts=True, u_str_keys=False,
|
|
134
|
+
read_size=16*1024, max_buffer_size=0,
|
|
135
|
+
max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
|
|
136
|
+
)
|
|
137
|
+
"""
|
|
138
|
+
Config object for unpack_ctrl.options
|
|
139
|
+
|
|
140
|
+
:param bool u_shortcuts:
|
|
141
|
+
If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
|
|
142
|
+
Object types: None => {...}/(...), False => [...]
|
|
143
|
+
(default: True)
|
|
144
|
+
|
|
145
|
+
:param bool u_str_keys:
|
|
146
|
+
If true, accept only strings as dict keys. (default: False)
|
|
147
|
+
|
|
148
|
+
:param int read_size:
|
|
149
|
+
Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
|
|
150
|
+
|
|
151
|
+
:param int max_buffer_size:
|
|
152
|
+
(default: 100*1024*1024 (100MiB))
|
|
153
|
+
Limits size of data waiting unpacked. 0 means 2**32-1.
|
|
154
|
+
Raises `BufferFull` exception when it is insufficient.
|
|
155
|
+
You should set this parameter when unpacking data from untrusted source.
|
|
156
|
+
|
|
157
|
+
:param int max_str_len:
|
|
158
|
+
Limits max length of str. (default: max_buffer_size)
|
|
159
|
+
|
|
160
|
+
:param int max_bin_len:
|
|
161
|
+
Limits max length of bin. (default: max_buffer_size)
|
|
162
|
+
|
|
163
|
+
:param int max_list_len:
|
|
164
|
+
Limits max length of list.
|
|
165
|
+
(default: max_buffer_size)
|
|
166
|
+
|
|
167
|
+
:param int max_dict_len:
|
|
168
|
+
Limits max length of dict.
|
|
169
|
+
(default: max_buffer_size//2)
|
|
170
|
+
"""
|
|
171
|
+
```
|
|
172
|
+
## Packing and Unpacking dataclass objects
|
|
173
|
+
Here are the parts of one unit test that shows end-to-end packing and unpacking
|
|
174
|
+
of dataclass objects:
|
|
175
|
+
|
|
176
|
+
For the setup, we import tools and define the controllers:
|
|
177
|
+
```python
|
|
178
|
+
from tmsgpack import packb, unpackb, PackConfig, UnpackConfig
|
|
179
|
+
from dataclasses import dataclass, is_dataclass, fields
|
|
180
|
+
from typing import Dict
|
|
181
|
+
|
|
182
|
+
@dataclass
|
|
183
|
+
class TypedPackCtrl:
|
|
184
|
+
def from_obj(self, obj):
|
|
185
|
+
if type(obj) is tuple: return False, 'tuple', obj # Special case for tuples.
|
|
186
|
+
if not is_dataclass(obj): raise TypeError(f'Cannot serialize {type(obj)} object.')
|
|
187
|
+
as_dict = not getattr(obj, 'as_list', False)
|
|
188
|
+
object_type = obj.__class__.__name__
|
|
189
|
+
if as_dict:
|
|
190
|
+
data = {
|
|
191
|
+
field.name: getattr(obj, field.name)
|
|
192
|
+
for field in fields(obj)
|
|
193
|
+
}
|
|
194
|
+
else:
|
|
195
|
+
data = [
|
|
196
|
+
getattr(obj, field.name)
|
|
197
|
+
for field in fields(obj)
|
|
198
|
+
]
|
|
199
|
+
return as_dict, object_type, data
|
|
200
|
+
options: PackConfig
|
|
201
|
+
|
|
202
|
+
def pack(self, data):
|
|
203
|
+
return packb(data, pack_ctrl=self)
|
|
204
|
+
|
|
205
|
+
@dataclass
|
|
206
|
+
class TypedUnpackCtrl:
|
|
207
|
+
constructors: Dict[str, callable]
|
|
208
|
+
def from_dict(self, ctype, data): return self.constructors[ctype](**data)
|
|
209
|
+
def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
|
|
210
|
+
options: UnpackConfig
|
|
211
|
+
|
|
212
|
+
def unpack(self, packed):
|
|
213
|
+
return unpackb(packed, unpack_ctrl=self)
|
|
214
|
+
|
|
215
|
+
def pc(**kwargs): return TypedPackCtrl(options=PackConfig(**kwargs))
|
|
216
|
+
def uc(fns, **kwargs):
|
|
217
|
+
return TypedUnpackCtrl(
|
|
218
|
+
constructors={fn.__name__:fn for fn in fns},
|
|
219
|
+
options=UnpackConfig(**kwargs),
|
|
220
|
+
)
|
|
221
|
+
```
|
|
222
|
+
Notes:
|
|
223
|
+
* For conveninence, we added methods `pack_ctrl.pack(data)` and
|
|
224
|
+
`unpack_ctrl.unpack(packed)`.
|
|
225
|
+
* The method `pack_ctrl.from_obj` decides whether to represent the dataclass object
|
|
226
|
+
as a key-value dict or as a more compact list of values.
|
|
227
|
+
* It extracts the properties of the `obj` and sets the values `as_dict`, `object_type`
|
|
228
|
+
and `data` appropriately.
|
|
229
|
+
* In this implementation, object types are the unqualified class names. There is
|
|
230
|
+
a possibility that one class from one package can have the same name as a different
|
|
231
|
+
class from a different package.
|
|
232
|
+
* Fully resolving naming spaces is a deep design problem. You need to decide what
|
|
233
|
+
you mean by 'meaning'. Here, we exploit this overloadability...
|
|
234
|
+
* The `unpack_ctrl` object is created with a list of available constructor functions.
|
|
235
|
+
|
|
236
|
+
Now, we can define the data classes to be packed and unpacked:
|
|
237
|
+
```python
|
|
238
|
+
@dataclass
|
|
239
|
+
class Foo:
|
|
240
|
+
x: int = 1
|
|
241
|
+
y: str = 'Y'
|
|
242
|
+
|
|
243
|
+
@dataclass
|
|
244
|
+
class Bar:
|
|
245
|
+
x: int = 1
|
|
246
|
+
y: str = 'Y'
|
|
247
|
+
as_list = True
|
|
248
|
+
```
|
|
249
|
+
Notes:
|
|
250
|
+
* Class Bar has a class property `as_list=True`. It will be packed compactly as
|
|
251
|
+
a parameter value list.
|
|
252
|
+
|
|
253
|
+
Here is a simple test runner to be used for several tests:
|
|
254
|
+
```python
|
|
255
|
+
def run(input, expected=None):
|
|
256
|
+
if expected is None: expected = input
|
|
257
|
+
|
|
258
|
+
constructors = [Foo, Bar]
|
|
259
|
+
|
|
260
|
+
pack_ctrl = pc()
|
|
261
|
+
unpack_ctrl = uc(constructors)
|
|
262
|
+
|
|
263
|
+
packed = pack_ctrl.pack(input)
|
|
264
|
+
output = unpack_ctrl.unpack(packed)
|
|
265
|
+
|
|
266
|
+
assert output == expected
|
|
267
|
+
```
|
|
268
|
+
|
|
269
|
+
The `Foo()` and `Bar()` objects are packed and unpacked correctly:
|
|
270
|
+
```python
|
|
271
|
+
def test_typed_foobar():
|
|
272
|
+
run(Foo()) # Encoded as a typed dict
|
|
273
|
+
run(Bar()) # Encoded as a typed list
|
|
274
|
+
run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
|
|
275
|
+
run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
|
|
276
|
+
run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
|
|
277
|
+
```
|
|
278
|
+
|
|
@@ -11,7 +11,7 @@ class TestPackCtrl:
|
|
|
11
11
|
class TestUnpackCtrl:
|
|
12
12
|
def from_dict(self, ctype, dct):
|
|
13
13
|
raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
|
|
14
|
-
def
|
|
14
|
+
def from_tuple(self, ctype, lst):
|
|
15
15
|
raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
|
|
16
16
|
options: UnpackConfig
|
|
17
17
|
|