tmsgpack 0.1.1__tar.gz → 0.1.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/COPYING +1 -1
  2. {tmsgpack-0.1.1/tmsgpack.egg-info → tmsgpack-0.1.2}/PKG-INFO +59 -210
  3. tmsgpack-0.1.2/README.md +278 -0
  4. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/contexts_for_tests.py +1 -1
  5. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_buffer.py +6 -6
  6. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_case.py +5 -23
  7. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_except.py +18 -7
  8. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_format.py +6 -11
  9. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_limits.py +9 -9
  10. tmsgpack-0.1.2/test/test_memoryview.py +72 -0
  11. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_newspec.py +9 -11
  12. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_pack.py +8 -28
  13. tmsgpack-0.1.2/test/test_roundtrip.py +51 -0
  14. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_seq.py +1 -1
  15. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_sequnpack.py +3 -3
  16. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_trace_from_x.py +7 -24
  17. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_typed_objects.py +2 -2
  18. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_unpack.py +3 -3
  19. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/__init__.py +31 -57
  20. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_ctmsgpack.cpp +1620 -1864
  21. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_packer.pyx +33 -47
  22. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_unpacker.pyx +20 -42
  23. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/buff_converter.h +1 -1
  24. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/fallback.py +59 -76
  25. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/pack.h +0 -22
  26. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/pack_template.h +4 -7
  27. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack.h +21 -39
  28. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack_define.h +4 -4
  29. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/unpack_template.h +13 -13
  30. {tmsgpack-0.1.1 → tmsgpack-0.1.2/tmsgpack.egg-info}/PKG-INFO +59 -210
  31. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/SOURCES.txt +1 -1
  32. tmsgpack-0.1.1/README.md +0 -429
  33. tmsgpack-0.1.1/test/test_memoryview.py +0 -100
  34. tmsgpack-0.1.1/tmsgpack/ext.py +0 -0
  35. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/MANIFEST.in +0 -0
  36. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/pyproject.toml +0 -0
  37. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/setup.cfg +0 -0
  38. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/setup.py +0 -0
  39. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/test/test_subtype.py +0 -0
  40. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/_ctmsgpack.pyx +0 -0
  41. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/exceptions.py +0 -0
  42. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack/sysdep.h +0 -0
  43. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/dependency_links.txt +0 -0
  44. {tmsgpack-0.1.1 → tmsgpack-0.1.2}/tmsgpack.egg-info/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
1
+ Copyright (C) 2025 Yaakov Belch <yaakov.belch@gmail.com>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tmsgpack
3
- Version: 0.1.1
3
+ Version: 0.1.2
4
4
  Summary: Typed MessagePack serializer (similar but incompatible with msgpack)
5
5
  Home-page: https://github.com/Yaakov-Belch/tmsgpack
6
6
  Author: Inada Naoki, Yaakov Belch
@@ -24,8 +24,10 @@ Dynamic: license-file
24
24
 
25
25
  # tmsgpack: Typed MessagePack-inspired pack/unpack component
26
26
 
27
- The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and lists)
28
- with an `object_type` property.
27
+ See also: [FORMAT.md](FORMAT.md) and [DEVELOP.md](DEVELOP.md)
28
+
29
+ The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
30
+ tuples/lists) with an `object_type` property.
29
31
 
30
32
  Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
31
33
  solution. It is a composable component that helps you to build end-to-end communication
@@ -47,7 +49,10 @@ This python package makes a minimal (backwards-incompatible) modification to the
47
49
  msgpack format to make all this elegantly possible. This package is based on
48
50
  `msgpack v1.0.5`.
49
51
 
50
- ## TODO: Installation
52
+ ## Installation
53
+ ```bash
54
+ pip install tmsgpack
55
+ ```
51
56
 
52
57
  ## Usage
53
58
  Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
@@ -93,7 +98,7 @@ class MinimalPackCtrl:
93
98
  class MinimalUnpackCtrl:
94
99
  def from_dict(self, ctype, dct):
95
100
  raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
96
- def from_list(self, ctype, lst):
101
+ def from_tuple(self, ctype, lst):
97
102
  raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
98
103
  options: UnpackConfig
99
104
 
@@ -106,15 +111,15 @@ minimal_unpack_ctrl = uctrl()
106
111
 
107
112
  ## The API and configuration
108
113
  As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
109
- object provides the methods `from_dict` and `from_array`:
114
+ object provides the methods `from_dict` and `from_tuple`:
110
115
  ```python
111
116
  as_dict, data_type, data = pack_ctrl.from(obj)
112
117
 
113
118
  # When `as_dict` is true, then `data` should be a dictionary.
114
- # When `as_dict` is false, then `data` should be a list.
119
+ # When `as_dict` is false, then `data` should be a tuple or a list.
115
120
 
116
121
  unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
117
- unpacked = unpack_ctrl.from_list(data_type, data) # used when as_dict is false.
122
+ unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
118
123
  ```
119
124
 
120
125
  ## PackConfig configuration objects for pack_ctrl
@@ -123,35 +128,24 @@ unpacked = unpack_ctrl.from_list(data_type, data) # used when as_dict is false.
123
128
  from tmsgpack import PackConfig
124
129
 
125
130
  config = PackConfig(
126
- use_single_float=False, use_bin_type=True,
127
- tuple_as_list=True, strict_types=False,
128
- unicode_errors='strict', sort_keys=False,
131
+ p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
129
132
  )
130
133
  """
131
134
  Config object for pack_ctrl.options
132
135
 
133
- :param bool use_single_float:
134
- Use single precision float type for float. (default: False)
135
-
136
- :param bool use_bin_type:
137
- Use bin type introduced in tmsgpack spec 2.0 for bytes.
138
- It also enables str8 type for unicode. (default: True)
139
-
140
- :param bool tuple_as_list:
141
- If true, tuples are serialized as lists. (default: True)
142
- Otherwise, tuples are passed to pack_ctrl.from_obj(?).
136
+ :param bool p_shortcuts:
137
+ If true, pack dicts, tuples, lists without calling from_obj(...).
138
+ Object types: {...} => None; (...) => None; [...] => False
139
+ (default: True)
143
140
 
144
- :param bool strict_types:
145
- If set to true, types will be checked to be exact. (default: False)
146
- Derived classes are distinct and passed to pack_ctrl.from_obj(?).
147
- Dicts, lists and tuples are not affected by strict_types.
141
+ :param bool sort_keys:
142
+ Sort output dictionaries by key. (default: False)
148
143
 
149
- :param str unicode_errors:
150
- The error handler for encoding unicode. (default: 'strict')
151
- DO NOT USE THIS!! This option is kept for very specific usage.
144
+ :param bool p_str_keys:
145
+ If true, accept only strings as dict keys. (default: False)
152
146
 
153
- :param bool sort_keys:
154
- Sort output dictionaries by key. (default: False)
147
+ :param bool use_single_float:
148
+ Use single precision float type for float. (default: False)
155
149
  """
156
150
  ```
157
151
  ## UnpackConfig configuration objects for unpack_ctrl
@@ -160,55 +154,43 @@ Config object for pack_ctrl.options
160
154
  from tmsgpack import UnpackConfig
161
155
 
162
156
  config = UnpackConfig(
163
- read_size=16*1024, use_tuple=False, raw=False,
164
- strict_dict_key=False, object_as_pairs=False,
165
- unicode_errors='strict', max_buffer_size=0,
157
+ u_shortcuts=True, u_str_keys=False,
158
+ read_size=16*1024, max_buffer_size=0,
166
159
  max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
167
160
  )
168
161
  """
169
162
  Config object for unpack_ctrl.options
170
163
 
171
- :param int read_size:
172
- Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
164
+ :param bool u_shortcuts:
165
+ If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
166
+ Object types: None => {...}/(...), False => [...]
167
+ (default: True)
173
168
 
174
- :param bool use_tuple:
175
- If true, unpack a tmsgpack list as a Python tuple. (default: False)
169
+ :param bool u_str_keys:
170
+ If true, accept only strings as dict keys. (default: False)
176
171
 
177
- :param bool raw:
178
- If true, unpack tmsgpack strings (raw) to Python bytes.
179
- Otherwise, unpack to Python str by decoding with UTF-8 encoding (default: False).
172
+ :param int read_size:
173
+ Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
180
174
 
181
- :param bool strict_dict_key:
182
- If true only str or bytes are accepted for dict (dict) keys. (default: False).
175
+ :param int max_buffer_size:
176
+ (default: 100*1024*1024 (100MiB))
177
+ Limits size of data waiting unpacked. 0 means 2**32-1.
178
+ Raises `BufferFull` exception when it is insufficient.
179
+ You should set this parameter when unpacking data from untrusted source.
183
180
 
184
- :param callable object_as_pairs:
185
- If true, handles dicts as tuples of pairs.
186
- Otherwise, as dicts (default: False).
181
+ :param int max_str_len:
182
+ Limits max length of str. (default: max_buffer_size)
187
183
 
188
- :param str unicode_errors:
189
- The error handler for decoding unicode. (default: 'strict')
190
- This option should be used only when you have tmsgpack data which
191
- contains invalid UTF-8 string.
184
+ :param int max_bin_len:
185
+ Limits max length of bin. (default: max_buffer_size)
192
186
 
193
- :param int max_buffer_size:
194
- Limits size of data waiting unpacked. 0 means 2**32-1.
195
- The default value is 100*1024*1024 (100MiB).
196
- Raises `BufferFull` exception when it is insufficient.
197
- You should set this parameter when unpacking data from untrusted source.
187
+ :param int max_list_len:
188
+ Limits max length of list.
189
+ (default: max_buffer_size)
198
190
 
199
- :param int max_str_len:
200
- Limits max length of str. (default: max_buffer_size)
201
-
202
- :param int max_bin_len:
203
- Limits max length of bin. (default: max_buffer_size)
204
-
205
- :param int max_list_len:
206
- Limits max length of list.
207
- (default: max_buffer_size)
208
-
209
- :param int max_dict_len:
210
- Limits max length of dict.
211
- (default: max_buffer_size//2)
191
+ :param int max_dict_len:
192
+ Limits max length of dict.
193
+ (default: max_buffer_size//2)
212
194
  """
213
195
  ```
214
196
  ## Packing and Unpacking dataclass objects
@@ -248,7 +230,7 @@ class TypedPackCtrl:
248
230
  class TypedUnpackCtrl:
249
231
  constructors: Dict[str, callable]
250
232
  def from_dict(self, ctype, data): return self.constructors[ctype](**data)
251
- def from_list(self, ctype, data): return self.constructors[ctype]( *data)
233
+ def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
252
234
  options: UnpackConfig
253
235
 
254
236
  def unpack(self, packed):
@@ -287,33 +269,20 @@ class Bar:
287
269
  x: int = 1
288
270
  y: str = 'Y'
289
271
  as_list = True
290
-
291
- @dataclass
292
- class Add:
293
- x: int = 10
294
- y: int = 20
295
-
296
- class Expr:
297
- @staticmethod
298
- def Add(x:int, y:int): return x+y
299
- @staticmethod
300
- def tuple(*args): return args
301
272
  ```
302
273
  Notes:
303
274
  * Class Bar has a class property `as_list=True`. It will be packed compactly as
304
275
  a parameter value list.
305
- * The function (static method) `Expr.Add` has the same name as the class constructor
306
- `Add`. We will soon exploit this...
307
276
 
308
277
  Here is a simple test runner to be used for several tests:
309
278
  ```python
310
279
  def run(input, expected=None):
311
280
  if expected is None: expected = input
312
281
 
313
- constructors = [Foo, Bar, Expr.tuple, Expr.Add]
282
+ constructors = [Foo, Bar]
314
283
 
315
- pack_ctrl = pc(tuple_as_list=False) # We want to distinguish between tuples and lists.
316
- unpack_ctrl = uc(constructors)
284
+ pack_ctrl = pc()
285
+ unpack_ctrl = uc(constructors)
317
286
 
318
287
  packed = pack_ctrl.pack(input)
319
288
  output = unpack_ctrl.unpack(packed)
@@ -321,133 +290,13 @@ def run(input, expected=None):
321
290
  assert output == expected
322
291
  ```
323
292
 
324
- And here are the first tests. The `Foo()` and `Bar()` objects are packed and unpacked
325
- correctly:
293
+ The `Foo()` and `Bar()` objects are packed and unpacked correctly:
326
294
  ```python
327
295
  def test_typed_foobar():
328
- run(Foo()) # Encoded as a typed dict
329
- run(Bar()) # Encoded as a typed list
330
- run((1,2,3)) # Tuples are encoded as a typed list with object_type='tuple'
331
- ```
332
-
333
- The second test shows that we can encode an object tree into an
334
- binary expression buffer. When unpacking this expression buffer, the expression
335
- is evaluated.
336
- ```python
337
- def test_simple_expression():
338
- run(Add(Add(1,2), Add(2,3)), 8) # Unpacking is expression evaluation.
339
- ```
340
- Variables (parameters) can be implemented using a dictionary inside the `unpack_ctrl`
341
- object.
342
-
343
- ## Development: Environment and testing
344
-
345
- ```
346
- # Clone the repository
347
- cd ~/git/
348
- # git clone https://github.com/Yaakov-Belch/tmsgpack.git # You need permissions for that.
349
- git clone https://github.com/Yaakov-Belch/tmsgpack.git
350
-
351
- # Create a virtual environment in your project directory
352
- cd ~/git/tmsgpack
353
- python -m venv venv
354
-
355
- # Activate the virtual environment
356
- source venv/bin/activate
357
-
358
- # Upgrade pip (recommended)
359
- pip install --upgrade pip
360
-
361
- # Install the required dependencies
362
- pip install -r requirements.txt
363
-
364
- # Now you can run the tests
365
- make test
366
-
367
- # Run one test
368
- pytest -v test/test_typed_objects.py
369
- ```
370
-
371
- ## Build, test, and publish to PyPI
372
- Follow the steps above in "Development": Clone this repository, create and activate
373
- a virtual environment, upgrade pip, install required dependencies; make test.
374
- ```bash
375
- cd ~/git/tmsgpack
376
- source venv/bin/activate
377
-
378
- pip install build
379
- rm -rf dist/ # remove old builds
380
- python -m build # puts results in dist/
381
-
382
- # Test the new package -- in a fresh test environment:
383
-
384
- # Create a fresh test environment
385
- deactivate
386
- python -m venv venv-test
387
- source venv-test/bin/activate
388
- pip install --upgrade pip
389
- pip install dist/tmsgpack-*.whl # use the latest whl built.
390
-
391
- pip install pytest
392
-
393
- # These tests uses the source files:
394
- pytest -v test/
395
- TMSGPACK_PUREPYTHON=1 pytest -v test
396
- python -c "import tmsgpack; print(tmsgpack.__file__)"
397
-
398
- # These tests use the installed files:
399
- cd test
400
- pytest -v .
401
- TMSGPACK_PUREPYTHON=1 pytest -v .
402
- python -c "import tmsgpack; print(tmsgpack.__file__)"
403
- cd ..
404
- deactivate
405
- rm -rf venv-test
406
- ```
407
- Upload to PyPi:
408
- ```bash
409
- pip install twine
410
-
411
- # Register at pypi.org if you haven't
412
- # Connecting to upload.pypi.org
413
- # twine upload dist/* # This does not work: Platform incompatible.
414
- twine upload dist/tmsgpack-*.tar.gz
296
+ run(Foo()) # Encoded as a typed dict
297
+ run(Bar()) # Encoded as a typed list
298
+ run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
299
+ run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
300
+ run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
415
301
  ```
416
302
 
417
- Test in a separate virtual environment
418
- ```bash
419
- deactivate
420
- rm -rf ~/test-tmsgpack
421
- mkdir -p ~/test-tmsgpack
422
- cd ~/test-tmsgpack
423
- python -m venv venv-test
424
- source venv-test/bin/activate
425
- pip install --upgrade pip
426
- pip install tmsgpack pytest
427
- python -c "import tmsgpack; print('Version:', tmsgpack.__version__)"
428
- pytest -v ~/git/tmsgpack/test/
429
- TMSGPACK_PUREPYTHON=1 pytest -v ~/git/tmsgpack/test/
430
- ```
431
-
432
- ## The tmsgpack format (version 0.1.0)
433
- The msgpack format defines two types of containers: maps and arrays (dicts and lists).
434
- They are encoded by a `container_header` that identifies the container type and the
435
- number of key-value pairs or array-elements that follow after the `container_header`:
436
- ```
437
- dict_container_header(3) key1 value1 key2 value2 key3 value3
438
- array_container_header(3) element1 element2 element3
439
- ```
440
- The tmsgpack format uses the same rules -- but adds an `object_type` entry right
441
- after every `container_header`:
442
- ```
443
- dict_container_header(3) object_type key1 value1 key2 value2 key3 value3
444
- array_container_header(3) object_type element1 element2 element3
445
- ```
446
-
447
- ## Future extensibility
448
- In msgpack, possible `ExtType` values for the first `data_header` byte declare
449
- this data element as an msgpack extension. The tmsgpack format does not use this
450
- extension mechanism -- and these eight values are available for future extensions.
451
-
452
- The value `0xC1` is never used by the original msgpack specification. It is also
453
- available for future extensions of the tmsgpack format.
@@ -0,0 +1,278 @@
1
+ # tmsgpack: Typed MessagePack-inspired pack/unpack component
2
+
3
+ See also: [FORMAT.md](FORMAT.md) and [DEVELOP.md](DEVELOP.md)
4
+
5
+ The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
6
+ tuples/lists) with an `object_type` property.
7
+
8
+ Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
9
+ solution. It is a composable component that helps you to build end-to-end communication
10
+ solutions.
11
+
12
+ Your system solution design will make decisions on:
13
+ * What objects are serializable and what objects are not.
14
+ * What code to use (and, maybe, dynamically load) to instantiate serialized objects.
15
+ * How to represent objects that are unpacked but not supposed to 'live' in this process.
16
+ * How to share dynamic data between different packs/unpacks.
17
+ * How to asynchronously load and integrate shared data from different sources.
18
+ * How to map typed object meaning between different programming languages.
19
+ * Whether and how to convert persisted "old" data to current, new semantics (schemas).
20
+ * How much to attach explicit meaning and predictable schemas to your object types.
21
+ * Whether or not to use the 'expression execution' capabilities of tmsgpack.
22
+ * etc.
23
+
24
+ This python package makes a minimal (backwards-incompatible) modification to the
25
+ msgpack format to make all this elegantly possible. This package is based on
26
+ `msgpack v1.0.5`.
27
+
28
+ ## Installation
29
+ ```bash
30
+ pip install tmsgpack
31
+ ```
32
+
33
+ ## Usage
34
+ Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
35
+ below for details how to create them):
36
+ ```python
37
+ from tmsgpack import packb, unpackb
38
+ packed = packb(data, pack_ctrl=pack_ctrl)
39
+ unpacked = unpackb(packed, unpack_ctrl=unpack_ctrl)
40
+ ```
41
+
42
+ ## Streaming unpacking
43
+ For multiple uses, you can use packer and unpacker objects:
44
+ ```python
45
+ from tmsgpack import Packer
46
+ packer = Packer(pack_ctrl=pack_ctrl)
47
+
48
+ packed = packer.pack(data) # Send these packages via a socket...
49
+
50
+ ---
51
+ from tmsgpack import Packer, Unpacker
52
+
53
+ unpacker = Unpacker(unpack_ctrl=unpack_ctrl)
54
+ while buf := sock.recv(1024**2):
55
+ unpacker.feed(buf)
56
+ for o in unpacker:
57
+ process(o)
58
+ ```
59
+
60
+ ## Minimal pack_ctrl and unpack_ctrl objects
61
+ Minimal controllers allow only JSON-like objects and raise errors when you ask for more
62
+ (below, we show examples for more useful controllers):
63
+ ```python
64
+ from tmsgpack import PackConfig, UnpackConfig
65
+ from dataclasses import dataclass
66
+
67
+ @dataclass
68
+ class MinimalPackCtrl:
69
+ def from_obj(self, obj):
70
+ raise TypeError(f'Cannot serialize {type(obj)} object.')
71
+ options: PackConfig
72
+
73
+ @dataclass
74
+ class MinimalUnpackCtrl:
75
+ def from_dict(self, ctype, dct):
76
+ raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
77
+ def from_tuple(self, ctype, lst):
78
+ raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
79
+ options: UnpackConfig
80
+
81
+ def pctrl(**kwargs): return MinimalPackCtrl(options=PackConfig(**kwargs))
82
+ def uctrl(**kwargs): return MinimalUnpackCtrl(options=UnpackConfig(**kwargs))
83
+
84
+ minimal_pack_ctrl = pctrl()
85
+ minimal_unpack_ctrl = uctrl()
86
+ ```
87
+
88
+ ## The API and configuration
89
+ As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
90
+ object provides the methods `from_dict` and `from_tuple`:
91
+ ```python
92
+ as_dict, data_type, data = pack_ctrl.from(obj)
93
+
94
+ # When `as_dict` is true, then `data` should be a dictionary.
95
+ # When `as_dict` is false, then `data` should be a tuple or a list.
96
+
97
+ unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
98
+ unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
99
+ ```
100
+
101
+ ## PackConfig configuration objects for pack_ctrl
102
+ `PackConfig` objects provide the following options:
103
+ ```python
104
+ from tmsgpack import PackConfig
105
+
106
+ config = PackConfig(
107
+ p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
108
+ )
109
+ """
110
+ Config object for pack_ctrl.options
111
+
112
+ :param bool p_shortcuts:
113
+ If true, pack dicts, tuples, lists without calling from_obj(...).
114
+ Object types: {...} => None; (...) => None; [...] => False
115
+ (default: True)
116
+
117
+ :param bool sort_keys:
118
+ Sort output dictionaries by key. (default: False)
119
+
120
+ :param bool p_str_keys:
121
+ If true, accept only strings as dict keys. (default: False)
122
+
123
+ :param bool use_single_float:
124
+ Use single precision float type for float. (default: False)
125
+ """
126
+ ```
127
+ ## UnpackConfig configuration objects for unpack_ctrl
128
+ `UnpackConfig` objects provide the following options:
129
+ ```python
130
+ from tmsgpack import UnpackConfig
131
+
132
+ config = UnpackConfig(
133
+ u_shortcuts=True, u_str_keys=False,
134
+ read_size=16*1024, max_buffer_size=0,
135
+ max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
136
+ )
137
+ """
138
+ Config object for unpack_ctrl.options
139
+
140
+ :param bool u_shortcuts:
141
+ If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
142
+ Object types: None => {...}/(...), False => [...]
143
+ (default: True)
144
+
145
+ :param bool u_str_keys:
146
+ If true, accept only strings as dict keys. (default: False)
147
+
148
+ :param int read_size:
149
+ Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
150
+
151
+ :param int max_buffer_size:
152
+ (default: 100*1024*1024 (100MiB))
153
+ Limits size of data waiting unpacked. 0 means 2**32-1.
154
+ Raises `BufferFull` exception when it is insufficient.
155
+ You should set this parameter when unpacking data from untrusted source.
156
+
157
+ :param int max_str_len:
158
+ Limits max length of str. (default: max_buffer_size)
159
+
160
+ :param int max_bin_len:
161
+ Limits max length of bin. (default: max_buffer_size)
162
+
163
+ :param int max_list_len:
164
+ Limits max length of list.
165
+ (default: max_buffer_size)
166
+
167
+ :param int max_dict_len:
168
+ Limits max length of dict.
169
+ (default: max_buffer_size//2)
170
+ """
171
+ ```
172
+ ## Packing and Unpacking dataclass objects
173
+ Here are the parts of one unit test that shows end-to-end packing and unpacking
174
+ of dataclass objects:
175
+
176
+ For the setup, we import tools and define the controllers:
177
+ ```python
178
+ from tmsgpack import packb, unpackb, PackConfig, UnpackConfig
179
+ from dataclasses import dataclass, is_dataclass, fields
180
+ from typing import Dict
181
+
182
+ @dataclass
183
+ class TypedPackCtrl:
184
+ def from_obj(self, obj):
185
+ if type(obj) is tuple: return False, 'tuple', obj # Special case for tuples.
186
+ if not is_dataclass(obj): raise TypeError(f'Cannot serialize {type(obj)} object.')
187
+ as_dict = not getattr(obj, 'as_list', False)
188
+ object_type = obj.__class__.__name__
189
+ if as_dict:
190
+ data = {
191
+ field.name: getattr(obj, field.name)
192
+ for field in fields(obj)
193
+ }
194
+ else:
195
+ data = [
196
+ getattr(obj, field.name)
197
+ for field in fields(obj)
198
+ ]
199
+ return as_dict, object_type, data
200
+ options: PackConfig
201
+
202
+ def pack(self, data):
203
+ return packb(data, pack_ctrl=self)
204
+
205
+ @dataclass
206
+ class TypedUnpackCtrl:
207
+ constructors: Dict[str, callable]
208
+ def from_dict(self, ctype, data): return self.constructors[ctype](**data)
209
+ def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
210
+ options: UnpackConfig
211
+
212
+ def unpack(self, packed):
213
+ return unpackb(packed, unpack_ctrl=self)
214
+
215
+ def pc(**kwargs): return TypedPackCtrl(options=PackConfig(**kwargs))
216
+ def uc(fns, **kwargs):
217
+ return TypedUnpackCtrl(
218
+ constructors={fn.__name__:fn for fn in fns},
219
+ options=UnpackConfig(**kwargs),
220
+ )
221
+ ```
222
+ Notes:
223
+ * For conveninence, we added methods `pack_ctrl.pack(data)` and
224
+ `unpack_ctrl.unpack(packed)`.
225
+ * The method `pack_ctrl.from_obj` decides whether to represent the dataclass object
226
+ as a key-value dict or as a more compact list of values.
227
+ * It extracts the properties of the `obj` and sets the values `as_dict`, `object_type`
228
+ and `data` appropriately.
229
+ * In this implementation, object types are the unqualified class names. There is
230
+ a possibility that one class from one package can have the same name as a different
231
+ class from a different package.
232
+ * Fully resolving naming spaces is a deep design problem. You need to decide what
233
+ you mean by 'meaning'. Here, we exploit this overloadability...
234
+ * The `unpack_ctrl` object is created with a list of available constructor functions.
235
+
236
+ Now, we can define the data classes to be packed and unpacked:
237
+ ```python
238
+ @dataclass
239
+ class Foo:
240
+ x: int = 1
241
+ y: str = 'Y'
242
+
243
+ @dataclass
244
+ class Bar:
245
+ x: int = 1
246
+ y: str = 'Y'
247
+ as_list = True
248
+ ```
249
+ Notes:
250
+ * Class Bar has a class property `as_list=True`. It will be packed compactly as
251
+ a parameter value list.
252
+
253
+ Here is a simple test runner to be used for several tests:
254
+ ```python
255
+ def run(input, expected=None):
256
+ if expected is None: expected = input
257
+
258
+ constructors = [Foo, Bar]
259
+
260
+ pack_ctrl = pc()
261
+ unpack_ctrl = uc(constructors)
262
+
263
+ packed = pack_ctrl.pack(input)
264
+ output = unpack_ctrl.unpack(packed)
265
+
266
+ assert output == expected
267
+ ```
268
+
269
+ The `Foo()` and `Bar()` objects are packed and unpacked correctly:
270
+ ```python
271
+ def test_typed_foobar():
272
+ run(Foo()) # Encoded as a typed dict
273
+ run(Bar()) # Encoded as a typed list
274
+ run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
275
+ run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
276
+ run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
277
+ ```
278
+
@@ -11,7 +11,7 @@ class TestPackCtrl:
11
11
  class TestUnpackCtrl:
12
12
  def from_dict(self, ctype, dct):
13
13
  raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
14
- def from_list(self, ctype, lst):
14
+ def from_tuple(self, ctype, lst):
15
15
  raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
16
16
  options: UnpackConfig
17
17