tmsgpack 0.1.1__tar.gz → 0.1.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/COPYING +1 -1
  2. {tmsgpack-0.1.1/tmsgpack.egg-info → tmsgpack-0.1.3}/PKG-INFO +63 -213
  3. tmsgpack-0.1.3/README.md +279 -0
  4. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/setup.cfg +3 -3
  5. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/contexts_for_tests.py +1 -1
  6. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_buffer.py +6 -6
  7. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_case.py +5 -23
  8. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_except.py +18 -7
  9. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_format.py +6 -11
  10. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_limits.py +9 -9
  11. tmsgpack-0.1.3/test/test_memoryview.py +72 -0
  12. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_newspec.py +9 -11
  13. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_pack.py +8 -28
  14. tmsgpack-0.1.3/test/test_roundtrip.py +51 -0
  15. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_seq.py +1 -1
  16. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_sequnpack.py +3 -3
  17. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_trace_from_x.py +7 -24
  18. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_typed_objects.py +2 -2
  19. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_unpack.py +3 -3
  20. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/__init__.py +31 -57
  21. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/_ctmsgpack.cpp +1620 -1864
  22. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/_packer.pyx +33 -47
  23. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/_unpacker.pyx +20 -42
  24. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/buff_converter.h +1 -1
  25. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/fallback.py +59 -76
  26. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/pack.h +0 -22
  27. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/pack_template.h +4 -7
  28. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/unpack.h +21 -39
  29. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/unpack_define.h +4 -4
  30. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/unpack_template.h +13 -13
  31. {tmsgpack-0.1.1 → tmsgpack-0.1.3/tmsgpack.egg-info}/PKG-INFO +63 -213
  32. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack.egg-info/SOURCES.txt +1 -1
  33. tmsgpack-0.1.1/README.md +0 -429
  34. tmsgpack-0.1.1/test/test_memoryview.py +0 -100
  35. tmsgpack-0.1.1/tmsgpack/ext.py +0 -0
  36. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/MANIFEST.in +0 -0
  37. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/pyproject.toml +0 -0
  38. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/setup.py +0 -0
  39. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/test/test_subtype.py +0 -0
  40. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/_ctmsgpack.pyx +0 -0
  41. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/exceptions.py +0 -0
  42. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack/sysdep.h +0 -0
  43. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack.egg-info/dependency_links.txt +0 -0
  44. {tmsgpack-0.1.1 → tmsgpack-0.1.3}/tmsgpack.egg-info/top_level.txt +0 -0
@@ -1,4 +1,4 @@
1
- Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com>
1
+ Copyright (C) 2025 Yaakov Belch <yaakov.belch@gmail.com>
2
2
 
3
3
  Licensed under the Apache License, Version 2.0 (the "License");
4
4
  you may not use this file except in compliance with the License.
@@ -1,13 +1,13 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: tmsgpack
3
- Version: 0.1.1
3
+ Version: 0.1.3
4
4
  Summary: Typed MessagePack serializer (similar but incompatible with msgpack)
5
- Home-page: https://github.com/Yaakov-Belch/tmsgpack
5
+ Home-page: https://github.com/Yaakov-Belch/tmsgpack-py
6
6
  Author: Inada Naoki, Yaakov Belch
7
7
  Author-email: yaakov.belch@gmail.com
8
8
  License: Apache-2.0
9
- Project-URL: Source, https://github.com/Yaakov-Belch/tmsgpack
10
- Project-URL: Tracker, https://github.com/Yaakov-Belch/tmsgpack/issues
9
+ Project-URL: Source, https://github.com/Yaakov-Belch/tmsgpack-py
10
+ Project-URL: Tracker, https://github.com/Yaakov-Belch/tmsgpack-py/issues
11
11
  Classifier: Programming Language :: Python :: 3
12
12
  Classifier: Programming Language :: Python :: 3.9
13
13
  Classifier: Programming Language :: Python :: 3.10
@@ -24,8 +24,11 @@ Dynamic: license-file
24
24
 
25
25
  # tmsgpack: Typed MessagePack-inspired pack/unpack component
26
26
 
27
- The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and lists)
28
- with an `object_type` property.
27
+ See also: [FORMAT.md](FORMAT.md), [DEVELOP.md](DEVELOP.md)
28
+ [tmsgpack-js](https://github.com/Yaakov-Belch/tmsgpack-js)
29
+
30
+ The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
31
+ tuples/lists) with an `object_type` property.
29
32
 
30
33
  Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
31
34
  solution. It is a composable component that helps you to build end-to-end communication
@@ -47,7 +50,10 @@ This python package makes a minimal (backwards-incompatible) modification to the
47
50
  msgpack format to make all this elegantly possible. This package is based on
48
51
  `msgpack v1.0.5`.
49
52
 
50
- ## TODO: Installation
53
+ ## Installation
54
+ ```bash
55
+ pip install tmsgpack
56
+ ```
51
57
 
52
58
  ## Usage
53
59
  Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
@@ -93,7 +99,7 @@ class MinimalPackCtrl:
93
99
  class MinimalUnpackCtrl:
94
100
  def from_dict(self, ctype, dct):
95
101
  raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
96
- def from_list(self, ctype, lst):
102
+ def from_tuple(self, ctype, lst):
97
103
  raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
98
104
  options: UnpackConfig
99
105
 
@@ -106,15 +112,15 @@ minimal_unpack_ctrl = uctrl()
106
112
 
107
113
  ## The API and configuration
108
114
  As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
109
- object provides the methods `from_dict` and `from_array`:
115
+ object provides the methods `from_dict` and `from_tuple`:
110
116
  ```python
111
117
  as_dict, data_type, data = pack_ctrl.from(obj)
112
118
 
113
119
  # When `as_dict` is true, then `data` should be a dictionary.
114
- # When `as_dict` is false, then `data` should be a list.
120
+ # When `as_dict` is false, then `data` should be a tuple or a list.
115
121
 
116
122
  unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
117
- unpacked = unpack_ctrl.from_list(data_type, data) # used when as_dict is false.
123
+ unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
118
124
  ```
119
125
 
120
126
  ## PackConfig configuration objects for pack_ctrl
@@ -123,35 +129,24 @@ unpacked = unpack_ctrl.from_list(data_type, data) # used when as_dict is false.
123
129
  from tmsgpack import PackConfig
124
130
 
125
131
  config = PackConfig(
126
- use_single_float=False, use_bin_type=True,
127
- tuple_as_list=True, strict_types=False,
128
- unicode_errors='strict', sort_keys=False,
132
+ p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
129
133
  )
130
134
  """
131
135
  Config object for pack_ctrl.options
132
136
 
133
- :param bool use_single_float:
134
- Use single precision float type for float. (default: False)
135
-
136
- :param bool use_bin_type:
137
- Use bin type introduced in tmsgpack spec 2.0 for bytes.
138
- It also enables str8 type for unicode. (default: True)
139
-
140
- :param bool tuple_as_list:
141
- If true, tuples are serialized as lists. (default: True)
142
- Otherwise, tuples are passed to pack_ctrl.from_obj(?).
137
+ :param bool p_shortcuts:
138
+ If true, pack dicts, tuples, lists without calling from_obj(...).
139
+ Object types: {...} => None; (...) => None; [...] => False
140
+ (default: True)
143
141
 
144
- :param bool strict_types:
145
- If set to true, types will be checked to be exact. (default: False)
146
- Derived classes are distinct and passed to pack_ctrl.from_obj(?).
147
- Dicts, lists and tuples are not affected by strict_types.
142
+ :param bool sort_keys:
143
+ Sort output dictionaries by key. (default: False)
148
144
 
149
- :param str unicode_errors:
150
- The error handler for encoding unicode. (default: 'strict')
151
- DO NOT USE THIS!! This option is kept for very specific usage.
145
+ :param bool p_str_keys:
146
+ If true, accept only strings as dict keys. (default: False)
152
147
 
153
- :param bool sort_keys:
154
- Sort output dictionaries by key. (default: False)
148
+ :param bool use_single_float:
149
+ Use single precision float type for float. (default: False)
155
150
  """
156
151
  ```
157
152
  ## UnpackConfig configuration objects for unpack_ctrl
@@ -160,55 +155,43 @@ Config object for pack_ctrl.options
160
155
  from tmsgpack import UnpackConfig
161
156
 
162
157
  config = UnpackConfig(
163
- read_size=16*1024, use_tuple=False, raw=False,
164
- strict_dict_key=False, object_as_pairs=False,
165
- unicode_errors='strict', max_buffer_size=0,
158
+ u_shortcuts=True, u_str_keys=False,
159
+ read_size=16*1024, max_buffer_size=0,
166
160
  max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
167
161
  )
168
162
  """
169
163
  Config object for unpack_ctrl.options
170
164
 
171
- :param int read_size:
172
- Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
165
+ :param bool u_shortcuts:
166
+ If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
167
+ Object types: None => {...}/(...), False => [...]
168
+ (default: True)
173
169
 
174
- :param bool use_tuple:
175
- If true, unpack a tmsgpack list as a Python tuple. (default: False)
170
+ :param bool u_str_keys:
171
+ If true, accept only strings as dict keys. (default: False)
176
172
 
177
- :param bool raw:
178
- If true, unpack tmsgpack strings (raw) to Python bytes.
179
- Otherwise, unpack to Python str by decoding with UTF-8 encoding (default: False).
173
+ :param int read_size:
174
+ Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
180
175
 
181
- :param bool strict_dict_key:
182
- If true only str or bytes are accepted for dict (dict) keys. (default: False).
176
+ :param int max_buffer_size:
177
+ (default: 100*1024*1024 (100MiB))
178
+ Limits size of data waiting unpacked. 0 means 2**32-1.
179
+ Raises `BufferFull` exception when it is insufficient.
180
+ You should set this parameter when unpacking data from untrusted source.
183
181
 
184
- :param callable object_as_pairs:
185
- If true, handles dicts as tuples of pairs.
186
- Otherwise, as dicts (default: False).
182
+ :param int max_str_len:
183
+ Limits max length of str. (default: max_buffer_size)
187
184
 
188
- :param str unicode_errors:
189
- The error handler for decoding unicode. (default: 'strict')
190
- This option should be used only when you have tmsgpack data which
191
- contains invalid UTF-8 string.
185
+ :param int max_bin_len:
186
+ Limits max length of bin. (default: max_buffer_size)
192
187
 
193
- :param int max_buffer_size:
194
- Limits size of data waiting unpacked. 0 means 2**32-1.
195
- The default value is 100*1024*1024 (100MiB).
196
- Raises `BufferFull` exception when it is insufficient.
197
- You should set this parameter when unpacking data from untrusted source.
188
+ :param int max_list_len:
189
+ Limits max length of list.
190
+ (default: max_buffer_size)
198
191
 
199
- :param int max_str_len:
200
- Limits max length of str. (default: max_buffer_size)
201
-
202
- :param int max_bin_len:
203
- Limits max length of bin. (default: max_buffer_size)
204
-
205
- :param int max_list_len:
206
- Limits max length of list.
207
- (default: max_buffer_size)
208
-
209
- :param int max_dict_len:
210
- Limits max length of dict.
211
- (default: max_buffer_size//2)
192
+ :param int max_dict_len:
193
+ Limits max length of dict.
194
+ (default: max_buffer_size//2)
212
195
  """
213
196
  ```
214
197
  ## Packing and Unpacking dataclass objects
@@ -248,7 +231,7 @@ class TypedPackCtrl:
248
231
  class TypedUnpackCtrl:
249
232
  constructors: Dict[str, callable]
250
233
  def from_dict(self, ctype, data): return self.constructors[ctype](**data)
251
- def from_list(self, ctype, data): return self.constructors[ctype]( *data)
234
+ def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
252
235
  options: UnpackConfig
253
236
 
254
237
  def unpack(self, packed):
@@ -287,33 +270,20 @@ class Bar:
287
270
  x: int = 1
288
271
  y: str = 'Y'
289
272
  as_list = True
290
-
291
- @dataclass
292
- class Add:
293
- x: int = 10
294
- y: int = 20
295
-
296
- class Expr:
297
- @staticmethod
298
- def Add(x:int, y:int): return x+y
299
- @staticmethod
300
- def tuple(*args): return args
301
273
  ```
302
274
  Notes:
303
275
  * Class Bar has a class property `as_list=True`. It will be packed compactly as
304
276
  a parameter value list.
305
- * The function (static method) `Expr.Add` has the same name as the class constructor
306
- `Add`. We will soon exploit this...
307
277
 
308
278
  Here is a simple test runner to be used for several tests:
309
279
  ```python
310
280
  def run(input, expected=None):
311
281
  if expected is None: expected = input
312
282
 
313
- constructors = [Foo, Bar, Expr.tuple, Expr.Add]
283
+ constructors = [Foo, Bar]
314
284
 
315
- pack_ctrl = pc(tuple_as_list=False) # We want to distinguish between tuples and lists.
316
- unpack_ctrl = uc(constructors)
285
+ pack_ctrl = pc()
286
+ unpack_ctrl = uc(constructors)
317
287
 
318
288
  packed = pack_ctrl.pack(input)
319
289
  output = unpack_ctrl.unpack(packed)
@@ -321,133 +291,13 @@ def run(input, expected=None):
321
291
  assert output == expected
322
292
  ```
323
293
 
324
- And here are the first tests. The `Foo()` and `Bar()` objects are packed and unpacked
325
- correctly:
294
+ The `Foo()` and `Bar()` objects are packed and unpacked correctly:
326
295
  ```python
327
296
  def test_typed_foobar():
328
- run(Foo()) # Encoded as a typed dict
329
- run(Bar()) # Encoded as a typed list
330
- run((1,2,3)) # Tuples are encoded as a typed list with object_type='tuple'
331
- ```
332
-
333
- The second test shows that we can encode an object tree into an
334
- binary expression buffer. When unpacking this expression buffer, the expression
335
- is evaluated.
336
- ```python
337
- def test_simple_expression():
338
- run(Add(Add(1,2), Add(2,3)), 8) # Unpacking is expression evaluation.
339
- ```
340
- Variables (parameters) can be implemented using a dictionary inside the `unpack_ctrl`
341
- object.
342
-
343
- ## Development: Environment and testing
344
-
345
- ```
346
- # Clone the repository
347
- cd ~/git/
348
- # git clone https://github.com/Yaakov-Belch/tmsgpack.git # You need permissions for that.
349
- git clone https://github.com/Yaakov-Belch/tmsgpack.git
350
-
351
- # Create a virtual environment in your project directory
352
- cd ~/git/tmsgpack
353
- python -m venv venv
354
-
355
- # Activate the virtual environment
356
- source venv/bin/activate
357
-
358
- # Upgrade pip (recommended)
359
- pip install --upgrade pip
360
-
361
- # Install the required dependencies
362
- pip install -r requirements.txt
363
-
364
- # Now you can run the tests
365
- make test
366
-
367
- # Run one test
368
- pytest -v test/test_typed_objects.py
369
- ```
370
-
371
- ## Build, test, and publish to PyPI
372
- Follow the steps above in "Development": Clone this repository, create and activate
373
- a virtual environment, upgrade pip, install required dependencies; make test.
374
- ```bash
375
- cd ~/git/tmsgpack
376
- source venv/bin/activate
377
-
378
- pip install build
379
- rm -rf dist/ # remove old builds
380
- python -m build # puts results in dist/
381
-
382
- # Test the new package -- in a fresh test environment:
383
-
384
- # Create a fresh test environment
385
- deactivate
386
- python -m venv venv-test
387
- source venv-test/bin/activate
388
- pip install --upgrade pip
389
- pip install dist/tmsgpack-*.whl # use the latest whl built.
390
-
391
- pip install pytest
392
-
393
- # These tests uses the source files:
394
- pytest -v test/
395
- TMSGPACK_PUREPYTHON=1 pytest -v test
396
- python -c "import tmsgpack; print(tmsgpack.__file__)"
397
-
398
- # These tests use the installed files:
399
- cd test
400
- pytest -v .
401
- TMSGPACK_PUREPYTHON=1 pytest -v .
402
- python -c "import tmsgpack; print(tmsgpack.__file__)"
403
- cd ..
404
- deactivate
405
- rm -rf venv-test
406
- ```
407
- Upload to PyPi:
408
- ```bash
409
- pip install twine
410
-
411
- # Register at pypi.org if you haven't
412
- # Connecting to upload.pypi.org
413
- # twine upload dist/* # This does not work: Platform incompatible.
414
- twine upload dist/tmsgpack-*.tar.gz
297
+ run(Foo()) # Encoded as a typed dict
298
+ run(Bar()) # Encoded as a typed list
299
+ run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
300
+ run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
301
+ run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
415
302
  ```
416
303
 
417
- Test in a separate virtual environment
418
- ```bash
419
- deactivate
420
- rm -rf ~/test-tmsgpack
421
- mkdir -p ~/test-tmsgpack
422
- cd ~/test-tmsgpack
423
- python -m venv venv-test
424
- source venv-test/bin/activate
425
- pip install --upgrade pip
426
- pip install tmsgpack pytest
427
- python -c "import tmsgpack; print('Version:', tmsgpack.__version__)"
428
- pytest -v ~/git/tmsgpack/test/
429
- TMSGPACK_PUREPYTHON=1 pytest -v ~/git/tmsgpack/test/
430
- ```
431
-
432
- ## The tmsgpack format (version 0.1.0)
433
- The msgpack format defines two types of containers: maps and arrays (dicts and lists).
434
- They are encoded by a `container_header` that identifies the container type and the
435
- number of key-value pairs or array-elements that follow after the `container_header`:
436
- ```
437
- dict_container_header(3) key1 value1 key2 value2 key3 value3
438
- array_container_header(3) element1 element2 element3
439
- ```
440
- The tmsgpack format uses the same rules -- but adds an `object_type` entry right
441
- after every `container_header`:
442
- ```
443
- dict_container_header(3) object_type key1 value1 key2 value2 key3 value3
444
- array_container_header(3) object_type element1 element2 element3
445
- ```
446
-
447
- ## Future extensibility
448
- In msgpack, possible `ExtType` values for the first `data_header` byte declare
449
- this data element as an msgpack extension. The tmsgpack format does not use this
450
- extension mechanism -- and these eight values are available for future extensions.
451
-
452
- The value `0xC1` is never used by the original msgpack specification. It is also
453
- available for future extensions of the tmsgpack format.
@@ -0,0 +1,279 @@
1
+ # tmsgpack: Typed MessagePack-inspired pack/unpack component
2
+
3
+ See also: [FORMAT.md](FORMAT.md), [DEVELOP.md](DEVELOP.md)
4
+ [tmsgpack-js](https://github.com/Yaakov-Belch/tmsgpack-js)
5
+
6
+ The tmsgpack format expresses **typed objects**: maps and arrays (or: dicts and
7
+ tuples/lists) with an `object_type` property.
8
+
9
+ Unlike msgpack and pickle, this is not a batteries-included end-to-end serialization
10
+ solution. It is a composable component that helps you to build end-to-end communication
11
+ solutions.
12
+
13
+ Your system solution design will make decisions on:
14
+ * What objects are serializable and what objects are not.
15
+ * What code to use (and, maybe, dynamically load) to instantiate serialized objects.
16
+ * How to represent objects that are unpacked but not supposed to 'live' in this process.
17
+ * How to share dynamic data between different packs/unpacks.
18
+ * How to asynchronously load and integrate shared data from different sources.
19
+ * How to map typed object meaning between different programming languages.
20
+ * Whether and how to convert persisted "old" data to current, new semantics (schemas).
21
+ * How much to attach explicit meaning and predictable schemas to your object types.
22
+ * Whether or not to use the 'expression execution' capabilities of tmsgpack.
23
+ * etc.
24
+
25
+ This python package makes a minimal (backwards-incompatible) modification to the
26
+ msgpack format to make all this elegantly possible. This package is based on
27
+ `msgpack v1.0.5`.
28
+
29
+ ## Installation
30
+ ```bash
31
+ pip install tmsgpack
32
+ ```
33
+
34
+ ## Usage
35
+ Packing and unpacking data is controlled by `pack_ctrl` and `unpack_ctrl` objects (see
36
+ below for details how to create them):
37
+ ```python
38
+ from tmsgpack import packb, unpackb
39
+ packed = packb(data, pack_ctrl=pack_ctrl)
40
+ unpacked = unpackb(packed, unpack_ctrl=unpack_ctrl)
41
+ ```
42
+
43
+ ## Streaming unpacking
44
+ For multiple uses, you can use packer and unpacker objects:
45
+ ```python
46
+ from tmsgpack import Packer
47
+ packer = Packer(pack_ctrl=pack_ctrl)
48
+
49
+ packed = packer.pack(data) # Send these packages via a socket...
50
+
51
+ ---
52
+ from tmsgpack import Packer, Unpacker
53
+
54
+ unpacker = Unpacker(unpack_ctrl=unpack_ctrl)
55
+ while buf := sock.recv(1024**2):
56
+ unpacker.feed(buf)
57
+ for o in unpacker:
58
+ process(o)
59
+ ```
60
+
61
+ ## Minimal pack_ctrl and unpack_ctrl objects
62
+ Minimal controllers allow only JSON-like objects and raise errors when you ask for more
63
+ (below, we show examples for more useful controllers):
64
+ ```python
65
+ from tmsgpack import PackConfig, UnpackConfig
66
+ from dataclasses import dataclass
67
+
68
+ @dataclass
69
+ class MinimalPackCtrl:
70
+ def from_obj(self, obj):
71
+ raise TypeError(f'Cannot serialize {type(obj)} object.')
72
+ options: PackConfig
73
+
74
+ @dataclass
75
+ class MinimalUnpackCtrl:
76
+ def from_dict(self, ctype, dct):
77
+ raise ValueError(f'Unpack type not supported: {ctype} data: {dct}')
78
+ def from_tuple(self, ctype, lst):
79
+ raise ValueError(f'Unpack type not supported: {ctype} data: {lst}')
80
+ options: UnpackConfig
81
+
82
+ def pctrl(**kwargs): return MinimalPackCtrl(options=PackConfig(**kwargs))
83
+ def uctrl(**kwargs): return MinimalUnpackCtrl(options=UnpackConfig(**kwargs))
84
+
85
+ minimal_pack_ctrl = pctrl()
86
+ minimal_unpack_ctrl = uctrl()
87
+ ```
88
+
89
+ ## The API and configuration
90
+ As you see, the `pack_ctrl` object provides a method `from_obj`. The `unpack_ctrl`
91
+ object provides the methods `from_dict` and `from_tuple`:
92
+ ```python
93
+ as_dict, data_type, data = pack_ctrl.from(obj)
94
+
95
+ # When `as_dict` is true, then `data` should be a dictionary.
96
+ # When `as_dict` is false, then `data` should be a tuple or a list.
97
+
98
+ unpacked = unpack_ctrl.from_dict(data_type, data) # used when as_dict is true.
99
+ unpacked = unpack_ctrl.from_tuple(data_type, data) # used when as_dict is false.
100
+ ```
101
+
102
+ ## PackConfig configuration objects for pack_ctrl
103
+ `PackConfig` objects provide the following options:
104
+ ```python
105
+ from tmsgpack import PackConfig
106
+
107
+ config = PackConfig(
108
+ p_shortcuts=True, sort_keys=False, p_str_keys=False, use_single_float=False,
109
+ )
110
+ """
111
+ Config object for pack_ctrl.options
112
+
113
+ :param bool p_shortcuts:
114
+ If true, pack dicts, tuples, lists without calling from_obj(...).
115
+ Object types: {...} => None; (...) => None; [...] => False
116
+ (default: True)
117
+
118
+ :param bool sort_keys:
119
+ Sort output dictionaries by key. (default: False)
120
+
121
+ :param bool p_str_keys:
122
+ If true, accept only strings as dict keys. (default: False)
123
+
124
+ :param bool use_single_float:
125
+ Use single precision float type for float. (default: False)
126
+ """
127
+ ```
128
+ ## UnpackConfig configuration objects for unpack_ctrl
129
+ `UnpackConfig` objects provide the following options:
130
+ ```python
131
+ from tmsgpack import UnpackConfig
132
+
133
+ config = UnpackConfig(
134
+ u_shortcuts=True, u_str_keys=False,
135
+ read_size=16*1024, max_buffer_size=0,
136
+ max_str_len=-1, max_bin_len=-1, max_list_len=-1, max_dict_len=-1,
137
+ )
138
+ """
139
+ Config object for unpack_ctrl.options
140
+
141
+ :param bool u_shortcuts:
142
+ If true, unpack dicts, tuples, lists without calling from_dict/from_tuple.
143
+ Object types: None => {...}/(...), False => [...]
144
+ (default: True)
145
+
146
+ :param bool u_str_keys:
147
+ If true, accept only strings as dict keys. (default: False)
148
+
149
+ :param int read_size:
150
+ Used as `file_like.read(read_size)`. (default: `min(16*1024, max_buffer_size)`)
151
+
152
+ :param int max_buffer_size:
153
+ (default: 100*1024*1024 (100MiB))
154
+ Limits size of data waiting unpacked. 0 means 2**32-1.
155
+ Raises `BufferFull` exception when it is insufficient.
156
+ You should set this parameter when unpacking data from untrusted source.
157
+
158
+ :param int max_str_len:
159
+ Limits max length of str. (default: max_buffer_size)
160
+
161
+ :param int max_bin_len:
162
+ Limits max length of bin. (default: max_buffer_size)
163
+
164
+ :param int max_list_len:
165
+ Limits max length of list.
166
+ (default: max_buffer_size)
167
+
168
+ :param int max_dict_len:
169
+ Limits max length of dict.
170
+ (default: max_buffer_size//2)
171
+ """
172
+ ```
173
+ ## Packing and Unpacking dataclass objects
174
+ Here are the parts of one unit test that shows end-to-end packing and unpacking
175
+ of dataclass objects:
176
+
177
+ For the setup, we import tools and define the controllers:
178
+ ```python
179
+ from tmsgpack import packb, unpackb, PackConfig, UnpackConfig
180
+ from dataclasses import dataclass, is_dataclass, fields
181
+ from typing import Dict
182
+
183
+ @dataclass
184
+ class TypedPackCtrl:
185
+ def from_obj(self, obj):
186
+ if type(obj) is tuple: return False, 'tuple', obj # Special case for tuples.
187
+ if not is_dataclass(obj): raise TypeError(f'Cannot serialize {type(obj)} object.')
188
+ as_dict = not getattr(obj, 'as_list', False)
189
+ object_type = obj.__class__.__name__
190
+ if as_dict:
191
+ data = {
192
+ field.name: getattr(obj, field.name)
193
+ for field in fields(obj)
194
+ }
195
+ else:
196
+ data = [
197
+ getattr(obj, field.name)
198
+ for field in fields(obj)
199
+ ]
200
+ return as_dict, object_type, data
201
+ options: PackConfig
202
+
203
+ def pack(self, data):
204
+ return packb(data, pack_ctrl=self)
205
+
206
+ @dataclass
207
+ class TypedUnpackCtrl:
208
+ constructors: Dict[str, callable]
209
+ def from_dict(self, ctype, data): return self.constructors[ctype](**data)
210
+ def from_tuple(self, ctype, data): return self.constructors[ctype]( *data)
211
+ options: UnpackConfig
212
+
213
+ def unpack(self, packed):
214
+ return unpackb(packed, unpack_ctrl=self)
215
+
216
+ def pc(**kwargs): return TypedPackCtrl(options=PackConfig(**kwargs))
217
+ def uc(fns, **kwargs):
218
+ return TypedUnpackCtrl(
219
+ constructors={fn.__name__:fn for fn in fns},
220
+ options=UnpackConfig(**kwargs),
221
+ )
222
+ ```
223
+ Notes:
224
+ * For conveninence, we added methods `pack_ctrl.pack(data)` and
225
+ `unpack_ctrl.unpack(packed)`.
226
+ * The method `pack_ctrl.from_obj` decides whether to represent the dataclass object
227
+ as a key-value dict or as a more compact list of values.
228
+ * It extracts the properties of the `obj` and sets the values `as_dict`, `object_type`
229
+ and `data` appropriately.
230
+ * In this implementation, object types are the unqualified class names. There is
231
+ a possibility that one class from one package can have the same name as a different
232
+ class from a different package.
233
+ * Fully resolving naming spaces is a deep design problem. You need to decide what
234
+ you mean by 'meaning'. Here, we exploit this overloadability...
235
+ * The `unpack_ctrl` object is created with a list of available constructor functions.
236
+
237
+ Now, we can define the data classes to be packed and unpacked:
238
+ ```python
239
+ @dataclass
240
+ class Foo:
241
+ x: int = 1
242
+ y: str = 'Y'
243
+
244
+ @dataclass
245
+ class Bar:
246
+ x: int = 1
247
+ y: str = 'Y'
248
+ as_list = True
249
+ ```
250
+ Notes:
251
+ * Class Bar has a class property `as_list=True`. It will be packed compactly as
252
+ a parameter value list.
253
+
254
+ Here is a simple test runner to be used for several tests:
255
+ ```python
256
+ def run(input, expected=None):
257
+ if expected is None: expected = input
258
+
259
+ constructors = [Foo, Bar]
260
+
261
+ pack_ctrl = pc()
262
+ unpack_ctrl = uc(constructors)
263
+
264
+ packed = pack_ctrl.pack(input)
265
+ output = unpack_ctrl.unpack(packed)
266
+
267
+ assert output == expected
268
+ ```
269
+
270
+ The `Foo()` and `Bar()` objects are packed and unpacked correctly:
271
+ ```python
272
+ def test_typed_foobar():
273
+ run(Foo()) # Encoded as a typed dict
274
+ run(Bar()) # Encoded as a typed list
275
+ run((1,2,3)) # Tuples are encoded as a typed list with object_type=None.
276
+ run([1,2,3]) # Tuples are encoded as a typed list with object_type=False.
277
+ run({'hello':'world'}) # Dicts are encoded as a typed dict with object_type=None.
278
+ ```
279
+
@@ -7,11 +7,11 @@ author_email = yaakov.belch@gmail.com
7
7
  description = Typed MessagePack serializer (similar but incompatible with msgpack)
8
8
  long_description = file: README.md
9
9
  long_description_content_type = text/markdown
10
- url = https://github.com/Yaakov-Belch/tmsgpack
10
+ url = https://github.com/Yaakov-Belch/tmsgpack-py
11
11
  python_requires = >=3.9
12
12
  project_urls =
13
- Source = https://github.com/Yaakov-Belch/tmsgpack
14
- Tracker = https://github.com/Yaakov-Belch/tmsgpack/issues
13
+ Source = https://github.com/Yaakov-Belch/tmsgpack-py
14
+ Tracker = https://github.com/Yaakov-Belch/tmsgpack-py/issues
15
15
  classifiers =
16
16
  Programming Language :: Python :: 3
17
17
  Programming Language :: Python :: 3.9