pamoja-codec 0.1.17__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Anthony Wiedman
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: pamoja-codec
|
|
3
|
+
Version: 0.1.17
|
|
4
|
+
Summary: CBOR, JSON, and raw codecs behind one trait, delta and varint batch packing, and an f32 quantizer for metered links.
|
|
5
|
+
Project-URL: Repository, https://github.com/molexxxx/pamoja
|
|
6
|
+
Project-URL: Documentation, https://pamoja.molex.cloud/docs/guides/codec.html
|
|
7
|
+
Author: molexxxx
|
|
8
|
+
License: MIT
|
|
9
|
+
License-File: LICENSE-MIT
|
|
10
|
+
Keywords: codec,iot,pamoja,robotics
|
|
11
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Typing :: Typed
|
|
15
|
+
Requires-Python: >=3.10
|
|
16
|
+
Requires-Dist: pamoja-native==0.1.17
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# pamoja-codec
|
|
20
|
+
|
|
21
|
+
CBOR, JSON, and raw codecs behind one trait, delta and varint batch packing, and an f32 quantizer for metered links. One capability of [pamoja](https://github.com/molexxxx/pamoja), one memory-safe Rust core with bindings for TypeScript, Python, and C#.
|
|
22
|
+
|
|
23
|
+
[](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html)
|
|
24
|
+
[](https://pamoja.molex.cloud/docs/guides/codec.html)
|
|
25
|
+
[](https://pamoja.molex.cloud/docs/)
|
|
26
|
+
|
|
27
|
+
## Install
|
|
28
|
+
|
|
29
|
+
```sh
|
|
30
|
+
pip install pamoja-codec
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
```python
|
|
34
|
+
from pamoja import codec
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
This pulls in `pamoja-native`, the compiled engine. `pip install pamoja` is the whole framework in one package.
|
|
38
|
+
|
|
39
|
+
## Example
|
|
40
|
+
|
|
41
|
+
The script the test suite runs, spliced here as it ran.
|
|
42
|
+
|
|
43
|
+
From [`bindings/python/guides/codec.py`](https://github.com/molexxxx/pamoja/blob/main/bindings/python/guides/codec.py):
|
|
44
|
+
|
|
45
|
+
```python
|
|
46
|
+
import json
|
|
47
|
+
|
|
48
|
+
from pamoja.codec import Quantizer, from_cbor, pack_samples, to_cbor, unpack_samples
|
|
49
|
+
|
|
50
|
+
# The same reading as JSON and as CBOR. Nothing is lost, and 21.5 rides as a
|
|
51
|
+
# half-precision float, the shortest form RFC 8949 allows for it.
|
|
52
|
+
reading = {"c": 21.5, "ok": True}
|
|
53
|
+
as_json = json.dumps(reading, separators=(",", ":")).encode()
|
|
54
|
+
cbor = to_cbor(reading)
|
|
55
|
+
print(f"json {len(as_json)} bytes")
|
|
56
|
+
print(f"cbor {len(cbor)} bytes")
|
|
57
|
+
|
|
58
|
+
# A gateway that speaks JSON gets it back unchanged, so the compact form is a transport
|
|
59
|
+
# choice rather than a different data model.
|
|
60
|
+
restored = from_cbor(cbor)
|
|
61
|
+
print(f"back to json, unchanged: {restored == reading}")
|
|
62
|
+
|
|
63
|
+
# A batch of readings packs to a count, then the difference between each sample and the
|
|
64
|
+
# one before it. Successive readings differ by very little, so the differences cost about
|
|
65
|
+
# a byte each where the samples would cost eight.
|
|
66
|
+
samples = [10, 11, 13, 12, 900]
|
|
67
|
+
packed = pack_samples(samples)
|
|
68
|
+
print(f"batch {len(samples)} samples in {len(packed)} bytes")
|
|
69
|
+
print(f"unpacked {unpack_samples(packed)}")
|
|
70
|
+
|
|
71
|
+
# Readings that arrive as floats pack the same way once a scale is chosen. Nothing in the
|
|
72
|
+
# bytes records that scale, so the sender and the receiver have to agree on it.
|
|
73
|
+
quantizer = Quantizer(100)
|
|
74
|
+
celsius = [20.0, 20.1, 20.2, 20.3]
|
|
75
|
+
packed_celsius = quantizer.encode(celsius)
|
|
76
|
+
recovered = quantizer.decode(packed_celsius)
|
|
77
|
+
print(f"degrees {len(celsius)} readings in {len(packed_celsius)} bytes")
|
|
78
|
+
print(f"recovered {[round(value, 1) for value in recovered]}")
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
## The same capability in every language
|
|
82
|
+
|
|
83
|
+
| Language | Package | Reference |
|
|
84
|
+
| --- | --- | --- |
|
|
85
|
+
| Rust | [`pamoja-codec`](https://crates.io/crates/pamoja-codec) | [reference](https://pamoja.molex.cloud/docs/reference/rust/pamoja_codec/index.html), [docs.rs](https://docs.rs/pamoja-codec), [install](https://pamoja.molex.cloud/docs/reference/rust.html#rust-codec) |
|
|
86
|
+
| TypeScript | [`@pamoja/codec`](https://www.npmjs.com/package/@pamoja/codec) | [reference](https://pamoja.molex.cloud/docs/reference/node/modules/_pamoja_codec.html), [install](https://pamoja.molex.cloud/docs/reference/node.html#node-codec) |
|
|
87
|
+
| Python | [`pamoja-codec`](https://pypi.org/project/pamoja-codec/) | [reference](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html), [install](https://pamoja.molex.cloud/docs/reference/python.html#python-codec) |
|
|
88
|
+
| C# | [`Pamoja.Codec`](https://www.nuget.org/packages/Pamoja.Codec) | [reference](https://pamoja.molex.cloud/docs/reference/dotnet/api/Pamoja.Codec.html), [install](https://pamoja.molex.cloud/docs/reference/dotnet.html#dotnet-codec) |
|
|
89
|
+
|
|
90
|
+
## Documentation
|
|
91
|
+
|
|
92
|
+
- [`pamoja.codec` reference](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html), every class and function in this module.
|
|
93
|
+
- [The Codecs guide](https://pamoja.molex.cloud/docs/guides/codec.html), with the same example in Rust, TypeScript, and C#.
|
|
94
|
+
- [Every capability](https://pamoja.molex.cloud/docs/), and the [install page](https://pamoja.molex.cloud/docs/install.html).
|
|
95
|
+
|
|
96
|
+
## License
|
|
97
|
+
|
|
98
|
+
MIT
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
# pamoja-codec
|
|
2
|
+
|
|
3
|
+
CBOR, JSON, and raw codecs behind one trait, delta and varint batch packing, and an f32 quantizer for metered links. One capability of [pamoja](https://github.com/molexxxx/pamoja), one memory-safe Rust core with bindings for TypeScript, Python, and C#.
|
|
4
|
+
|
|
5
|
+
[](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html)
|
|
6
|
+
[](https://pamoja.molex.cloud/docs/guides/codec.html)
|
|
7
|
+
[](https://pamoja.molex.cloud/docs/)
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```sh
|
|
12
|
+
pip install pamoja-codec
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
```python
|
|
16
|
+
from pamoja import codec
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
This pulls in `pamoja-native`, the compiled engine. `pip install pamoja` is the whole framework in one package.
|
|
20
|
+
|
|
21
|
+
## Example
|
|
22
|
+
|
|
23
|
+
The script the test suite runs, spliced here as it ran.
|
|
24
|
+
|
|
25
|
+
From [`bindings/python/guides/codec.py`](https://github.com/molexxxx/pamoja/blob/main/bindings/python/guides/codec.py):
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
import json
|
|
29
|
+
|
|
30
|
+
from pamoja.codec import Quantizer, from_cbor, pack_samples, to_cbor, unpack_samples
|
|
31
|
+
|
|
32
|
+
# The same reading as JSON and as CBOR. Nothing is lost, and 21.5 rides as a
|
|
33
|
+
# half-precision float, the shortest form RFC 8949 allows for it.
|
|
34
|
+
reading = {"c": 21.5, "ok": True}
|
|
35
|
+
as_json = json.dumps(reading, separators=(",", ":")).encode()
|
|
36
|
+
cbor = to_cbor(reading)
|
|
37
|
+
print(f"json {len(as_json)} bytes")
|
|
38
|
+
print(f"cbor {len(cbor)} bytes")
|
|
39
|
+
|
|
40
|
+
# A gateway that speaks JSON gets it back unchanged, so the compact form is a transport
|
|
41
|
+
# choice rather than a different data model.
|
|
42
|
+
restored = from_cbor(cbor)
|
|
43
|
+
print(f"back to json, unchanged: {restored == reading}")
|
|
44
|
+
|
|
45
|
+
# A batch of readings packs to a count, then the difference between each sample and the
|
|
46
|
+
# one before it. Successive readings differ by very little, so the differences cost about
|
|
47
|
+
# a byte each where the samples would cost eight.
|
|
48
|
+
samples = [10, 11, 13, 12, 900]
|
|
49
|
+
packed = pack_samples(samples)
|
|
50
|
+
print(f"batch {len(samples)} samples in {len(packed)} bytes")
|
|
51
|
+
print(f"unpacked {unpack_samples(packed)}")
|
|
52
|
+
|
|
53
|
+
# Readings that arrive as floats pack the same way once a scale is chosen. Nothing in the
|
|
54
|
+
# bytes records that scale, so the sender and the receiver have to agree on it.
|
|
55
|
+
quantizer = Quantizer(100)
|
|
56
|
+
celsius = [20.0, 20.1, 20.2, 20.3]
|
|
57
|
+
packed_celsius = quantizer.encode(celsius)
|
|
58
|
+
recovered = quantizer.decode(packed_celsius)
|
|
59
|
+
print(f"degrees {len(celsius)} readings in {len(packed_celsius)} bytes")
|
|
60
|
+
print(f"recovered {[round(value, 1) for value in recovered]}")
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
## The same capability in every language
|
|
64
|
+
|
|
65
|
+
| Language | Package | Reference |
|
|
66
|
+
| --- | --- | --- |
|
|
67
|
+
| Rust | [`pamoja-codec`](https://crates.io/crates/pamoja-codec) | [reference](https://pamoja.molex.cloud/docs/reference/rust/pamoja_codec/index.html), [docs.rs](https://docs.rs/pamoja-codec), [install](https://pamoja.molex.cloud/docs/reference/rust.html#rust-codec) |
|
|
68
|
+
| TypeScript | [`@pamoja/codec`](https://www.npmjs.com/package/@pamoja/codec) | [reference](https://pamoja.molex.cloud/docs/reference/node/modules/_pamoja_codec.html), [install](https://pamoja.molex.cloud/docs/reference/node.html#node-codec) |
|
|
69
|
+
| Python | [`pamoja-codec`](https://pypi.org/project/pamoja-codec/) | [reference](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html), [install](https://pamoja.molex.cloud/docs/reference/python.html#python-codec) |
|
|
70
|
+
| C# | [`Pamoja.Codec`](https://www.nuget.org/packages/Pamoja.Codec) | [reference](https://pamoja.molex.cloud/docs/reference/dotnet/api/Pamoja.Codec.html), [install](https://pamoja.molex.cloud/docs/reference/dotnet.html#dotnet-codec) |
|
|
71
|
+
|
|
72
|
+
## Documentation
|
|
73
|
+
|
|
74
|
+
- [`pamoja.codec` reference](https://pamoja.molex.cloud/docs/reference/python/pamoja/codec.html), every class and function in this module.
|
|
75
|
+
- [The Codecs guide](https://pamoja.molex.cloud/docs/guides/codec.html), with the same example in Rust, TypeScript, and C#.
|
|
76
|
+
- [Every capability](https://pamoja.molex.cloud/docs/), and the [install page](https://pamoja.molex.cloud/docs/install.html).
|
|
77
|
+
|
|
78
|
+
## License
|
|
79
|
+
|
|
80
|
+
MIT
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Idiomatic codec facade.
|
|
2
|
+
|
|
3
|
+
Python already has :mod:`json`, so this facade takes and returns ordinary values
|
|
4
|
+
and does the encoding itself, leaving callers to think in documents rather than
|
|
5
|
+
buffers. The conversion and the packing happen in the Rust core.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
from typing import Any, Sequence
|
|
12
|
+
|
|
13
|
+
from pamoja._native import Quantizer as _NativeQuantizer
|
|
14
|
+
from pamoja._native import cbor_to_json_bytes as _cbor_to_json_bytes
|
|
15
|
+
from pamoja._native import decode_delta_samples as _decode_delta_samples
|
|
16
|
+
from pamoja._native import encode_delta_samples as _encode_delta_samples
|
|
17
|
+
from pamoja._native import json_to_cbor_bytes as _json_to_cbor_bytes
|
|
18
|
+
|
|
19
|
+
__all__ = ["Quantizer", "from_cbor", "pack_samples", "to_cbor", "unpack_samples"]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def to_cbor(value: Any) -> bytes:
|
|
23
|
+
"""Encode a value as CBOR, typically much smaller than its JSON form.
|
|
24
|
+
|
|
25
|
+
:param value: Any JSON-serializable value, or the raw bytes of a JSON
|
|
26
|
+
document.
|
|
27
|
+
:returns: The CBOR encoding.
|
|
28
|
+
:raises PamojaError: If the value cannot be encoded.
|
|
29
|
+
"""
|
|
30
|
+
if isinstance(value, (bytes, bytearray, memoryview)):
|
|
31
|
+
document = bytes(value)
|
|
32
|
+
else:
|
|
33
|
+
document = json.dumps(value).encode("utf-8")
|
|
34
|
+
return _json_to_cbor_bytes(document)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def from_cbor(cbor: bytes) -> Any:
|
|
38
|
+
"""Decode a CBOR document back into an ordinary Python value.
|
|
39
|
+
|
|
40
|
+
:param cbor: The CBOR document to decode.
|
|
41
|
+
:returns: The decoded value.
|
|
42
|
+
:raises PamojaError: If the document is malformed, or holds a construct with
|
|
43
|
+
no JSON equivalent such as a non-string map key.
|
|
44
|
+
"""
|
|
45
|
+
return json.loads(_cbor_to_json_bytes(bytes(cbor)).decode("utf-8"))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def pack_samples(samples: Sequence[int]) -> bytes:
|
|
49
|
+
"""Delta-encode a series of integer samples into a compact buffer.
|
|
50
|
+
|
|
51
|
+
:param samples: The samples, in order.
|
|
52
|
+
:returns: The packed encoding, far smaller than the samples for a
|
|
53
|
+
slow-moving series.
|
|
54
|
+
"""
|
|
55
|
+
return _encode_delta_samples(list(samples))
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def unpack_samples(data: bytes) -> list[int]:
|
|
59
|
+
"""Unpack a buffer produced by :func:`pack_samples`.
|
|
60
|
+
|
|
61
|
+
:param data: The packed encoding.
|
|
62
|
+
:returns: The samples, in order.
|
|
63
|
+
:raises PamojaError: If the buffer is malformed.
|
|
64
|
+
"""
|
|
65
|
+
return _decode_delta_samples(bytes(data))
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Quantizer:
|
|
69
|
+
"""Packs float readings to a fixed precision, for a link charging per byte.
|
|
70
|
+
|
|
71
|
+
Example::
|
|
72
|
+
|
|
73
|
+
quantizer = Quantizer(100) # keep two decimal places
|
|
74
|
+
packed = quantizer.encode([20.0, 20.1, 20.2])
|
|
75
|
+
quantizer.decode(packed) # [20.0, 20.1, 20.2], to within 0.01
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
__slots__ = ("_native",)
|
|
79
|
+
|
|
80
|
+
def __init__(self, scale: float) -> None:
|
|
81
|
+
"""Create a quantizer at the given precision.
|
|
82
|
+
|
|
83
|
+
:param scale: The multiplier applied before rounding; ``100`` keeps two
|
|
84
|
+
decimal places. Must be positive and finite.
|
|
85
|
+
:raises ValueError: If the scale is not positive and finite.
|
|
86
|
+
"""
|
|
87
|
+
self._native = _NativeQuantizer(scale)
|
|
88
|
+
|
|
89
|
+
def encode(self, readings: Sequence[float]) -> bytes:
|
|
90
|
+
"""Quantize and pack a batch of readings.
|
|
91
|
+
|
|
92
|
+
:param readings: The readings, in order.
|
|
93
|
+
:returns: The packed encoding.
|
|
94
|
+
"""
|
|
95
|
+
return self._native.encode(list(readings))
|
|
96
|
+
|
|
97
|
+
def decode(self, data: bytes) -> list[float]:
|
|
98
|
+
"""Unpack a batch, to within this quantizer's precision.
|
|
99
|
+
|
|
100
|
+
:param data: The encoding produced by :meth:`encode` at the same scale.
|
|
101
|
+
:returns: The readings, in order.
|
|
102
|
+
:raises PamojaError: If the buffer is malformed.
|
|
103
|
+
"""
|
|
104
|
+
return self._native.decode(bytes(data))
|
|
File without changes
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling>=1.27"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pamoja-codec"
|
|
7
|
+
version = "0.1.17"
|
|
8
|
+
description = "CBOR, JSON, and raw codecs behind one trait, delta and varint batch packing, and an f32 quantizer for metered links."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
license = { text = "MIT" }
|
|
11
|
+
license-files = ["LICENSE-MIT"]
|
|
12
|
+
requires-python = ">=3.10"
|
|
13
|
+
authors = [{ name = "molexxxx" }]
|
|
14
|
+
keywords = ["pamoja", "iot", "robotics", "codec"]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"License :: OSI Approved :: MIT License",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Typing :: Typed",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"pamoja-native==0.1.17",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[project.urls]
|
|
26
|
+
Repository = "https://github.com/molexxxx/pamoja"
|
|
27
|
+
Documentation = "https://pamoja.molex.cloud/docs/guides/codec.html"
|
|
28
|
+
|
|
29
|
+
[tool.hatch.build.targets.wheel]
|
|
30
|
+
packages = ["pamoja"]
|