zs-yaml 0.8.3.dev3__tar.gz → 0.8.4.dev4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/.github/workflows/ci.yml +14 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/CHANGELOG.md +22 -2
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/PKG-INFO +10 -3
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/README.md +9 -2
- zs_yaml-0.8.4.dev4/examples/extern_compression/extern_compression.zs +23 -0
- zs_yaml-0.8.4.dev4/examples/extern_compression/payload.yaml +10 -0
- zs_yaml-0.8.4.dev4/examples/extern_compression/test_compression.py +153 -0
- zs_yaml-0.8.4.dev4/zs_yaml/_version.py +24 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml/built_in_transformations.py +69 -28
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/PKG-INFO +10 -3
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/SOURCES.txt +3 -0
- zs_yaml-0.8.3.dev3/zs_yaml/_version.py +0 -34
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/.gitignore +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/LICENSE +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/doc/zs-yaml.png +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/doc/zs-yaml.svg +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/addresses.yaml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/custom_transformations.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/hobbies.yaml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/skills.yaml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/team.zs +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/team1.yaml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/test_all_conversions.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/test_pyobj_to_yaml.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/examples/team/work_experience.yaml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/pyproject.toml +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/setup.cfg +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml/__init__.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml/convert.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml/main.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml/yaml_transformer.py +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/dependency_links.txt +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/entry_points.txt +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/requires.txt +0 -0
- {zs_yaml-0.8.3.dev3 → zs_yaml-0.8.4.dev4}/zs_yaml.egg-info/top_level.txt +0 -0
|
@@ -63,6 +63,20 @@ jobs:
|
|
|
63
63
|
echo "Conversion tests failed"
|
|
64
64
|
exit 1
|
|
65
65
|
fi
|
|
66
|
+
- name: Generate schema API for extern_compression example
|
|
67
|
+
run: |
|
|
68
|
+
cd examples/extern_compression
|
|
69
|
+
zserio extern_compression.zs -withTypeInfoCode -python zs_gen_api
|
|
70
|
+
- name: Test extern compression
|
|
71
|
+
run: |
|
|
72
|
+
cd examples/extern_compression
|
|
73
|
+
PYTHONPATH="$PWD/zs_gen_api" python test_compression.py
|
|
74
|
+
if [ $? -eq 0 ]; then
|
|
75
|
+
echo "Compression tests successful"
|
|
76
|
+
else
|
|
77
|
+
echo "Compression tests failed"
|
|
78
|
+
exit 1
|
|
79
|
+
fi
|
|
66
80
|
- name: Build package
|
|
67
81
|
run: |
|
|
68
82
|
python -m build
|
|
@@ -5,11 +5,31 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
-
## [0.
|
|
8
|
+
## [0.9.0] - 2026-04-23
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
- `insert_yaml_as_extern` supports compressing the produced extern bytes via a new `compression_type` argument (zlib, zstd, lz4, brotli). Fixes #19
|
|
12
|
+
- `extern_compression` example schema plus `test_compression.py` covering unit and end-to-end round-trips for all compression types
|
|
13
|
+
|
|
14
|
+
### Changed
|
|
15
|
+
- `extract_extern_as_yaml` and `insert_yaml_as_extern` now share a single `_decompress` / `_compress` helper pair so new algorithms plug in one place
|
|
16
|
+
- `extract_extern_as_yaml` default for `compression_type` aligned to `None` (was `0`); behavior is unchanged for all callers
|
|
17
|
+
|
|
18
|
+
## [0.8.3] - 2025-12-07
|
|
9
19
|
|
|
10
20
|
### Added
|
|
11
21
|
- `bin_to_dict` function to deserialize binary directly to dict
|
|
12
22
|
|
|
23
|
+
### Changed
|
|
24
|
+
- Use `guess-next-dev` version scheme for pre-release uploads from main
|
|
25
|
+
|
|
26
|
+
### Note
|
|
27
|
+
- v0.8.2 was published to PyPI without the `bin_to_dict` export due to a release process issue; use v0.8.3 instead
|
|
28
|
+
|
|
29
|
+
## [0.8.2] - 2025-12-06
|
|
30
|
+
|
|
31
|
+
_Superseded by v0.8.3 - missing `bin_to_dict` export_
|
|
32
|
+
|
|
13
33
|
## [0.8.1] - 2025-12-03
|
|
14
34
|
|
|
15
35
|
### Added
|
|
@@ -28,7 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
28
48
|
|
|
29
49
|
### Added
|
|
30
50
|
- `py_eval` transformation function for generating node values via Python snippets
|
|
31
|
-
- `extract_extern_as_yaml` supports compressed externals (
|
|
51
|
+
- `extract_extern_as_yaml` supports compressed externals (zlib, zstd, lz4, brotli)
|
|
32
52
|
- `extract_extern_as_yaml` supports generating YAMLs without null-value fields
|
|
33
53
|
|
|
34
54
|
## [0.6.1] - 2024-12-04
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zs-yaml
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4.dev4
|
|
4
4
|
Summary: YAML support for zserio
|
|
5
5
|
Author-email: "Navigation Data Standard e.V." <support@nds-association.org>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -248,10 +248,17 @@ zs-yaml person.yaml person.bin
|
|
|
248
248
|
|
|
249
249
|
zs-yaml comes with several built-in transformation functions that can be used in your YAML files. Here's a brief overview of the available functions:
|
|
250
250
|
|
|
251
|
-
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio.
|
|
251
|
+
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio. Optionally compresses the produced bytes via `compression_type` (`zlib`, `zstd`, `lz4`, `brotli`; omit or set to `no_compression` for raw). Example:
|
|
252
|
+
```yaml
|
|
253
|
+
data:
|
|
254
|
+
_f: insert_yaml_as_extern
|
|
255
|
+
_a:
|
|
256
|
+
file: payload.yaml
|
|
257
|
+
compression_type: zstd # enum name, integer (0-4) or CompressionType member
|
|
258
|
+
```
|
|
252
259
|
- `insert_yaml`: Inserts YAML content directly from an external file.
|
|
253
260
|
- `repeat_node`: Repeats a specific node a specified number of times.
|
|
254
|
-
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file.
|
|
261
|
+
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file. Accepts the same `compression_type` values as `insert_yaml_as_extern` and decompresses the buffer before deserialization.
|
|
255
262
|
- `py_eval`: Allows to write small snippets like `myArray: {_f: py_eval, _a: "list(range(1, 100))"}` to generate the value for a yaml node.
|
|
256
263
|
|
|
257
264
|
For more detailed information about these functions and their usage, please refer to the [built_in_transformations.py](https://github.com/ndsev/zs-yaml/blob/main/zs_yaml/built_in_transformations.py) source file.
|
|
@@ -218,10 +218,17 @@ zs-yaml person.yaml person.bin
|
|
|
218
218
|
|
|
219
219
|
zs-yaml comes with several built-in transformation functions that can be used in your YAML files. Here's a brief overview of the available functions:
|
|
220
220
|
|
|
221
|
-
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio.
|
|
221
|
+
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio. Optionally compresses the produced bytes via `compression_type` (`zlib`, `zstd`, `lz4`, `brotli`; omit or set to `no_compression` for raw). Example:
|
|
222
|
+
```yaml
|
|
223
|
+
data:
|
|
224
|
+
_f: insert_yaml_as_extern
|
|
225
|
+
_a:
|
|
226
|
+
file: payload.yaml
|
|
227
|
+
compression_type: zstd # enum name, integer (0-4) or CompressionType member
|
|
228
|
+
```
|
|
222
229
|
- `insert_yaml`: Inserts YAML content directly from an external file.
|
|
223
230
|
- `repeat_node`: Repeats a specific node a specified number of times.
|
|
224
|
-
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file.
|
|
231
|
+
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file. Accepts the same `compression_type` values as `insert_yaml_as_extern` and decompresses the buffer before deserialization.
|
|
225
232
|
- `py_eval`: Allows to write small snippets like `myArray: {_f: py_eval, _a: "list(range(1, 100))"}` to generate the value for a yaml node.
|
|
226
233
|
|
|
227
234
|
For more detailed information about these functions and their usage, please refer to the [built_in_transformations.py](https://github.com/ndsev/zs-yaml/blob/main/zs_yaml/built_in_transformations.py) source file.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
package extern_compression;
|
|
2
|
+
|
|
3
|
+
enum uint8 CompressionType
|
|
4
|
+
{
|
|
5
|
+
NO_COMPRESSION = 0,
|
|
6
|
+
ZLIB = 1,
|
|
7
|
+
ZSTD = 2,
|
|
8
|
+
LZ4 = 3,
|
|
9
|
+
BROTLI = 4
|
|
10
|
+
};
|
|
11
|
+
|
|
12
|
+
struct Payload
|
|
13
|
+
{
|
|
14
|
+
string name;
|
|
15
|
+
uint32 value;
|
|
16
|
+
string tags[];
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
struct CompressedBlob
|
|
20
|
+
{
|
|
21
|
+
CompressionType compressionType;
|
|
22
|
+
extern data;
|
|
23
|
+
};
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Tests for compression support in insert_yaml_as_extern / extract_extern_as_yaml."""
|
|
3
|
+
|
|
4
|
+
import os
|
|
5
|
+
import sys
|
|
6
|
+
import tempfile
|
|
7
|
+
import yaml
|
|
8
|
+
|
|
9
|
+
SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
|
|
10
|
+
sys.path.insert(0, os.path.abspath(os.path.join(SCRIPT_DIR, '..', '..')))
|
|
11
|
+
sys.path.insert(0, os.path.join(SCRIPT_DIR, 'zs_gen_api'))
|
|
12
|
+
|
|
13
|
+
from zs_yaml.built_in_transformations import (
|
|
14
|
+
CompressionType,
|
|
15
|
+
_compress,
|
|
16
|
+
extract_extern_as_yaml,
|
|
17
|
+
)
|
|
18
|
+
from zs_yaml.convert import yaml_to_bin, bin_to_dict
|
|
19
|
+
from zs_yaml.yaml_transformer import YamlTransformer
|
|
20
|
+
|
|
21
|
+
COMPRESSION_CASES = [
|
|
22
|
+
('NONE', CompressionType.NO_COMPRESSION, 0),
|
|
23
|
+
('ZLIB', CompressionType.ZLIB, 1),
|
|
24
|
+
('ZSTD', CompressionType.ZSTD, 2),
|
|
25
|
+
('LZ4', CompressionType.LZ4, 3),
|
|
26
|
+
('BROTLI', CompressionType.BROTLI, 4),
|
|
27
|
+
]
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def test_compress_unit():
|
|
31
|
+
"""_compress round-trips via each algorithm's own decompressor."""
|
|
32
|
+
print("Testing _compress unit round-trips...")
|
|
33
|
+
import zlib, zstandard, lz4.frame, brotli
|
|
34
|
+
data = b'hello world, repeatable payload for compression testing. ' * 20
|
|
35
|
+
|
|
36
|
+
decompressors = {
|
|
37
|
+
CompressionType.ZLIB: zlib.decompress,
|
|
38
|
+
CompressionType.ZSTD: lambda b: zstandard.ZstdDecompressor().decompress(b),
|
|
39
|
+
CompressionType.LZ4: lz4.frame.decompress,
|
|
40
|
+
CompressionType.BROTLI: brotli.decompress,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
for name, ct, _ in COMPRESSION_CASES:
|
|
44
|
+
compressed = _compress(data, ct)
|
|
45
|
+
if ct == CompressionType.NO_COMPRESSION:
|
|
46
|
+
assert compressed == data, "NO_COMPRESSION should be identity"
|
|
47
|
+
print(f" ✓ _compress[{name}] identity")
|
|
48
|
+
continue
|
|
49
|
+
assert decompressors[ct](compressed) == data, f"{name} round-trip failed"
|
|
50
|
+
print(f" ✓ _compress[{name}] round-trip")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def test_compression_type_arg_resolution():
|
|
54
|
+
"""compression_type accepts enum / string / int and rejects junk."""
|
|
55
|
+
print("\nTesting compression_type argument resolution...")
|
|
56
|
+
assert CompressionType.from_string("zstd") == CompressionType.ZSTD
|
|
57
|
+
assert CompressionType.from_string("ZLIB") == CompressionType.ZLIB
|
|
58
|
+
assert CompressionType(3) == CompressionType.LZ4
|
|
59
|
+
try:
|
|
60
|
+
CompressionType.from_string("nope")
|
|
61
|
+
except ValueError:
|
|
62
|
+
pass
|
|
63
|
+
else:
|
|
64
|
+
raise AssertionError("unknown compression name should raise")
|
|
65
|
+
print(" ✓ enum/string/int resolution ok")
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _wrapper_yaml(ct_int, abs_payload_path):
|
|
69
|
+
return (
|
|
70
|
+
"_meta:\n"
|
|
71
|
+
" schema_module: extern_compression.api\n"
|
|
72
|
+
" schema_type: CompressedBlob\n"
|
|
73
|
+
"\n"
|
|
74
|
+
f"compressionType: {ct_int}\n"
|
|
75
|
+
"data:\n"
|
|
76
|
+
" _f: insert_yaml_as_extern\n"
|
|
77
|
+
" _a:\n"
|
|
78
|
+
f" file: \"{abs_payload_path}\"\n"
|
|
79
|
+
f" compression_type: {ct_int}\n"
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _normalize_ct(value):
|
|
84
|
+
if isinstance(value, CompressionType):
|
|
85
|
+
return value.value
|
|
86
|
+
if isinstance(value, str):
|
|
87
|
+
return CompressionType.from_string(value).value
|
|
88
|
+
return int(value)
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def test_roundtrip_via_extract_extern():
|
|
92
|
+
"""wrapper.yaml -> bin -> dict -> extract_extern_as_yaml -> compare."""
|
|
93
|
+
print("\nTesting end-to-end compression roundtrip...")
|
|
94
|
+
|
|
95
|
+
payload_abs = os.path.join(SCRIPT_DIR, 'payload.yaml')
|
|
96
|
+
with open(payload_abs) as f:
|
|
97
|
+
expected = yaml.safe_load(f)
|
|
98
|
+
expected_fields = {k: v for k, v in expected.items() if k != '_meta'}
|
|
99
|
+
|
|
100
|
+
for name, ct, ct_int in COMPRESSION_CASES:
|
|
101
|
+
with tempfile.TemporaryDirectory() as td:
|
|
102
|
+
wrapper_path = os.path.join(td, f'wrapper_{name}.yaml')
|
|
103
|
+
bin_path = os.path.join(td, f'blob_{name}.bin')
|
|
104
|
+
with open(wrapper_path, 'w') as f:
|
|
105
|
+
f.write(_wrapper_yaml(ct_int, payload_abs))
|
|
106
|
+
|
|
107
|
+
yaml_to_bin(wrapper_path, bin_path)
|
|
108
|
+
|
|
109
|
+
decoded, _ = bin_to_dict(bin_path, 'extern_compression.api', 'CompressedBlob')
|
|
110
|
+
assert _normalize_ct(decoded['compressionType']) == ct_int, (
|
|
111
|
+
f"[{name}] compressionType mismatch: got {decoded['compressionType']!r}, want {ct_int}"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
host_yaml_path = os.path.join(td, f'host_{name}.yaml')
|
|
115
|
+
with open(host_yaml_path, 'w') as f:
|
|
116
|
+
f.write("_meta:\n schema_module: extern_compression.api\n schema_type: Payload\n")
|
|
117
|
+
host_transformer = YamlTransformer(host_yaml_path)
|
|
118
|
+
|
|
119
|
+
extracted_name = f'extracted_{name}.yaml'
|
|
120
|
+
extract_extern_as_yaml(
|
|
121
|
+
host_transformer,
|
|
122
|
+
buffer=decoded['data']['buffer'],
|
|
123
|
+
bitSize=decoded['data']['bitSize'],
|
|
124
|
+
schema_module='extern_compression.api',
|
|
125
|
+
schema_type='Payload',
|
|
126
|
+
file_name=extracted_name,
|
|
127
|
+
compression_type=ct_int,
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
with open(os.path.join(td, extracted_name)) as f:
|
|
131
|
+
extracted = yaml.safe_load(f)
|
|
132
|
+
extracted_fields = {k: v for k, v in extracted.items() if k != '_meta'}
|
|
133
|
+
|
|
134
|
+
assert extracted_fields == expected_fields, (
|
|
135
|
+
f"[{name}] extracted payload differs from input:\n"
|
|
136
|
+
f" got: {extracted_fields!r}\n"
|
|
137
|
+
f" expected: {expected_fields!r}"
|
|
138
|
+
)
|
|
139
|
+
print(f" ✓ roundtrip[{name}] ok")
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
if __name__ == "__main__":
|
|
143
|
+
try:
|
|
144
|
+
test_compress_unit()
|
|
145
|
+
test_compression_type_arg_resolution()
|
|
146
|
+
test_roundtrip_via_extract_extern()
|
|
147
|
+
print("\n✅ All compression tests passed!")
|
|
148
|
+
sys.exit(0)
|
|
149
|
+
except Exception as e:
|
|
150
|
+
print(f"\n❌ Test failed: {e}")
|
|
151
|
+
import traceback
|
|
152
|
+
traceback.print_exc()
|
|
153
|
+
sys.exit(1)
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.8.4.dev4'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 8, 4, 'dev4')
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = 'g88e2cec8b'
|
|
@@ -30,7 +30,48 @@ class CompressionType(Enum):
|
|
|
30
30
|
except KeyError:
|
|
31
31
|
raise ValueError(f"Unknown compression type: {value}. Valid values are: {', '.join(cls.__members__.keys())}")
|
|
32
32
|
|
|
33
|
-
def
|
|
33
|
+
def _compress(data: bytes, compression_type: 'CompressionType') -> bytes:
|
|
34
|
+
"""Compress data using the specified compression algorithm."""
|
|
35
|
+
if compression_type == CompressionType.ZLIB:
|
|
36
|
+
return zlib.compress(data)
|
|
37
|
+
elif compression_type == CompressionType.ZSTD:
|
|
38
|
+
cctx = zstandard.ZstdCompressor()
|
|
39
|
+
return cctx.compress(data)
|
|
40
|
+
elif compression_type == CompressionType.LZ4:
|
|
41
|
+
return lz4.frame.compress(data)
|
|
42
|
+
elif compression_type == CompressionType.BROTLI:
|
|
43
|
+
return brotli.compress(data)
|
|
44
|
+
return data
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _decompress(data: bytes, compression_type: 'CompressionType') -> bytes:
|
|
48
|
+
"""Decompress data using the specified compression algorithm."""
|
|
49
|
+
if compression_type == CompressionType.ZLIB:
|
|
50
|
+
return zlib.decompress(data)
|
|
51
|
+
elif compression_type == CompressionType.ZSTD:
|
|
52
|
+
dctx = zstandard.ZstdDecompressor()
|
|
53
|
+
return dctx.decompress(data)
|
|
54
|
+
elif compression_type == CompressionType.LZ4:
|
|
55
|
+
return lz4.frame.decompress(data)
|
|
56
|
+
elif compression_type == CompressionType.BROTLI:
|
|
57
|
+
return brotli.decompress(data)
|
|
58
|
+
return data
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _resolve_compression_type(compression_type):
|
|
62
|
+
"""Normalize a user-provided compression_type (enum/str/int/None) to a CompressionType or None."""
|
|
63
|
+
if compression_type is None:
|
|
64
|
+
return None
|
|
65
|
+
if isinstance(compression_type, CompressionType):
|
|
66
|
+
return compression_type
|
|
67
|
+
if isinstance(compression_type, str):
|
|
68
|
+
return CompressionType.from_string(compression_type)
|
|
69
|
+
if isinstance(compression_type, int):
|
|
70
|
+
return CompressionType(compression_type)
|
|
71
|
+
raise ValueError("compression_type must be a CompressionType enum, string, or integer value")
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def insert_yaml_as_extern(transformer, file, template_args=None, compression_type=None):
|
|
34
75
|
"""
|
|
35
76
|
Include external YAML by transforming it to JSON and using zserio.
|
|
36
77
|
|
|
@@ -38,12 +79,17 @@ def insert_yaml_as_extern(transformer, file, template_args=None):
|
|
|
38
79
|
transformer (YamlTransformer): The transformer instance.
|
|
39
80
|
file (str): Path to the external YAML file.
|
|
40
81
|
template_args (dict, optional): A dictionary of template arguments for placeholder replacement.
|
|
82
|
+
compression_type (Union[CompressionType, str, int, None]): Compression to apply
|
|
83
|
+
after serialization. Can be a CompressionType enum, string, or integer.
|
|
84
|
+
Defaults to None (no compression).
|
|
41
85
|
|
|
42
86
|
Returns:
|
|
43
87
|
dict: A dictionary containing the binary data and its bit size.
|
|
44
88
|
"""
|
|
45
89
|
from .yaml_transformer import TransformationError
|
|
46
|
-
|
|
90
|
+
|
|
91
|
+
ct_enum = _resolve_compression_type(compression_type)
|
|
92
|
+
|
|
47
93
|
abs_path = transformer.resolve_path(file)
|
|
48
94
|
try:
|
|
49
95
|
external_transformer = transformer.__class__(abs_path, template_args, initial_transformations=transformer.transformations)
|
|
@@ -56,7 +102,7 @@ def insert_yaml_as_extern(transformer, file, template_args=None):
|
|
|
56
102
|
file_path=abs_path,
|
|
57
103
|
original_error=e
|
|
58
104
|
)
|
|
59
|
-
|
|
105
|
+
|
|
60
106
|
processed_data = external_transformer.data
|
|
61
107
|
meta = external_transformer.metadata
|
|
62
108
|
|
|
@@ -84,11 +130,10 @@ def insert_yaml_as_extern(transformer, file, template_args=None):
|
|
|
84
130
|
file_path=abs_path,
|
|
85
131
|
original_error=e
|
|
86
132
|
)
|
|
87
|
-
|
|
133
|
+
|
|
88
134
|
try:
|
|
89
135
|
writer = zserio.BitStreamWriter()
|
|
90
136
|
zserio_object.write(writer)
|
|
91
|
-
bits = zserio.BitBuffer(writer.byte_array, writer.bitposition)
|
|
92
137
|
except Exception as e:
|
|
93
138
|
raise TransformationError(
|
|
94
139
|
f"Failed to serialize {schema_type}: {e}",
|
|
@@ -96,11 +141,18 @@ def insert_yaml_as_extern(transformer, file, template_args=None):
|
|
|
96
141
|
original_error=e
|
|
97
142
|
)
|
|
98
143
|
|
|
99
|
-
#
|
|
100
|
-
|
|
101
|
-
|
|
144
|
+
# Apply compression if requested
|
|
145
|
+
if ct_enum is not None and ct_enum != CompressionType.NO_COMPRESSION:
|
|
146
|
+
raw_bytes = bytes(writer.byte_array)
|
|
147
|
+
compressed = _compress(raw_bytes, ct_enum)
|
|
148
|
+
encoded_bytes = list(compressed)
|
|
149
|
+
bit_size = len(compressed) * 8
|
|
150
|
+
else:
|
|
151
|
+
bits = zserio.BitBuffer(writer.byte_array, writer.bitposition)
|
|
152
|
+
encoded_bytes = list(bits.buffer)
|
|
153
|
+
bit_size = bits.bitsize
|
|
102
154
|
|
|
103
|
-
return
|
|
155
|
+
return {"buffer": encoded_bytes, "bitSize": bit_size}
|
|
104
156
|
|
|
105
157
|
def insert_yaml(transformer, file, node_path='', template_args=None, cache_file=True):
|
|
106
158
|
"""
|
|
@@ -186,7 +238,7 @@ def repeat_node(transformer, node, count):
|
|
|
186
238
|
return [copy.deepcopy(node) for _ in range(count)]
|
|
187
239
|
|
|
188
240
|
|
|
189
|
-
def extract_extern_as_yaml(transformer, buffer, bitSize, schema_module, schema_type, file_name, compression_type=
|
|
241
|
+
def extract_extern_as_yaml(transformer, buffer, bitSize, schema_module, schema_type, file_name, compression_type=None, remove_nulls=False):
|
|
190
242
|
"""
|
|
191
243
|
Extract binary data and save as an external YAML file.
|
|
192
244
|
|
|
@@ -205,14 +257,7 @@ def extract_extern_as_yaml(transformer, buffer, bitSize, schema_module, schema_t
|
|
|
205
257
|
Returns:
|
|
206
258
|
dict: A reference to the extracted file.
|
|
207
259
|
"""
|
|
208
|
-
|
|
209
|
-
if compression_type is not None:
|
|
210
|
-
if isinstance(compression_type, str):
|
|
211
|
-
compression_type = CompressionType.from_string(compression_type)
|
|
212
|
-
elif isinstance(compression_type, int):
|
|
213
|
-
compression_type = CompressionType(compression_type)
|
|
214
|
-
elif not isinstance(compression_type, CompressionType):
|
|
215
|
-
raise ValueError("compression_type must be a CompressionType enum, string, or integer value")
|
|
260
|
+
compression_type = _resolve_compression_type(compression_type)
|
|
216
261
|
|
|
217
262
|
# Ensure the output directory exists
|
|
218
263
|
output_dir = os.path.dirname(transformer.yaml_file_path)
|
|
@@ -224,15 +269,7 @@ def extract_extern_as_yaml(transformer, buffer, bitSize, schema_module, schema_t
|
|
|
224
269
|
# Extract and decompress binary data if needed
|
|
225
270
|
buffer = bytes(buffer)
|
|
226
271
|
if compression_type is not None:
|
|
227
|
-
|
|
228
|
-
buffer = zlib.decompress(buffer)
|
|
229
|
-
elif compression_type == CompressionType.ZSTD:
|
|
230
|
-
dctx = zstandard.ZstdDecompressor()
|
|
231
|
-
buffer = dctx.decompress(buffer)
|
|
232
|
-
elif compression_type == CompressionType.LZ4:
|
|
233
|
-
buffer = lz4.frame.decompress(buffer)
|
|
234
|
-
elif compression_type == CompressionType.BROTLI:
|
|
235
|
-
buffer = brotli.decompress(buffer)
|
|
272
|
+
buffer = _decompress(buffer, compression_type)
|
|
236
273
|
|
|
237
274
|
# Import the module and get the type
|
|
238
275
|
module = importlib.import_module(schema_module)
|
|
@@ -269,12 +306,16 @@ def extract_extern_as_yaml(transformer, buffer, bitSize, schema_module, schema_t
|
|
|
269
306
|
yaml.dump(data_to_write, f, default_flow_style=False, sort_keys=False)
|
|
270
307
|
|
|
271
308
|
# Return a reference to the extracted file
|
|
272
|
-
|
|
309
|
+
ref = {
|
|
273
310
|
'_f': 'insert_yaml_as_extern',
|
|
274
311
|
'_a': {
|
|
275
312
|
'file': file_name
|
|
276
313
|
}
|
|
277
314
|
}
|
|
315
|
+
# Propagate compression type so insert_yaml_as_extern re-compresses
|
|
316
|
+
if compression_type is not None and compression_type != CompressionType.NO_COMPRESSION:
|
|
317
|
+
ref['_a']['compression_type'] = compression_type.value
|
|
318
|
+
return ref
|
|
278
319
|
|
|
279
320
|
def py_eval(transformer, expr):
|
|
280
321
|
"""
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: zs-yaml
|
|
3
|
-
Version: 0.8.
|
|
3
|
+
Version: 0.8.4.dev4
|
|
4
4
|
Summary: YAML support for zserio
|
|
5
5
|
Author-email: "Navigation Data Standard e.V." <support@nds-association.org>
|
|
6
6
|
License-Expression: MIT
|
|
@@ -248,10 +248,17 @@ zs-yaml person.yaml person.bin
|
|
|
248
248
|
|
|
249
249
|
zs-yaml comes with several built-in transformation functions that can be used in your YAML files. Here's a brief overview of the available functions:
|
|
250
250
|
|
|
251
|
-
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio.
|
|
251
|
+
- `insert_yaml_as_extern`: Includes external YAML content by transforming it to JSON and using zserio. Optionally compresses the produced bytes via `compression_type` (`zlib`, `zstd`, `lz4`, `brotli`; omit or set to `no_compression` for raw). Example:
|
|
252
|
+
```yaml
|
|
253
|
+
data:
|
|
254
|
+
_f: insert_yaml_as_extern
|
|
255
|
+
_a:
|
|
256
|
+
file: payload.yaml
|
|
257
|
+
compression_type: zstd # enum name, integer (0-4) or CompressionType member
|
|
258
|
+
```
|
|
252
259
|
- `insert_yaml`: Inserts YAML content directly from an external file.
|
|
253
260
|
- `repeat_node`: Repeats a specific node a specified number of times.
|
|
254
|
-
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file.
|
|
261
|
+
- `extract_extern_as_yaml`: Extracts binary data and saves it as an external YAML file. Accepts the same `compression_type` values as `insert_yaml_as_extern` and decompresses the buffer before deserialization.
|
|
255
262
|
- `py_eval`: Allows to write small snippets like `myArray: {_f: py_eval, _a: "list(range(1, 100))"}` to generate the value for a yaml node.
|
|
256
263
|
|
|
257
264
|
For more detailed information about these functions and their usage, please refer to the [built_in_transformations.py](https://github.com/ndsev/zs-yaml/blob/main/zs_yaml/built_in_transformations.py) source file.
|
|
@@ -6,6 +6,9 @@ pyproject.toml
|
|
|
6
6
|
.github/workflows/ci.yml
|
|
7
7
|
doc/zs-yaml.png
|
|
8
8
|
doc/zs-yaml.svg
|
|
9
|
+
examples/extern_compression/extern_compression.zs
|
|
10
|
+
examples/extern_compression/payload.yaml
|
|
11
|
+
examples/extern_compression/test_compression.py
|
|
9
12
|
examples/team/addresses.yaml
|
|
10
13
|
examples/team/custom_transformations.py
|
|
11
14
|
examples/team/hobbies.yaml
|
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
# file generated by setuptools-scm
|
|
2
|
-
# don't change, don't track in version control
|
|
3
|
-
|
|
4
|
-
__all__ = [
|
|
5
|
-
"__version__",
|
|
6
|
-
"__version_tuple__",
|
|
7
|
-
"version",
|
|
8
|
-
"version_tuple",
|
|
9
|
-
"__commit_id__",
|
|
10
|
-
"commit_id",
|
|
11
|
-
]
|
|
12
|
-
|
|
13
|
-
TYPE_CHECKING = False
|
|
14
|
-
if TYPE_CHECKING:
|
|
15
|
-
from typing import Tuple
|
|
16
|
-
from typing import Union
|
|
17
|
-
|
|
18
|
-
VERSION_TUPLE = Tuple[Union[int, str], ...]
|
|
19
|
-
COMMIT_ID = Union[str, None]
|
|
20
|
-
else:
|
|
21
|
-
VERSION_TUPLE = object
|
|
22
|
-
COMMIT_ID = object
|
|
23
|
-
|
|
24
|
-
version: str
|
|
25
|
-
__version__: str
|
|
26
|
-
__version_tuple__: VERSION_TUPLE
|
|
27
|
-
version_tuple: VERSION_TUPLE
|
|
28
|
-
commit_id: COMMIT_ID
|
|
29
|
-
__commit_id__: COMMIT_ID
|
|
30
|
-
|
|
31
|
-
__version__ = version = '0.8.3.dev3'
|
|
32
|
-
__version_tuple__ = version_tuple = (0, 8, 3, 'dev3')
|
|
33
|
-
|
|
34
|
-
__commit_id__ = commit_id = 'gc8ec0d36c'
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|