omnibuf 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnibuf-0.1.0/.github/workflows/ci.yml +40 -0
- omnibuf-0.1.0/.github/workflows/release.yml +187 -0
- omnibuf-0.1.0/.gitignore +16 -0
- omnibuf-0.1.0/AGENTS.md +35 -0
- omnibuf-0.1.0/Cargo.lock +158 -0
- omnibuf-0.1.0/Cargo.toml +27 -0
- omnibuf-0.1.0/LICENSE +21 -0
- omnibuf-0.1.0/PKG-INFO +204 -0
- omnibuf-0.1.0/README.md +177 -0
- omnibuf-0.1.0/benchmarks/bench.py +1008 -0
- omnibuf-0.1.0/buf.gen.yaml +8 -0
- omnibuf-0.1.0/buf.yaml +11 -0
- omnibuf-0.1.0/justfile +59 -0
- omnibuf-0.1.0/mise.toml +7 -0
- omnibuf-0.1.0/proto/omnibuf/bench/peer.proto +34 -0
- omnibuf-0.1.0/proto/omnibuf/conformance/conformance.proto +78 -0
- omnibuf-0.1.0/pyproject.toml +91 -0
- omnibuf-0.1.0/rust/blind.rs +272 -0
- omnibuf-0.1.0/rust/codec.rs +2459 -0
- omnibuf-0.1.0/rust/lib.rs +541 -0
- omnibuf-0.1.0/rust/opaque.rs +2291 -0
- omnibuf-0.1.0/src/omnibuf/__init__.py +297 -0
- omnibuf-0.1.0/src/omnibuf/__main__.py +8 -0
- omnibuf-0.1.0/src/omnibuf/_omnibuf_ext.pyi +226 -0
- omnibuf-0.1.0/src/omnibuf/blind.py +44 -0
- omnibuf-0.1.0/src/omnibuf/conformance.py +704 -0
- omnibuf-0.1.0/src/omnibuf/dataclass_codec.py +972 -0
- omnibuf-0.1.0/src/omnibuf/decoder.py +436 -0
- omnibuf-0.1.0/src/omnibuf/encoder.py +427 -0
- omnibuf-0.1.0/src/omnibuf/field0.py +221 -0
- omnibuf-0.1.0/src/omnibuf/gen/__init__.py +1 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/__init__.py +1 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/bench/__init__.py +1 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/bench/peer_omnibuf_pb.py +307 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/bench/peer_pb.py +227 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/bench/peer_pb2.py +41 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/conformance/__init__.py +1 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/conformance/conformance_omnibuf_pb.py +480 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/conformance/conformance_pb.py +424 -0
- omnibuf-0.1.0/src/omnibuf/gen/omnibuf/conformance/conformance_pb2.py +51 -0
- omnibuf-0.1.0/src/omnibuf/opaque.py +381 -0
- omnibuf-0.1.0/src/omnibuf/plugin.py +801 -0
- omnibuf-0.1.0/src/omnibuf/protobuf_codec.py +1025 -0
- omnibuf-0.1.0/src/omnibuf/py.typed +0 -0
- omnibuf-0.1.0/src/omnibuf/wire.py +1081 -0
- omnibuf-0.1.0/testdata/benchmarks.json +276 -0
- omnibuf-0.1.0/testdata/benchmarks.txt +74 -0
- omnibuf-0.1.0/tests/test_blind.py +105 -0
- omnibuf-0.1.0/tests/test_codec.py +124 -0
- omnibuf-0.1.0/tests/test_dataclass_and_proto.py +259 -0
- omnibuf-0.1.0/tests/test_review_and_edge_cases.py +828 -0
- omnibuf-0.1.0/tests/test_wire.py +191 -0
- omnibuf-0.1.0/uv.lock +691 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
branches: [main]
|
|
8
|
+
workflow_dispatch:
|
|
9
|
+
|
|
10
|
+
concurrency:
|
|
11
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
12
|
+
cancel-in-progress: true
|
|
13
|
+
|
|
14
|
+
jobs:
|
|
15
|
+
ci:
|
|
16
|
+
name: Python & PyO3 Format, Lint, Mypy, Pytest & Conformance
|
|
17
|
+
runs-on: ubuntu-24.04
|
|
18
|
+
steps:
|
|
19
|
+
- name: Checkout repository
|
|
20
|
+
uses: actions/checkout@v7
|
|
21
|
+
|
|
22
|
+
- name: Install tools via mise
|
|
23
|
+
uses: jdx/mise-action@v5
|
|
24
|
+
with:
|
|
25
|
+
install: true
|
|
26
|
+
|
|
27
|
+
- name: Install rustfmt and clippy components
|
|
28
|
+
run: rustup component add rustfmt clippy
|
|
29
|
+
|
|
30
|
+
- name: Sync uv environment and build PyO3 extension
|
|
31
|
+
run: just install
|
|
32
|
+
|
|
33
|
+
- name: Install conformance-runner
|
|
34
|
+
run: |
|
|
35
|
+
mise use -g go@1.27
|
|
36
|
+
GOBIN="$HOME/.local/bin" mise exec -- go install github.com/omnibuf/omnibuf-go/cmd/conformance-runner@71dad83aadd7dc65401d662521d34cfaeb1011ab
|
|
37
|
+
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
38
|
+
|
|
39
|
+
- name: Run full CI suite
|
|
40
|
+
run: just ci
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
name: Release & Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*'
|
|
7
|
+
workflow_dispatch:
|
|
8
|
+
|
|
9
|
+
concurrency:
|
|
10
|
+
group: ${{ github.workflow }}-${{ github.ref }}
|
|
11
|
+
cancel-in-progress: false
|
|
12
|
+
|
|
13
|
+
permissions:
|
|
14
|
+
contents: read
|
|
15
|
+
|
|
16
|
+
jobs:
|
|
17
|
+
verify:
|
|
18
|
+
name: Format, Lint, Mypy, Pytest & Conformance
|
|
19
|
+
runs-on: ubuntu-24.04
|
|
20
|
+
steps:
|
|
21
|
+
- name: Checkout repository
|
|
22
|
+
uses: actions/checkout@v7
|
|
23
|
+
|
|
24
|
+
- name: Install tools via mise
|
|
25
|
+
uses: jdx/mise-action@v5
|
|
26
|
+
with:
|
|
27
|
+
install: true
|
|
28
|
+
|
|
29
|
+
- name: Install rustfmt and clippy components
|
|
30
|
+
run: rustup component add rustfmt clippy
|
|
31
|
+
|
|
32
|
+
- name: Sync uv environment and build PyO3 extension
|
|
33
|
+
run: just install
|
|
34
|
+
|
|
35
|
+
- name: Install conformance-runner
|
|
36
|
+
run: |
|
|
37
|
+
mise use -g go@1.27
|
|
38
|
+
GOBIN="$HOME/.local/bin" mise exec -- go install github.com/omnibuf/omnibuf-go/cmd/conformance-runner@71dad83aadd7dc65401d662521d34cfaeb1011ab
|
|
39
|
+
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
40
|
+
|
|
41
|
+
- name: Run full CI suite
|
|
42
|
+
run: just ci
|
|
43
|
+
|
|
44
|
+
linux:
|
|
45
|
+
name: Wheels (manylinux ${{ matrix.platform.target }})
|
|
46
|
+
needs: verify
|
|
47
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
48
|
+
strategy:
|
|
49
|
+
matrix:
|
|
50
|
+
platform:
|
|
51
|
+
- runner: ubuntu-24.04
|
|
52
|
+
target: x86_64
|
|
53
|
+
- runner: ubuntu-24.04-arm
|
|
54
|
+
target: aarch64
|
|
55
|
+
steps:
|
|
56
|
+
- uses: actions/checkout@v7
|
|
57
|
+
- uses: actions/setup-python@v6
|
|
58
|
+
with:
|
|
59
|
+
python-version: '3.12'
|
|
60
|
+
- name: Build manylinux wheels
|
|
61
|
+
uses: PyO3/maturin-action@v1
|
|
62
|
+
with:
|
|
63
|
+
target: ${{ matrix.platform.target }}
|
|
64
|
+
args: --release --out dist --find-interpreter
|
|
65
|
+
manylinux: auto
|
|
66
|
+
- uses: actions/upload-artifact@v6
|
|
67
|
+
with:
|
|
68
|
+
name: wheels-linux-${{ matrix.platform.target }}
|
|
69
|
+
path: dist
|
|
70
|
+
|
|
71
|
+
musllinux:
|
|
72
|
+
name: Wheels (musllinux ${{ matrix.platform.target }})
|
|
73
|
+
needs: verify
|
|
74
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
75
|
+
strategy:
|
|
76
|
+
matrix:
|
|
77
|
+
platform:
|
|
78
|
+
- runner: ubuntu-24.04
|
|
79
|
+
target: x86_64
|
|
80
|
+
- runner: ubuntu-24.04-arm
|
|
81
|
+
target: aarch64
|
|
82
|
+
steps:
|
|
83
|
+
- uses: actions/checkout@v7
|
|
84
|
+
- uses: actions/setup-python@v6
|
|
85
|
+
with:
|
|
86
|
+
python-version: '3.12'
|
|
87
|
+
- name: Build musllinux wheels
|
|
88
|
+
uses: PyO3/maturin-action@v1
|
|
89
|
+
with:
|
|
90
|
+
target: ${{ matrix.platform.target }}
|
|
91
|
+
args: --release --out dist --find-interpreter
|
|
92
|
+
manylinux: musllinux_1_2
|
|
93
|
+
- uses: actions/upload-artifact@v6
|
|
94
|
+
with:
|
|
95
|
+
name: wheels-musllinux-${{ matrix.platform.target }}
|
|
96
|
+
path: dist
|
|
97
|
+
|
|
98
|
+
macos:
|
|
99
|
+
name: Wheels (macOS ${{ matrix.platform.target }})
|
|
100
|
+
needs: verify
|
|
101
|
+
runs-on: ${{ matrix.platform.runner }}
|
|
102
|
+
strategy:
|
|
103
|
+
matrix:
|
|
104
|
+
platform:
|
|
105
|
+
- runner: macos-15-intel
|
|
106
|
+
target: x86_64
|
|
107
|
+
- runner: macos-latest
|
|
108
|
+
target: aarch64
|
|
109
|
+
steps:
|
|
110
|
+
- uses: actions/checkout@v7
|
|
111
|
+
- uses: actions/setup-python@v6
|
|
112
|
+
with:
|
|
113
|
+
python-version: |
|
|
114
|
+
3.11
|
|
115
|
+
3.12
|
|
116
|
+
3.13
|
|
117
|
+
3.14
|
|
118
|
+
- name: Build macOS wheels
|
|
119
|
+
uses: PyO3/maturin-action@v1
|
|
120
|
+
with:
|
|
121
|
+
target: ${{ matrix.platform.target }}
|
|
122
|
+
args: --release --out dist --find-interpreter
|
|
123
|
+
- uses: actions/upload-artifact@v6
|
|
124
|
+
with:
|
|
125
|
+
name: wheels-macos-${{ matrix.platform.target }}
|
|
126
|
+
path: dist
|
|
127
|
+
|
|
128
|
+
windows:
|
|
129
|
+
name: Wheels (Windows x64)
|
|
130
|
+
needs: verify
|
|
131
|
+
runs-on: windows-latest
|
|
132
|
+
steps:
|
|
133
|
+
- uses: actions/checkout@v7
|
|
134
|
+
- uses: actions/setup-python@v6
|
|
135
|
+
with:
|
|
136
|
+
python-version: |
|
|
137
|
+
3.11
|
|
138
|
+
3.12
|
|
139
|
+
3.13
|
|
140
|
+
3.14
|
|
141
|
+
architecture: x64
|
|
142
|
+
- name: Build Windows x64 wheels
|
|
143
|
+
uses: PyO3/maturin-action@v1
|
|
144
|
+
with:
|
|
145
|
+
target: x64
|
|
146
|
+
args: --release --out dist --find-interpreter
|
|
147
|
+
- uses: actions/upload-artifact@v6
|
|
148
|
+
with:
|
|
149
|
+
name: wheels-windows-x64
|
|
150
|
+
path: dist
|
|
151
|
+
|
|
152
|
+
sdist:
|
|
153
|
+
name: Source Distribution (sdist)
|
|
154
|
+
needs: verify
|
|
155
|
+
runs-on: ubuntu-24.04
|
|
156
|
+
steps:
|
|
157
|
+
- uses: actions/checkout@v7
|
|
158
|
+
- name: Build sdist
|
|
159
|
+
uses: PyO3/maturin-action@v1
|
|
160
|
+
with:
|
|
161
|
+
command: sdist
|
|
162
|
+
args: --out dist
|
|
163
|
+
- uses: actions/upload-artifact@v6
|
|
164
|
+
with:
|
|
165
|
+
name: wheels-sdist
|
|
166
|
+
path: dist
|
|
167
|
+
|
|
168
|
+
publish:
|
|
169
|
+
name: Publish to PyPI via OIDC
|
|
170
|
+
needs: [linux, musllinux, macos, windows, sdist]
|
|
171
|
+
runs-on: ubuntu-24.04
|
|
172
|
+
permissions:
|
|
173
|
+
contents: read
|
|
174
|
+
id-token: write
|
|
175
|
+
steps:
|
|
176
|
+
- name: Download built wheels and sdist
|
|
177
|
+
uses: actions/download-artifact@v7
|
|
178
|
+
with:
|
|
179
|
+
pattern: wheels-*
|
|
180
|
+
path: dist
|
|
181
|
+
merge-multiple: true
|
|
182
|
+
|
|
183
|
+
- name: Publish distributions to PyPI
|
|
184
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
185
|
+
with:
|
|
186
|
+
packages-dir: dist/
|
|
187
|
+
skip-existing: true
|
omnibuf-0.1.0/.gitignore
ADDED
omnibuf-0.1.0/AGENTS.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# AGENTS.md
|
|
2
|
+
|
|
3
|
+
Repository guide for autonomous coding agents working inside `omnibuf-py`.
|
|
4
|
+
|
|
5
|
+
## Toolchain & Commands
|
|
6
|
+
|
|
7
|
+
Tools are pinned in `mise.toml` (`python = "3.14.7"`, `uv`, `just = "1.58.0"`, `protobuf = "35.1"`, `buf = "1.75.0"`).
|
|
8
|
+
|
|
9
|
+
- `just install`: Sync `.venv` and build the compiled Rust extension via `uv sync`.
|
|
10
|
+
- `just proto`: Run `buf lint` and `buf generate` (`buf.yaml` + `buf.gen.yaml`). Never invoke raw `protoc` CLI commands directly.
|
|
11
|
+
- `just fmt` / `just fmt-check`: Format or verify Rust (`cargo fmt`) and Python (`ruff format`) sources.
|
|
12
|
+
- `just lint`: Run `buf lint`, `cargo clippy --all-targets -- -D warnings`, `ruff check .`, and strict `mypy src tests benchmarks`.
|
|
13
|
+
- `just test`: Run unit tests (`uv run pytest`).
|
|
14
|
+
- `just conformance`: Run the 758-case cross-language conformance runner (`0 skipped, 0 failed` required).
|
|
15
|
+
- `just bench`: Run `benchmarks/bench.py` and write `testdata/benchmarks.json` plus `testdata/benchmarks.txt` (pass `--smoke` to smoke-check timing logic leaving `testdata/benchmarks.json` untouched).
|
|
16
|
+
- `just ci`: Full verification gate (`fmt-check`, `lint`, `test`, `conformance`).
|
|
17
|
+
|
|
18
|
+
## Architecture
|
|
19
|
+
|
|
20
|
+
- `rust/lib.rs`, `rust/codec.rs`, `rust/opaque.rs`, `rust/blind.rs`: Compiled PyO3 + Rust extension (`omnibuf._omnibuf_ext`, typed in `src/omnibuf/_omnibuf_ext.pyi`) backed by `omnibuf-rs`. Uses `LeafBytes` (`Py_buffer` RAII guard) and `InputBytes` (upfront snapshot for mutable buffers) for UAF-safe buffer handling.
|
|
21
|
+
- `src/omnibuf/wire.py`: OmniBuf v0.1 Prefix-Varint (`TZCNT_WIDTH_TABLE`), ZigZag 32/64, container length backpatching, strict UTF-8, shortest round-trip `float32`, `locate_field`, and aliased-safe `splice_fields`.
|
|
22
|
+
- `src/omnibuf/encoder.py` & `src/omnibuf/decoder.py`: Streaming single-pass writer and depth-bounded reader (`MAX_CONTAINER_DEPTH = 32`).
|
|
23
|
+
- `src/omnibuf/field0.py` & `src/omnibuf/blind.py`: `SchemaMeta` encoding/merging and two-pass `decode_blind` / `decode_blind_json`.
|
|
24
|
+
- `src/omnibuf/protobuf_codec.py`: Compiled reflection codec with primary support for `protobuf-py` (`bufbuild/protobuf-py` `_pb.py` messages and `DescMessage` descriptors) and lazy optional compatibility for `google.protobuf` (`_pb2.py`).
|
|
25
|
+
- `src/omnibuf/dataclass_codec.py`: `@omnibuf_message` and `omnibuf_field` codec supporting recursive types, `enum.IntEnum` / integer-valued `enum.Enum`, explicit wire overrides, `omitempty`, and instance-aware `self_describing=True`.
|
|
26
|
+
- `src/omnibuf/opaque.py` & `src/omnibuf/plugin.py`: `protoc-gen-omnibuf-py` code generator emitting slotted message dataclasses, `oneof` and explicit-presence tracking, compiled codecs, and `Opaque<Message>` header-hop views.
|
|
27
|
+
- `src/omnibuf/conformance.py`: Length-prefixed stdin/stdout conformance adapter exercising `TIER_PLUGIN`, `TIER_PROTOBUF_TYPES`, `TIER_ANNOTATED_STRUCT`, and `TIER_BLIND`.
|
|
28
|
+
|
|
29
|
+
## Invariants
|
|
30
|
+
|
|
31
|
+
1. `MAX_NESTING_DEPTH` and `MAX_CONTAINER_DEPTH` are `32`. Arrays and maps increment container depth.
|
|
32
|
+
2. Reserved Prefix-Varint markers (`(b & 0x07) == 0` when `b != 0`) must always fail decoding.
|
|
33
|
+
3. `WireExtended` (`15`) requires at least 1 payload byte whose subtype byte is in `0x00..0xFF`.
|
|
34
|
+
4. `Field #0` (`tag = 0x19`) requires `WireSubMsg` (`12`) everywhere it appears and may appear anywhere in a message or concatenated stream.
|
|
35
|
+
5. Never use em-dashes in documentation, comments, or strings.
|
omnibuf-0.1.0/Cargo.lock
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
# This file is automatically @generated by Cargo.
|
|
2
|
+
# It is not intended for manual editing.
|
|
3
|
+
version = 4
|
|
4
|
+
|
|
5
|
+
[[package]]
|
|
6
|
+
name = "heck"
|
|
7
|
+
version = "0.5.0"
|
|
8
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
9
|
+
checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
|
|
10
|
+
|
|
11
|
+
[[package]]
|
|
12
|
+
name = "itoa"
|
|
13
|
+
version = "1.0.19"
|
|
14
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
15
|
+
checksum = "3bd1574e617a9a6c1f8f8181bd7e841e1f20e4277607ef28945765188d6284d4"
|
|
16
|
+
|
|
17
|
+
[[package]]
|
|
18
|
+
name = "libc"
|
|
19
|
+
version = "0.2.191"
|
|
20
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
21
|
+
checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
|
|
22
|
+
|
|
23
|
+
[[package]]
|
|
24
|
+
name = "omnibuf"
|
|
25
|
+
version = "0.1.0"
|
|
26
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
27
|
+
checksum = "5fffb3295e8aa45e15ce3f539f3b1b7bb9a39057f9c2ddf60d18bb2da1d74140"
|
|
28
|
+
dependencies = [
|
|
29
|
+
"itoa",
|
|
30
|
+
"ryu",
|
|
31
|
+
]
|
|
32
|
+
|
|
33
|
+
[[package]]
|
|
34
|
+
name = "omnibuf-py-ext"
|
|
35
|
+
version = "0.1.0"
|
|
36
|
+
dependencies = [
|
|
37
|
+
"itoa",
|
|
38
|
+
"omnibuf",
|
|
39
|
+
"pyo3",
|
|
40
|
+
"ryu",
|
|
41
|
+
]
|
|
42
|
+
|
|
43
|
+
[[package]]
|
|
44
|
+
name = "once_cell"
|
|
45
|
+
version = "1.21.4"
|
|
46
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
47
|
+
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
|
48
|
+
|
|
49
|
+
[[package]]
|
|
50
|
+
name = "portable-atomic"
|
|
51
|
+
version = "1.15.0"
|
|
52
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
53
|
+
checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85"
|
|
54
|
+
|
|
55
|
+
[[package]]
|
|
56
|
+
name = "proc-macro2"
|
|
57
|
+
version = "1.0.107"
|
|
58
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
59
|
+
checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9"
|
|
60
|
+
dependencies = [
|
|
61
|
+
"unicode-ident",
|
|
62
|
+
]
|
|
63
|
+
|
|
64
|
+
[[package]]
|
|
65
|
+
name = "pyo3"
|
|
66
|
+
version = "0.28.3"
|
|
67
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
68
|
+
checksum = "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12"
|
|
69
|
+
dependencies = [
|
|
70
|
+
"libc",
|
|
71
|
+
"once_cell",
|
|
72
|
+
"portable-atomic",
|
|
73
|
+
"pyo3-build-config",
|
|
74
|
+
"pyo3-ffi",
|
|
75
|
+
"pyo3-macros",
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
[[package]]
|
|
79
|
+
name = "pyo3-build-config"
|
|
80
|
+
version = "0.28.3"
|
|
81
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
82
|
+
checksum = "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e"
|
|
83
|
+
dependencies = [
|
|
84
|
+
"target-lexicon",
|
|
85
|
+
]
|
|
86
|
+
|
|
87
|
+
[[package]]
|
|
88
|
+
name = "pyo3-ffi"
|
|
89
|
+
version = "0.28.3"
|
|
90
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
91
|
+
checksum = "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e"
|
|
92
|
+
dependencies = [
|
|
93
|
+
"libc",
|
|
94
|
+
"pyo3-build-config",
|
|
95
|
+
]
|
|
96
|
+
|
|
97
|
+
[[package]]
|
|
98
|
+
name = "pyo3-macros"
|
|
99
|
+
version = "0.28.3"
|
|
100
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
101
|
+
checksum = "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813"
|
|
102
|
+
dependencies = [
|
|
103
|
+
"proc-macro2",
|
|
104
|
+
"pyo3-macros-backend",
|
|
105
|
+
"quote",
|
|
106
|
+
"syn",
|
|
107
|
+
]
|
|
108
|
+
|
|
109
|
+
[[package]]
|
|
110
|
+
name = "pyo3-macros-backend"
|
|
111
|
+
version = "0.28.3"
|
|
112
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
113
|
+
checksum = "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb"
|
|
114
|
+
dependencies = [
|
|
115
|
+
"heck",
|
|
116
|
+
"proc-macro2",
|
|
117
|
+
"pyo3-build-config",
|
|
118
|
+
"quote",
|
|
119
|
+
"syn",
|
|
120
|
+
]
|
|
121
|
+
|
|
122
|
+
[[package]]
|
|
123
|
+
name = "quote"
|
|
124
|
+
version = "1.0.47"
|
|
125
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
126
|
+
checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001"
|
|
127
|
+
dependencies = [
|
|
128
|
+
"proc-macro2",
|
|
129
|
+
]
|
|
130
|
+
|
|
131
|
+
[[package]]
|
|
132
|
+
name = "ryu"
|
|
133
|
+
version = "1.0.23"
|
|
134
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
135
|
+
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
|
136
|
+
|
|
137
|
+
[[package]]
|
|
138
|
+
name = "syn"
|
|
139
|
+
version = "2.0.119"
|
|
140
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
141
|
+
checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297"
|
|
142
|
+
dependencies = [
|
|
143
|
+
"proc-macro2",
|
|
144
|
+
"quote",
|
|
145
|
+
"unicode-ident",
|
|
146
|
+
]
|
|
147
|
+
|
|
148
|
+
[[package]]
|
|
149
|
+
name = "target-lexicon"
|
|
150
|
+
version = "0.13.6"
|
|
151
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
152
|
+
checksum = "8c447be3a2abfa84b2d91680c56600bd6b2873866de27b36ec911be705e291da"
|
|
153
|
+
|
|
154
|
+
[[package]]
|
|
155
|
+
name = "unicode-ident"
|
|
156
|
+
version = "1.0.27"
|
|
157
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
158
|
+
checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563"
|
omnibuf-0.1.0/Cargo.toml
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
[package]
|
|
2
|
+
name = "omnibuf-py-ext"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
edition = "2024"
|
|
5
|
+
rust-version = "1.88"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
publish = false
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
|
|
10
|
+
[lib]
|
|
11
|
+
name = "_omnibuf_ext"
|
|
12
|
+
path = "rust/lib.rs"
|
|
13
|
+
crate-type = ["cdylib"]
|
|
14
|
+
|
|
15
|
+
[dependencies]
|
|
16
|
+
omnibuf_core = { package = "omnibuf", version = "0.1.0", default-features = false }
|
|
17
|
+
pyo3 = { version = "0.28", features = ["extension-module"] }
|
|
18
|
+
ryu = "1.0"
|
|
19
|
+
itoa = "1.0"
|
|
20
|
+
|
|
21
|
+
[profile.dev]
|
|
22
|
+
opt-level = 3
|
|
23
|
+
|
|
24
|
+
[profile.release]
|
|
25
|
+
opt-level = 3
|
|
26
|
+
lto = "thin"
|
|
27
|
+
codegen-units = 1
|
omnibuf-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 OmniBuf Contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
omnibuf-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: omnibuf
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Classifier: Development Status :: 4 - Beta
|
|
5
|
+
Classifier: Intended Audience :: Developers
|
|
6
|
+
Classifier: Programming Language :: Python :: 3
|
|
7
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
8
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
11
|
+
Classifier: Typing :: Typed
|
|
12
|
+
Requires-Dist: protobuf-py>=0.7.0
|
|
13
|
+
Requires-Dist: protobuf-py-ext>=0.7.0
|
|
14
|
+
Requires-Dist: protobuf>=6.30.0 ; extra == 'protobuf'
|
|
15
|
+
Provides-Extra: protobuf
|
|
16
|
+
License-File: LICENSE
|
|
17
|
+
Summary: Explicit-typed, header-hop binary wire codec for Python
|
|
18
|
+
Keywords: omnibuf,protobuf,serialization,binary,codec,varint
|
|
19
|
+
Author: OmniBuf Contributors
|
|
20
|
+
License-Expression: MIT
|
|
21
|
+
Requires-Python: >=3.11
|
|
22
|
+
Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
|
|
23
|
+
Project-URL: Homepage, https://omnibuf.com/py
|
|
24
|
+
Project-URL: Issues, https://github.com/omnibuf/omnibuf-py/issues
|
|
25
|
+
Project-URL: Repository, https://github.com/omnibuf/omnibuf-py
|
|
26
|
+
|
|
27
|
+
# omnibuf
|
|
28
|
+
|
|
29
|
+
[](https://github.com/omnibuf/omnibuf-py/actions/workflows/ci.yml)
|
|
30
|
+
|
|
31
|
+
Explicit-typed, header-hop binary wire codec for Python (`v0.1`).
|
|
32
|
+
|
|
33
|
+
`omnibuf` implements the OmniBuf v0.1 wire format for Python 3.11+ backed by a compiled Rust extension. Every package distribution ships strict type annotations (`py.typed`), `buf` CLI code generation, native [`protobuf-py`](https://github.com/bufbuild/protobuf-py) support, `@omnibuf_message` dataclasses, lazy `Opaque` views, and schema-less `Field #0` inspection.
|
|
34
|
+
|
|
35
|
+
## Highlights
|
|
36
|
+
|
|
37
|
+
- **Prefix-Varint Framing**: A single leading byte fixes integer width (`1`, `2`, `4`, or `9` bytes) through trailing-zero count. Reserved markers (`(b & 0x07) == 0` with `b != 0`) fail immediately during decode.
|
|
38
|
+
- **16 Explicit Wire Types**: Distinct 4-bit tags separate signed integers, unsigned integers, booleans, 32-bit and 64-bit floats, UTF-8 strings, raw bytes, sub-messages, typed arrays, typed maps, and extended payloads.
|
|
39
|
+
- **Four Integration Tiers**:
|
|
40
|
+
- `protoc-gen-omnibuf-py` codegen emitting slotted dataclasses, compiled codecs, `oneof` tracking, and lazy `Opaque<Message>` views
|
|
41
|
+
- Direct `marshal` and `unmarshal` over `protobuf-py` (`_pb.py` classes and `DescMessage` descriptors), plus optional `google.protobuf` (`_pb2.py`) support
|
|
42
|
+
- Annotated Python dataclasses via `@omnibuf_message`, `omnibuf_field(...)`, and `enum.IntEnum` or integer-valued `enum.Enum` types
|
|
43
|
+
- Schema-less `decode_blind` and `decode_blind_json` driven by embedded `Field #0` metadata
|
|
44
|
+
- **758/758 Conformance Verified**: Passes the entire cross-language OmniBuf conformance suite with zero skips.
|
|
45
|
+
|
|
46
|
+
## Installation
|
|
47
|
+
|
|
48
|
+
```bash
|
|
49
|
+
uv add omnibuf protobuf-py protobuf-py-ext
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Projects that run `protoc-gen-omnibuf-py` or decode legacy `google.protobuf` `_pb2` classes install the `protobuf` extra alongside `protoc-gen-py`:
|
|
53
|
+
|
|
54
|
+
```bash
|
|
55
|
+
uv add "omnibuf[protobuf]"
|
|
56
|
+
uv add --dev protoc-gen-py
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Code Generation with Buf
|
|
60
|
+
|
|
61
|
+
Add `buf.yaml` and `buf.gen.yaml` at the repository root:
|
|
62
|
+
|
|
63
|
+
```yaml
|
|
64
|
+
# buf.yaml
|
|
65
|
+
version: v2
|
|
66
|
+
modules:
|
|
67
|
+
- path: proto
|
|
68
|
+
lint:
|
|
69
|
+
use:
|
|
70
|
+
- STANDARD
|
|
71
|
+
except:
|
|
72
|
+
- PACKAGE_VERSION_SUFFIX
|
|
73
|
+
breaking:
|
|
74
|
+
use:
|
|
75
|
+
- FILE
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
```yaml
|
|
79
|
+
# buf.gen.yaml
|
|
80
|
+
version: v2
|
|
81
|
+
plugins:
|
|
82
|
+
- local: ["uv", "run", "--no-sync", "protoc-gen-py"]
|
|
83
|
+
out: src/omnibuf/gen
|
|
84
|
+
- local: ["uv", "run", "--no-sync", "protoc-gen-omnibuf-py"]
|
|
85
|
+
out: src/omnibuf/gen
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
Run `buf lint` and `buf generate`. Buf emits `<name>_pb.py` (`protobuf-py`) and `<name>_omnibuf_pb.py` (`omnibuf`) modules in one pass.
|
|
89
|
+
|
|
90
|
+
## Usage
|
|
91
|
+
|
|
92
|
+
### 1. `protobuf-py` Messages
|
|
93
|
+
|
|
94
|
+
`omnibuf.marshal` and `omnibuf.unmarshal` accept `protobuf-py` generated `_pb.py` message classes, instances, and `DescMessage` descriptors directly.
|
|
95
|
+
|
|
96
|
+
```python
|
|
97
|
+
import omnibuf
|
|
98
|
+
from omnibuf.gen.omnibuf.bench import peer_pb
|
|
99
|
+
|
|
100
|
+
msg = peer_pb.PeerUpdate(
|
|
101
|
+
asn=64512,
|
|
102
|
+
router_id="edge-rtr-fra01.core.backbone.net",
|
|
103
|
+
active=True,
|
|
104
|
+
rssi_dbm=-62,
|
|
105
|
+
location=peer_pb.Location(region="eu-central-1", city="Frankfurt", rack=42),
|
|
106
|
+
sequence=9876543210,
|
|
107
|
+
)
|
|
108
|
+
|
|
109
|
+
packet = omnibuf.marshal(msg, deterministic=True)
|
|
110
|
+
decoded = omnibuf.unmarshal(packet, peer_pb.PeerUpdate)
|
|
111
|
+
assert decoded.location.city == "Frankfurt"
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 2. Generated Codecs and Lazy `Opaque` Views
|
|
115
|
+
|
|
116
|
+
`protoc-gen-omnibuf-py` emits slotted dataclasses paired with `Opaque<Message>` views. `Opaque` views hop across length headers in `O(1)` time per field. Nested slices read directly from `bytes` buffers, and single-field edits splice into existing wire bytes.
|
|
117
|
+
|
|
118
|
+
```python
|
|
119
|
+
from omnibuf.gen.omnibuf.bench import peer_omnibuf_pb
|
|
120
|
+
|
|
121
|
+
view = peer_omnibuf_pb.OpaquePeerUpdate(packet)
|
|
122
|
+
loc = view.opaque_location()
|
|
123
|
+
assert loc is not None
|
|
124
|
+
assert loc.get_city() == "Frankfurt"
|
|
125
|
+
|
|
126
|
+
view.set_sequence(9876543211)
|
|
127
|
+
updated_packet = view.marshal()
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
### 3. Annotated Python Dataclasses
|
|
131
|
+
|
|
132
|
+
Pure Python schemas declare wire field numbers with `@omnibuf_message`, `omnibuf_field`, and explicit scalar aliases (`Uint32`, `Uint64`, `Sint32`, `Sint64`, `Fixed32`, `Sfixed32`, `Fixed64`, `Sfixed64`, `Float32`, `Float64`).
|
|
133
|
+
|
|
134
|
+
```python
|
|
135
|
+
import omnibuf
|
|
136
|
+
from omnibuf import Float32, Sint32, Uint32, Uint64, omnibuf_field, omnibuf_message
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
@omnibuf_message("Location")
|
|
140
|
+
class Location:
|
|
141
|
+
region: str = omnibuf_field(1, default="")
|
|
142
|
+
city: str = omnibuf_field(3, default="")
|
|
143
|
+
rack: Uint32 = omnibuf_field(4, default=0)
|
|
144
|
+
latitude: Float32 = omnibuf_field(5, default=0.0)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
@omnibuf_message("PeerSummary")
|
|
148
|
+
class PeerSummary:
|
|
149
|
+
asn: Uint32 = omnibuf_field(1, default=0)
|
|
150
|
+
rssi_dbm: Sint32 = omnibuf_field(5, default=0)
|
|
151
|
+
location: Location = omnibuf_field(9, default_factory=Location)
|
|
152
|
+
sequence: Uint64 = omnibuf_field(11, default=0)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
raw = omnibuf.marshal(
|
|
156
|
+
PeerSummary(asn=64512, rssi_dbm=-62, location=Location(city="Frankfurt"), sequence=7),
|
|
157
|
+
self_describing=True,
|
|
158
|
+
)
|
|
159
|
+
restored = omnibuf.unmarshal(raw, PeerSummary)
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
### 4. Schema-less Inspection via `Field #0`
|
|
163
|
+
|
|
164
|
+
Self-describing packets embed recursive field names inside `Field #0`. `decode_blind` and `decode_blind_json` resolve `Field #0` metadata in two passes, format shortest round-trip `float32` decimals, encode raw bytes as Base64, and preserve full 64-bit integer precision.
|
|
165
|
+
|
|
166
|
+
```python
|
|
167
|
+
import omnibuf
|
|
168
|
+
|
|
169
|
+
payload_dict = omnibuf.decode_blind(raw)
|
|
170
|
+
payload_json = omnibuf.decode_blind_json(raw)
|
|
171
|
+
```
|
|
172
|
+
|
|
173
|
+
## Benchmarks
|
|
174
|
+
|
|
175
|
+
`benchmarks/bench.py` measures `PeerUpdate` across six categories:
|
|
176
|
+
|
|
177
|
+
- Full Message Encode Speed
|
|
178
|
+
- Full Message Decode Speed
|
|
179
|
+
- Read a Single Nested Field
|
|
180
|
+
- Selective Single-Field Mutation
|
|
181
|
+
- Large Embedded Message Encode
|
|
182
|
+
- Embedded Message Growth
|
|
183
|
+
|
|
184
|
+
Methodology: canonical `PeerUpdate` carries 11 fields (`366 B` in OmniBuf, `341 B` in Protobuf). Full decode materializes and reads all 11 fields across every codec. Mutation benchmarks check `location.city == "Frankfurt"` before incrementing `sequence` or growing `location.datacenter` past `16 KiB` ahead of 32 trailing `Neighbor` entries.
|
|
185
|
+
|
|
186
|
+
## Development Workflow
|
|
187
|
+
|
|
188
|
+
Toolchains (`python`, `rust`, `uv`, `buf`, `just`) are pinned in `mise.toml`:
|
|
189
|
+
|
|
190
|
+
```bash
|
|
191
|
+
just install # Build Rust extension and sync .venv
|
|
192
|
+
just proto # Run buf lint and buf generate
|
|
193
|
+
just fmt # Format Rust and Python sources
|
|
194
|
+
just lint # Run buf lint, cargo clippy, ruff check, and strict mypy
|
|
195
|
+
just test # Run pytest suite
|
|
196
|
+
just conformance # Run all 758 cross-language conformance cases
|
|
197
|
+
just bench # Run PeerUpdate benchmarks -> testdata/benchmarks.json
|
|
198
|
+
just ci # Run fmt-check, lint, test, and conformance
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
## License
|
|
202
|
+
|
|
203
|
+
MIT
|
|
204
|
+
|