inferport 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- inferport-0.1.0/.github/workflows/ci.yml +66 -0
- inferport-0.1.0/.github/workflows/publish.yml +72 -0
- inferport-0.1.0/.gitignore +10 -0
- inferport-0.1.0/CHANGELOG.md +15 -0
- inferport-0.1.0/LICENSE +21 -0
- inferport-0.1.0/PKG-INFO +190 -0
- inferport-0.1.0/README.md +174 -0
- inferport-0.1.0/benchmarks/roundtrip.py +190 -0
- inferport-0.1.0/docs/benchmark-results.json +107 -0
- inferport-0.1.0/docs/inferport-design.md +715 -0
- inferport-0.1.0/docs/releasing.md +76 -0
- inferport-0.1.0/docs/roadmap.md +84 -0
- inferport-0.1.0/docs/validation.md +229 -0
- inferport-0.1.0/examples/call_value.py +15 -0
- inferport-0.1.0/examples/serve_value.py +22 -0
- inferport-0.1.0/examples/stateful_counter.py +32 -0
- inferport-0.1.0/examples/thread_bound_backend.py +75 -0
- inferport-0.1.0/pyproject.toml +38 -0
- inferport-0.1.0/src/inferport/__init__.py +20 -0
- inferport-0.1.0/src/inferport/_io.py +59 -0
- inferport-0.1.0/src/inferport/backend.py +20 -0
- inferport-0.1.0/src/inferport/client.py +236 -0
- inferport-0.1.0/src/inferport/codec.py +139 -0
- inferport-0.1.0/src/inferport/errors.py +36 -0
- inferport-0.1.0/src/inferport/protocol.py +72 -0
- inferport-0.1.0/src/inferport/py.typed +0 -0
- inferport-0.1.0/src/inferport/server.py +262 -0
- inferport-0.1.0/tests/__init__.py +0 -0
- inferport-0.1.0/tests/conftest.py +61 -0
- inferport-0.1.0/tests/cross_environment.py +147 -0
- inferport-0.1.0/tests/peers.py +104 -0
- inferport-0.1.0/tests/test_adapter_contracts.py +103 -0
- inferport-0.1.0/tests/test_codec.py +111 -0
- inferport-0.1.0/tests/test_lifecycle.py +311 -0
- inferport-0.1.0/tests/test_packaging.py +84 -0
- inferport-0.1.0/tests/test_protocol.py +40 -0
- inferport-0.1.0/tests/test_transport.py +303 -0
- inferport-0.1.0/uv.lock +719 -0
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
name: InferPort
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
pull_request:
|
|
6
|
+
workflow_call:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
strategy:
|
|
15
|
+
fail-fast: false
|
|
16
|
+
matrix:
|
|
17
|
+
python: ['3.10', '3.11', '3.12', '3.13', '3.14']
|
|
18
|
+
dependencies: [latest]
|
|
19
|
+
include:
|
|
20
|
+
- python: '3.10'
|
|
21
|
+
dependencies: minimum
|
|
22
|
+
- python: '3.11'
|
|
23
|
+
dependencies: numpy123
|
|
24
|
+
- python: '3.12'
|
|
25
|
+
dependencies: numpy126
|
|
26
|
+
steps:
|
|
27
|
+
- uses: actions/checkout@v4
|
|
28
|
+
- uses: astral-sh/setup-uv@v6
|
|
29
|
+
- run: uv sync --python '${{ matrix.python }}' --group dev
|
|
30
|
+
- if: matrix.dependencies == 'minimum'
|
|
31
|
+
run: uv pip install numpy==1.21.3 msgpack==1.1.0 websockets==16.1.1
|
|
32
|
+
- if: matrix.dependencies == 'numpy123'
|
|
33
|
+
run: uv pip install numpy==1.23.5
|
|
34
|
+
- if: matrix.dependencies == 'numpy126'
|
|
35
|
+
run: uv pip install numpy==1.26.4
|
|
36
|
+
- if: matrix.dependencies == 'latest'
|
|
37
|
+
run: uv pip install --upgrade 'numpy>=1.21.3,<3' 'msgpack>=1.1,<2' 'websockets>=16.1.1,<18'
|
|
38
|
+
- run: uv pip check
|
|
39
|
+
- run: uv run --no-sync pytest -q
|
|
40
|
+
- run: uv run --no-sync ruff check .
|
|
41
|
+
- run: uv run --no-sync ruff format --check .
|
|
42
|
+
- run: uv build
|
|
43
|
+
|
|
44
|
+
cross-environment:
|
|
45
|
+
runs-on: ubuntu-latest
|
|
46
|
+
steps:
|
|
47
|
+
- uses: actions/checkout@v4
|
|
48
|
+
- uses: astral-sh/setup-uv@v6
|
|
49
|
+
- run: |
|
|
50
|
+
uv build
|
|
51
|
+
uv venv --python 3.10 .venv-310
|
|
52
|
+
uv venv --python 3.12 .venv-312
|
|
53
|
+
uv pip install --python .venv-310/bin/python dist/*.whl numpy==1.21.3 msgpack==1.1.0 websockets==16.1.1
|
|
54
|
+
uv pip install --python .venv-312/bin/python dist/*.whl 'numpy>=2,<3'
|
|
55
|
+
uv run --no-project --python .venv-312/bin/python tests/cross_environment.py --python-a .venv-310/bin/python --python-b .venv-312/bin/python
|
|
56
|
+
|
|
57
|
+
platform-smoke:
|
|
58
|
+
strategy:
|
|
59
|
+
matrix:
|
|
60
|
+
os: [windows-latest, macos-latest]
|
|
61
|
+
runs-on: ${{ matrix.os }}
|
|
62
|
+
steps:
|
|
63
|
+
- uses: actions/checkout@v4
|
|
64
|
+
- uses: astral-sh/setup-uv@v6
|
|
65
|
+
- run: uv sync --python 3.12 --group dev
|
|
66
|
+
- run: uv run pytest -q tests/test_codec.py tests/test_protocol.py tests/test_lifecycle.py tests/test_adapter_contracts.py tests/test_packaging.py
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
name: Publish to PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
workflow_dispatch:
|
|
5
|
+
release:
|
|
6
|
+
types: [published]
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
concurrency:
|
|
12
|
+
group: pypi-${{ github.event_name }}-${{ github.ref }}
|
|
13
|
+
cancel-in-progress: false
|
|
14
|
+
|
|
15
|
+
jobs:
|
|
16
|
+
verify:
|
|
17
|
+
uses: ./.github/workflows/ci.yml
|
|
18
|
+
|
|
19
|
+
build:
|
|
20
|
+
needs: verify
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
steps:
|
|
23
|
+
- uses: actions/checkout@v4
|
|
24
|
+
with:
|
|
25
|
+
persist-credentials: false
|
|
26
|
+
- uses: astral-sh/setup-uv@v6
|
|
27
|
+
- run: uv sync --locked --python 3.12 --group dev
|
|
28
|
+
- name: Check release tag matches package version
|
|
29
|
+
if: github.event_name == 'release'
|
|
30
|
+
env:
|
|
31
|
+
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
|
32
|
+
run: |
|
|
33
|
+
uv run --no-sync python - <<'PY'
|
|
34
|
+
import os
|
|
35
|
+
import inferport
|
|
36
|
+
|
|
37
|
+
expected = f"v{inferport.__version__}"
|
|
38
|
+
if os.environ["RELEASE_TAG"] != expected:
|
|
39
|
+
raise SystemExit(f"Release tag must be {expected}")
|
|
40
|
+
PY
|
|
41
|
+
- name: Build distributions and validate metadata
|
|
42
|
+
run: |
|
|
43
|
+
uv build --no-sources
|
|
44
|
+
uvx twine check --strict dist/*
|
|
45
|
+
- name: Test the built wheel in a fresh environment
|
|
46
|
+
run: |
|
|
47
|
+
uv venv --python 3.12 .venv-dist
|
|
48
|
+
uv pip install --python .venv-dist/bin/python dist/*.whl 'pytest>=8,<10'
|
|
49
|
+
uv pip check --python .venv-dist/bin/python
|
|
50
|
+
uv run --no-project --python .venv-dist/bin/python -m pytest -q
|
|
51
|
+
- uses: actions/upload-artifact@v5
|
|
52
|
+
with:
|
|
53
|
+
name: python-package-distributions
|
|
54
|
+
path: dist/
|
|
55
|
+
if-no-files-found: error
|
|
56
|
+
|
|
57
|
+
publish:
|
|
58
|
+
if: github.event_name == 'release' && github.repository == 'jeremy775885/InferPort'
|
|
59
|
+
needs: build
|
|
60
|
+
runs-on: ubuntu-latest
|
|
61
|
+
environment:
|
|
62
|
+
name: pypi
|
|
63
|
+
url: https://pypi.org/p/inferport
|
|
64
|
+
permissions:
|
|
65
|
+
id-token: write
|
|
66
|
+
steps:
|
|
67
|
+
- uses: actions/download-artifact@v6
|
|
68
|
+
with:
|
|
69
|
+
name: python-package-distributions
|
|
70
|
+
path: dist/
|
|
71
|
+
- name: Publish using the PyPI trusted publisher
|
|
72
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 0.1.0
|
|
4
|
+
|
|
5
|
+
- Add model-independent `Backend`, `Client`, and `serve` APIs for policy, value,
|
|
6
|
+
reward, and other inference adapters.
|
|
7
|
+
- Exchange binary MessagePack messages and numeric NumPy arrays over WebSocket
|
|
8
|
+
using the `inferport.v1` protocol.
|
|
9
|
+
- Define connection ownership, reset and cleanup behavior, bounded network waits,
|
|
10
|
+
error responses, optional bearer authentication, and TLS support.
|
|
11
|
+
- Support Python 3.10+ and NumPy >=1.21.3,<3 with three runtime dependencies.
|
|
12
|
+
- Include examples, protocol documentation, compatibility tests, and benchmarks.
|
|
13
|
+
|
|
14
|
+
Real model and robot integrations remain in their owning repositories. See
|
|
15
|
+
[validation](docs/validation.md) for the tested scope and remaining gaps.
|
inferport-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 InferPort contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
inferport-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: inferport
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Lightweight inference exchange between model backends and execution environments.
|
|
5
|
+
Project-URL: Homepage, https://github.com/jeremy775885/InferPort
|
|
6
|
+
Project-URL: Documentation, https://github.com/jeremy775885/InferPort/blob/main/README.md
|
|
7
|
+
Project-URL: Issues, https://github.com/jeremy775885/InferPort/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/jeremy775885/InferPort/blob/main/CHANGELOG.md
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Requires-Python: >=3.10
|
|
12
|
+
Requires-Dist: msgpack<2,>=1.1
|
|
13
|
+
Requires-Dist: numpy<3,>=1.21.3
|
|
14
|
+
Requires-Dist: websockets<18,>=16.1.1
|
|
15
|
+
Description-Content-Type: text/markdown
|
|
16
|
+
|
|
17
|
+
# InferPort
|
|
18
|
+
|
|
19
|
+
A small Python library for exchanging inference inputs and results between a model
|
|
20
|
+
repository and a robot, simulator, or evaluation program. Supports policy, value,
|
|
21
|
+
reward, and other backends without importing a model framework or robot SDK.
|
|
22
|
+
|
|
23
|
+
The public API is `Backend`, `serve`, and `Client`. The execution loop belongs to the
|
|
24
|
+
calling application. Transport is WebSocket with binary MessagePack and NumPy arrays.
|
|
25
|
+
One connection owns one backend; batch is simply part of your array shapes.
|
|
26
|
+
|
|
27
|
+
## Install and run
|
|
28
|
+
|
|
29
|
+
Python **3.10+**, NumPy **>=1.21.3,<3**. Runtime dependencies are NumPy, msgpack,
|
|
30
|
+
and websockets. Install a published release from PyPI with:
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install inferport
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
For installation from source and local development:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
# From this repository; uv is a development convenience, not a runtime requirement.
|
|
40
|
+
uv sync --group dev
|
|
41
|
+
uv run examples/serve_value.py
|
|
42
|
+
|
|
43
|
+
# In another terminal, from this repository:
|
|
44
|
+
uv run examples/call_value.py
|
|
45
|
+
# [7. 7. 7. 7.]
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Install into either consuming repository's environment with `uv pip install /path/to/InferPort`.
|
|
49
|
+
The repository is named `InferPort`; the installed distribution and import are `inferport`.
|
|
50
|
+
Existing projects can retain a compatible NumPy version, including `1.21.3` on Python 3.10,
|
|
51
|
+
`1.23.5` on Python 3.11, and `1.26.4` on Python 3.10–3.12. The selected NumPy version must
|
|
52
|
+
also support the environment's Python version; newer Python versions need newer NumPy releases.
|
|
53
|
+
|
|
54
|
+
Model repository:
|
|
55
|
+
|
|
56
|
+
```python
|
|
57
|
+
import numpy as np
|
|
58
|
+
from inferport import Backend, InvalidInput, serve
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class ValueBackend(Backend):
|
|
62
|
+
def infer(self, inputs):
|
|
63
|
+
state = inputs.get("state")
|
|
64
|
+
if not isinstance(state, np.ndarray) or state.ndim != 2:
|
|
65
|
+
raise InvalidInput("expected state with shape [B, D]")
|
|
66
|
+
return {"value": np.sum(state * state, axis=-1)}
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
serve(ValueBackend()) # localhost:8000; blocks until Ctrl+C
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Execution repository:
|
|
73
|
+
|
|
74
|
+
```python
|
|
75
|
+
import numpy as np
|
|
76
|
+
from inferport import Client
|
|
77
|
+
|
|
78
|
+
with Client("ws://127.0.0.1:8000") as client:
|
|
79
|
+
result = client.infer({"state": np.ones((4, 7), dtype=np.float32)})
|
|
80
|
+
print(result["value"])
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
For a robot or simulator, call `infer()` inside your own loop and consume the returned
|
|
84
|
+
actions there. Define image layout, joint order, units, action chunk handling, and
|
|
85
|
+
normalization in your adapters. InferPort does not resize images or control hardware.
|
|
86
|
+
|
|
87
|
+
## State and errors
|
|
88
|
+
|
|
89
|
+
Only `Backend.infer(inputs) -> dict` is required. Stateful backends implement
|
|
90
|
+
`reset(context) -> None` to clear **all** model/processor/cache state and replace the
|
|
91
|
+
context. An empty context must always work. See [the counter example](https://github.com/jeremy775885/InferPort/blob/main/examples/stateful_counter.py).
|
|
92
|
+
`close() -> None` releases backend resources at service shutdown.
|
|
93
|
+
|
|
94
|
+
For engines that must be created and used on the same thread, see
|
|
95
|
+
[the thread-bound model example](https://github.com/jeremy775885/InferPort/blob/main/examples/thread_bound_backend.py). It initializes the
|
|
96
|
+
model on the backend worker before READY; later resets retain the loaded model.
|
|
97
|
+
Allow sufficient `open_timeout` for first-time model loading.
|
|
98
|
+
|
|
99
|
+
The server calls `reset({})` before READY and after disconnect, then `close()` once
|
|
100
|
+
when it exits. All hooks run serially on one worker thread; the Backend constructor
|
|
101
|
+
runs in the caller. `serve()` owns the backend even if startup fails. A second connection
|
|
102
|
+
receives `RemoteError(code="busy")`, including while old work is finishing or cleaning up.
|
|
103
|
+
If initialization or disconnect cleanup fails, the service stops.
|
|
104
|
+
|
|
105
|
+
`Client()` performs no I/O. Use `with` or explicitly call `connect()` and `close()`.
|
|
106
|
+
Client calls must be sequential; concurrent calls raise `RuntimeError`. Cross-thread
|
|
107
|
+
`close()` interrupts network waits. Closing is idempotent. Create a new Client after
|
|
108
|
+
closing or a fatal failure; there is no reconnect, retry, or automatic request replay.
|
|
109
|
+
|
|
110
|
+
- Local invalid input types raise `TypeError`/`ValueError` before sending; the connection stays usable.
|
|
111
|
+
- A backend raises `InvalidInput` **before changing state** for an expected input problem.
|
|
112
|
+
The caller gets nonfatal `RemoteError(code="invalid_input")` and may continue.
|
|
113
|
+
Its message is public validation guidance (bounded to 512 characters).
|
|
114
|
+
- Unexpected backend exceptions and unsupported outputs produce fatal `RemoteError`.
|
|
115
|
+
These unexpected errors use generic client messages; server tracebacks are logged locally.
|
|
116
|
+
- `ProtocolError` rejects malformed messages or mismatched responses.
|
|
117
|
+
- `TransportError` covers connection failures; `RequestTimeout` is its subclass and
|
|
118
|
+
provides `.stage` (`open`, `ready`, `send`, or `response`).
|
|
119
|
+
- All library exceptions inherit `Error`. `RemoteError` exposes `code`, `message`,
|
|
120
|
+
`request_id`, and `fatal`.
|
|
121
|
+
|
|
122
|
+
## Deadlines and limits
|
|
123
|
+
|
|
124
|
+
`Client(uri, timeout=30, open_timeout=10, max_message_bytes=64*1024*1024)`:
|
|
125
|
+
|
|
126
|
+
- `open_timeout` covers connection, handshake, and READY together.
|
|
127
|
+
- `infer(data, timeout=...)` / `reset(context, timeout=...)` cover send and receive,
|
|
128
|
+
including model execution. Local encoding and decoding are outside this deadline.
|
|
129
|
+
- Timeout makes the Client unusable. Network cleanup has a separate five-second budget.
|
|
130
|
+
A timeout cannot cancel an already running backend operation. Its result is dropped;
|
|
131
|
+
ownership remains held until that work and cleanup finish.
|
|
132
|
+
- A permanently stuck backend requires external process termination. InferPort cannot
|
|
133
|
+
safely kill GPU/Python worker operations or promise hard realtime behavior.
|
|
134
|
+
|
|
135
|
+
Payloads are string-keyed dictionaries containing nested dictionaries/lists, basic
|
|
136
|
+
scalars, bytes, and real numeric/bool NumPy arrays. Tuples become lists; NumPy scalars
|
|
137
|
+
become Python scalars. Arrays preserve values, shape, and type width and arrive as
|
|
138
|
+
writable, C-contiguous, native-endian arrays. Object, structured, complex, text,
|
|
139
|
+
datetime, and custom array dtypes are rejected. Convert Torch/JAX tensors explicitly.
|
|
140
|
+
|
|
141
|
+
The default **64 MiB complete message** limit applies on both sides, including the
|
|
142
|
+
envelope. Configure the same limit on Client and serve (minimum 128 bytes). Arrays
|
|
143
|
+
have at most 32 dimensions; messages at most 32 nesting levels and 100,000 nodes.
|
|
144
|
+
This is not a total process memory limit: serialization, buffers, and writable arrays
|
|
145
|
+
require additional memory. No automatic chunking or compression is performed.
|
|
146
|
+
|
|
147
|
+
## Deployment
|
|
148
|
+
|
|
149
|
+
`serve(backend, host="127.0.0.1", port=8000, token=None, ssl=None,
|
|
150
|
+
max_message_bytes=64*1024*1024, send_timeout=30, stop_event=None)`.
|
|
151
|
+
Use `threading.Event` for an embedded service's stop request; Ctrl+C works in a script.
|
|
152
|
+
`send_timeout` bounds network writes, not backend execution.
|
|
153
|
+
|
|
154
|
+
To listen across machines, explicitly select the listening address. Both sides accept
|
|
155
|
+
`token="..."` for bearer authentication and an `ssl.SSLContext` for TLS. The server
|
|
156
|
+
context must load its certificate; `wss://` clients verify certificates and hostnames
|
|
157
|
+
by default. Use a client context for a private CA. Tokens must be nonempty printable
|
|
158
|
+
ASCII without whitespace; keep them out of URLs. Use TLS or a trusted encrypted tunnel
|
|
159
|
+
across untrusted networks; a token does not encrypt `ws://` traffic.
|
|
160
|
+
|
|
161
|
+
The only endpoint is `/`, with required subprotocol `inferport.v1`. Browser Origin
|
|
162
|
+
requests are rejected. Compression and system proxy discovery are disabled. Heartbeats
|
|
163
|
+
check connection liveness, not model progress. Standard Python logging provides request
|
|
164
|
+
IDs, operations, durations, and errors without automatically logging payloads.
|
|
165
|
+
|
|
166
|
+
## Development and scope
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
uv run pytest -q
|
|
170
|
+
uv run ruff check .
|
|
171
|
+
uv run ruff format --check .
|
|
172
|
+
uv build
|
|
173
|
+
uv run benchmarks/roundtrip.py --iterations 100
|
|
174
|
+
```
|
|
175
|
+
|
|
176
|
+
See [the full protocol and design](https://github.com/jeremy775885/InferPort/blob/main/docs/inferport-design.md),
|
|
177
|
+
[verification results and remaining gaps](https://github.com/jeremy775885/InferPort/blob/main/docs/validation.md), and
|
|
178
|
+
[the maturity assessment and roadmap](https://github.com/jeremy775885/InferPort/blob/main/docs/roadmap.md).
|
|
179
|
+
Maintainers can follow [the release guide](https://github.com/jeremy775885/InferPort/blob/main/docs/releasing.md); changes are recorded in
|
|
180
|
+
[the changelog](https://github.com/jeremy775885/InferPort/blob/main/CHANGELOG.md).
|
|
181
|
+
CI covers Python 3.10–3.14, minimum dependencies (NumPy 1.21.3), NumPy 1.23.5 / 1.26.4,
|
|
182
|
+
and newer combinations.
|
|
183
|
+
It includes Windows/macOS smoke jobs; completed runs and their scope are recorded in
|
|
184
|
+
[the validation record](https://github.com/jeremy775885/InferPort/blob/main/docs/validation.md#github-actions).
|
|
185
|
+
|
|
186
|
+
InferPort replaces `policy_runtime` with no compatibility alias or TCP/JSON fallback.
|
|
187
|
+
This repository doesn't implement RTC, robot/environment base classes, action scheduling,
|
|
188
|
+
automatic batching, multiple sessions/models, or a schema/description framework.
|
|
189
|
+
Real model and robot integrations remain in their owning repositories and require
|
|
190
|
+
separate validation.
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
# InferPort
|
|
2
|
+
|
|
3
|
+
A small Python library for exchanging inference inputs and results between a model
|
|
4
|
+
repository and a robot, simulator, or evaluation program. Supports policy, value,
|
|
5
|
+
reward, and other backends without importing a model framework or robot SDK.
|
|
6
|
+
|
|
7
|
+
The public API is `Backend`, `serve`, and `Client`. The execution loop belongs to the
|
|
8
|
+
calling application. Transport is WebSocket with binary MessagePack and NumPy arrays.
|
|
9
|
+
One connection owns one backend; batch is simply part of your array shapes.
|
|
10
|
+
|
|
11
|
+
## Install and run
|
|
12
|
+
|
|
13
|
+
Python **3.10+**, NumPy **>=1.21.3,<3**. Runtime dependencies are NumPy, msgpack,
|
|
14
|
+
and websockets. Install a published release from PyPI with:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
pip install inferport
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
For installation from source and local development:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
# From this repository; uv is a development convenience, not a runtime requirement.
|
|
24
|
+
uv sync --group dev
|
|
25
|
+
uv run examples/serve_value.py
|
|
26
|
+
|
|
27
|
+
# In another terminal, from this repository:
|
|
28
|
+
uv run examples/call_value.py
|
|
29
|
+
# [7. 7. 7. 7.]
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
Install into either consuming repository's environment with `uv pip install /path/to/InferPort`.
|
|
33
|
+
The repository is named `InferPort`; the installed distribution and import are `inferport`.
|
|
34
|
+
Existing projects can retain a compatible NumPy version, including `1.21.3` on Python 3.10,
|
|
35
|
+
`1.23.5` on Python 3.11, and `1.26.4` on Python 3.10–3.12. The selected NumPy version must
|
|
36
|
+
also support the environment's Python version; newer Python versions need newer NumPy releases.
|
|
37
|
+
|
|
38
|
+
Model repository:
|
|
39
|
+
|
|
40
|
+
```python
|
|
41
|
+
import numpy as np
|
|
42
|
+
from inferport import Backend, InvalidInput, serve
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
class ValueBackend(Backend):
|
|
46
|
+
def infer(self, inputs):
|
|
47
|
+
state = inputs.get("state")
|
|
48
|
+
if not isinstance(state, np.ndarray) or state.ndim != 2:
|
|
49
|
+
raise InvalidInput("expected state with shape [B, D]")
|
|
50
|
+
return {"value": np.sum(state * state, axis=-1)}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
serve(ValueBackend()) # localhost:8000; blocks until Ctrl+C
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Execution repository:
|
|
57
|
+
|
|
58
|
+
```python
|
|
59
|
+
import numpy as np
|
|
60
|
+
from inferport import Client
|
|
61
|
+
|
|
62
|
+
with Client("ws://127.0.0.1:8000") as client:
|
|
63
|
+
result = client.infer({"state": np.ones((4, 7), dtype=np.float32)})
|
|
64
|
+
print(result["value"])
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
For a robot or simulator, call `infer()` inside your own loop and consume the returned
|
|
68
|
+
actions there. Define image layout, joint order, units, action chunk handling, and
|
|
69
|
+
normalization in your adapters. InferPort does not resize images or control hardware.
|
|
70
|
+
|
|
71
|
+
## State and errors
|
|
72
|
+
|
|
73
|
+
Only `Backend.infer(inputs) -> dict` is required. Stateful backends implement
|
|
74
|
+
`reset(context) -> None` to clear **all** model/processor/cache state and replace the
|
|
75
|
+
context. An empty context must always work. See [the counter example](https://github.com/jeremy775885/InferPort/blob/main/examples/stateful_counter.py).
|
|
76
|
+
`close() -> None` releases backend resources at service shutdown.
|
|
77
|
+
|
|
78
|
+
For engines that must be created and used on the same thread, see
|
|
79
|
+
[the thread-bound model example](https://github.com/jeremy775885/InferPort/blob/main/examples/thread_bound_backend.py). It initializes the
|
|
80
|
+
model on the backend worker before READY; later resets retain the loaded model.
|
|
81
|
+
Allow sufficient `open_timeout` for first-time model loading.
|
|
82
|
+
|
|
83
|
+
The server calls `reset({})` before READY and after disconnect, then `close()` once
|
|
84
|
+
when it exits. All hooks run serially on one worker thread; the Backend constructor
|
|
85
|
+
runs in the caller. `serve()` owns the backend even if startup fails. A second connection
|
|
86
|
+
receives `RemoteError(code="busy")`, including while old work is finishing or cleaning up.
|
|
87
|
+
If initialization or disconnect cleanup fails, the service stops.
|
|
88
|
+
|
|
89
|
+
`Client()` performs no I/O. Use `with` or explicitly call `connect()` and `close()`.
|
|
90
|
+
Client calls must be sequential; concurrent calls raise `RuntimeError`. Cross-thread
|
|
91
|
+
`close()` interrupts network waits. Closing is idempotent. Create a new Client after
|
|
92
|
+
closing or a fatal failure; there is no reconnect, retry, or automatic request replay.
|
|
93
|
+
|
|
94
|
+
- Local invalid input types raise `TypeError`/`ValueError` before sending; the connection stays usable.
|
|
95
|
+
- A backend raises `InvalidInput` **before changing state** for an expected input problem.
|
|
96
|
+
The caller gets nonfatal `RemoteError(code="invalid_input")` and may continue.
|
|
97
|
+
Its message is public validation guidance (bounded to 512 characters).
|
|
98
|
+
- Unexpected backend exceptions and unsupported outputs produce fatal `RemoteError`.
|
|
99
|
+
These unexpected errors use generic client messages; server tracebacks are logged locally.
|
|
100
|
+
- `ProtocolError` rejects malformed messages or mismatched responses.
|
|
101
|
+
- `TransportError` covers connection failures; `RequestTimeout` is its subclass and
|
|
102
|
+
provides `.stage` (`open`, `ready`, `send`, or `response`).
|
|
103
|
+
- All library exceptions inherit `Error`. `RemoteError` exposes `code`, `message`,
|
|
104
|
+
`request_id`, and `fatal`.
|
|
105
|
+
|
|
106
|
+
## Deadlines and limits
|
|
107
|
+
|
|
108
|
+
`Client(uri, timeout=30, open_timeout=10, max_message_bytes=64*1024*1024)`:
|
|
109
|
+
|
|
110
|
+
- `open_timeout` covers connection, handshake, and READY together.
|
|
111
|
+
- `infer(data, timeout=...)` / `reset(context, timeout=...)` cover send and receive,
|
|
112
|
+
including model execution. Local encoding and decoding are outside this deadline.
|
|
113
|
+
- Timeout makes the Client unusable. Network cleanup has a separate five-second budget.
|
|
114
|
+
A timeout cannot cancel an already running backend operation. Its result is dropped;
|
|
115
|
+
ownership remains held until that work and cleanup finish.
|
|
116
|
+
- A permanently stuck backend requires external process termination. InferPort cannot
|
|
117
|
+
safely kill GPU/Python worker operations or promise hard realtime behavior.
|
|
118
|
+
|
|
119
|
+
Payloads are string-keyed dictionaries containing nested dictionaries/lists, basic
|
|
120
|
+
scalars, bytes, and real numeric/bool NumPy arrays. Tuples become lists; NumPy scalars
|
|
121
|
+
become Python scalars. Arrays preserve values, shape, and type width and arrive as
|
|
122
|
+
writable, C-contiguous, native-endian arrays. Object, structured, complex, text,
|
|
123
|
+
datetime, and custom array dtypes are rejected. Convert Torch/JAX tensors explicitly.
|
|
124
|
+
|
|
125
|
+
The default **64 MiB complete message** limit applies on both sides, including the
|
|
126
|
+
envelope. Configure the same limit on Client and serve (minimum 128 bytes). Arrays
|
|
127
|
+
have at most 32 dimensions; messages at most 32 nesting levels and 100,000 nodes.
|
|
128
|
+
This is not a total process memory limit: serialization, buffers, and writable arrays
|
|
129
|
+
require additional memory. No automatic chunking or compression is performed.
|
|
130
|
+
|
|
131
|
+
## Deployment
|
|
132
|
+
|
|
133
|
+
`serve(backend, host="127.0.0.1", port=8000, token=None, ssl=None,
|
|
134
|
+
max_message_bytes=64*1024*1024, send_timeout=30, stop_event=None)`.
|
|
135
|
+
Use `threading.Event` for an embedded service's stop request; Ctrl+C works in a script.
|
|
136
|
+
`send_timeout` bounds network writes, not backend execution.
|
|
137
|
+
|
|
138
|
+
To listen across machines, explicitly select the listening address. Both sides accept
|
|
139
|
+
`token="..."` for bearer authentication and an `ssl.SSLContext` for TLS. The server
|
|
140
|
+
context must load its certificate; `wss://` clients verify certificates and hostnames
|
|
141
|
+
by default. Use a client context for a private CA. Tokens must be nonempty printable
|
|
142
|
+
ASCII without whitespace; keep them out of URLs. Use TLS or a trusted encrypted tunnel
|
|
143
|
+
across untrusted networks; a token does not encrypt `ws://` traffic.
|
|
144
|
+
|
|
145
|
+
The only endpoint is `/`, with required subprotocol `inferport.v1`. Browser Origin
|
|
146
|
+
requests are rejected. Compression and system proxy discovery are disabled. Heartbeats
|
|
147
|
+
check connection liveness, not model progress. Standard Python logging provides request
|
|
148
|
+
IDs, operations, durations, and errors without automatically logging payloads.
|
|
149
|
+
|
|
150
|
+
## Development and scope
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
uv run pytest -q
|
|
154
|
+
uv run ruff check .
|
|
155
|
+
uv run ruff format --check .
|
|
156
|
+
uv build
|
|
157
|
+
uv run benchmarks/roundtrip.py --iterations 100
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
See [the full protocol and design](https://github.com/jeremy775885/InferPort/blob/main/docs/inferport-design.md),
|
|
161
|
+
[verification results and remaining gaps](https://github.com/jeremy775885/InferPort/blob/main/docs/validation.md), and
|
|
162
|
+
[the maturity assessment and roadmap](https://github.com/jeremy775885/InferPort/blob/main/docs/roadmap.md).
|
|
163
|
+
Maintainers can follow [the release guide](https://github.com/jeremy775885/InferPort/blob/main/docs/releasing.md); changes are recorded in
|
|
164
|
+
[the changelog](https://github.com/jeremy775885/InferPort/blob/main/CHANGELOG.md).
|
|
165
|
+
CI covers Python 3.10–3.14, minimum dependencies (NumPy 1.21.3), NumPy 1.23.5 / 1.26.4,
|
|
166
|
+
and newer combinations.
|
|
167
|
+
It includes Windows/macOS smoke jobs; completed runs and their scope are recorded in
|
|
168
|
+
[the validation record](https://github.com/jeremy775885/InferPort/blob/main/docs/validation.md#github-actions).
|
|
169
|
+
|
|
170
|
+
InferPort replaces `policy_runtime` with no compatibility alias or TCP/JSON fallback.
|
|
171
|
+
This repository doesn't implement RTC, robot/environment base classes, action scheduling,
|
|
172
|
+
automatic batching, multiple sessions/models, or a schema/description framework.
|
|
173
|
+
Real model and robot integrations remain in their owning repositories and require
|
|
174
|
+
separate validation.
|