inspect-robots-agent 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,221 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ # .vscode/
203
+ # Temporary file for partial code execution
204
+ tempCodeRunnerFile.py
205
+
206
+ # Ruff stuff:
207
+ .ruff_cache/
208
+
209
+ # PyPI configuration file
210
+ .pypirc
211
+
212
+ # Marimo
213
+ marimo/_static/
214
+ marimo/_lsp/
215
+ __marimo__/
216
+
217
+ # Streamlit
218
+ .streamlit/secrets.toml
219
+
220
+ # uv.lock is committed: CI installs with `uv sync --locked` for reproducible
221
+ # builds. Run `uv lock` after changing dependencies in pyproject.toml.
@@ -0,0 +1,73 @@
1
+ Metadata-Version: 2.4
2
+ Name: inspect-robots-agent
3
+ Version: 0.1.0
4
+ Summary: LLM agent policy for Inspect Robots: frontier LLMs (Claude, GPT, anything OpenAI-compatible) drive any registered embodiment through tool calls.
5
+ Project-URL: Homepage, https://github.com/robocurve/inspect-robots/tree/main/plugins/inspect-robots-agent
6
+ Project-URL: Repository, https://github.com/robocurve/inspect-robots
7
+ Author: Inspect Robots contributors
8
+ License-Expression: MIT
9
+ Keywords: agent,evaluation,inspect_robots,llm,robotics,vla
10
+ Classifier: Development Status :: 3 - Alpha
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: License :: OSI Approved :: MIT License
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
15
+ Classifier: Typing :: Typed
16
+ Requires-Python: >=3.10
17
+ Requires-Dist: httpx>=0.27
18
+ Requires-Dist: inspect-robots>=0.4
19
+ Requires-Dist: numpy>=1.24
20
+ Provides-Extra: dev
21
+ Requires-Dist: mypy>=1.11; extra == 'dev'
22
+ Requires-Dist: pytest-cov>=5.0; extra == 'dev'
23
+ Requires-Dist: pytest>=8.0; extra == 'dev'
24
+ Requires-Dist: ruff>=0.6; extra == 'dev'
25
+ Description-Content-Type: text/markdown
26
+
27
+ # inspect-robots-agent
28
+
29
+ LLM agent policy for [Inspect Robots](https://github.com/robocurve/inspect-robots):
30
+ frontier LLMs (Claude, GPT, anything behind an OpenAI-compatible API) drive any
31
+ registered embodiment through tool calls, as a first-class `Policy` named
32
+ `agent`. The same policy runs ad-hoc instructions and scores on registered
33
+ tasks next to fine-tuned VLAs.
34
+
35
+ ## Install
36
+
37
+ ```bash
38
+ pip install inspect-robots inspect-robots-agent
39
+ ```
40
+
41
+ ## Quickstart (no hardware)
42
+
43
+ ```bash
44
+ export ANTHROPIC_API_KEY=sk-ant-...
45
+
46
+ inspect-robots "pick up the cube" --policy agent \
47
+ -P model=anthropic/claude-fable-5 --embodiment cubepick
48
+ ```
49
+
50
+ Model strings are OpenRouter-style `provider/model`, resolved from
51
+ `-P model=...` or `$INSPECT_ROBOTS_MODEL`. API keys come from the environment:
52
+
53
+ 1. `-P base_url=...` (with `-P api_key_env=NAME`): any OpenAI-compatible endpoint
54
+ 2. `anthropic/*` model with `ANTHROPIC_API_KEY`: the Anthropic compat endpoint
55
+ 3. `openai/*` model with `OPENAI_API_KEY`: OpenAI
56
+ 4. `OPENROUTER_API_KEY`: OpenRouter, any model string
57
+
58
+ ## How it works
59
+
60
+ Each LLM tool call becomes one smooth, open-loop action chunk: named partial
61
+ joint targets are interpolated at the embodiment's control rate
62
+ (`move_joints`), displacements are split across steps (`move_by`), and
63
+ `done`/`give_up` end the trial through the core's policy-stop channel. Every
64
+ action still passes the CLI's default safety approvers (bounds clamp plus
65
+ per-step delta limit); the plugin contains no safety-critical code path of
66
+ its own.
67
+
68
+ > **Warning:**
69
+ > Guardrails are on by default at the CLI. **Never pass `--disable-guardrails`
70
+ > on real hardware** unless you fully trust the policy and the rig.
71
+
72
+ Configuration knobs (all `-P key=value`): `model`, `base_url`, `api_key_env`,
73
+ `max_llm_calls`, `temperature`.
@@ -0,0 +1,47 @@
1
+ # inspect-robots-agent
2
+
3
+ LLM agent policy for [Inspect Robots](https://github.com/robocurve/inspect-robots):
4
+ frontier LLMs (Claude, GPT, anything behind an OpenAI-compatible API) drive any
5
+ registered embodiment through tool calls, as a first-class `Policy` named
6
+ `agent`. The same policy runs ad-hoc instructions and scores on registered
7
+ tasks next to fine-tuned VLAs.
8
+
9
+ ## Install
10
+
11
+ ```bash
12
+ pip install inspect-robots inspect-robots-agent
13
+ ```
14
+
15
+ ## Quickstart (no hardware)
16
+
17
+ ```bash
18
+ export ANTHROPIC_API_KEY=sk-ant-...
19
+
20
+ inspect-robots "pick up the cube" --policy agent \
21
+ -P model=anthropic/claude-fable-5 --embodiment cubepick
22
+ ```
23
+
24
+ Model strings are OpenRouter-style `provider/model`, resolved from
25
+ `-P model=...` or `$INSPECT_ROBOTS_MODEL`. API keys come from the environment:
26
+
27
+ 1. `-P base_url=...` (with `-P api_key_env=NAME`): any OpenAI-compatible endpoint
28
+ 2. `anthropic/*` model with `ANTHROPIC_API_KEY`: the Anthropic compat endpoint
29
+ 3. `openai/*` model with `OPENAI_API_KEY`: OpenAI
30
+ 4. `OPENROUTER_API_KEY`: OpenRouter, any model string
31
+
32
+ ## How it works
33
+
34
+ Each LLM tool call becomes one smooth, open-loop action chunk: named partial
35
+ joint targets are interpolated at the embodiment's control rate
36
+ (`move_joints`), displacements are split across steps (`move_by`), and
37
+ `done`/`give_up` end the trial through the core's policy-stop channel. Every
38
+ action still passes the CLI's default safety approvers (bounds clamp plus
39
+ per-step delta limit); the plugin contains no safety-critical code path of
40
+ its own.
41
+
42
+ > [!WARNING]
43
+ > Guardrails are on by default at the CLI. **Never pass `--disable-guardrails`
44
+ > on real hardware** unless you fully trust the policy and the rig.
45
+
46
+ Configuration knobs (all `-P key=value`): `model`, `base_url`, `api_key_env`,
47
+ `max_llm_calls`, `temperature`.
@@ -0,0 +1,90 @@
1
+ [build-system]
2
+ requires = ["hatchling>=1.18", "hatch-fancy-pypi-readme>=24.1"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "inspect-robots-agent"
7
+ version = "0.1.0"
8
+ description = "LLM agent policy for Inspect Robots: frontier LLMs (Claude, GPT, anything OpenAI-compatible) drive any registered embodiment through tool calls."
9
+ dynamic = ["readme"]
10
+ requires-python = ">=3.10"
11
+ license = "MIT"
12
+ authors = [{ name = "Inspect Robots contributors" }]
13
+ keywords = ["robotics", "llm", "agent", "vla", "inspect_robots", "evaluation"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Intended Audience :: Science/Research",
17
+ "License :: OSI Approved :: MIT License",
18
+ "Programming Language :: Python :: 3",
19
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
20
+ "Typing :: Typed",
21
+ ]
22
+ # NOTE: no provider SDKs. The adapter speaks the OpenAI chat-completions wire
23
+ # format over one httpx client, which covers OpenRouter, OpenAI, local
24
+ # vLLM/Ollama, and Anthropic's OpenAI-compat endpoint (plan 0008 §4a) — the
25
+ # same "speak the protocol, don't import the package" doctrine as the
26
+ # xpolicylab plugin.
27
+ dependencies = [
28
+ "inspect-robots>=0.4",
29
+ "numpy>=1.24",
30
+ "httpx>=0.27",
31
+ ]
32
+
33
+ [project.optional-dependencies]
34
+ dev = ["pytest>=8.0", "pytest-cov>=5.0", "mypy>=1.11", "ruff>=0.6"]
35
+
36
+ # Entry-point discovery: an installed inspect-robots-agent appears in
37
+ # `inspect-robots list policies` without being imported first. The registry
38
+ # calls this factory by the name `agent`.
39
+ [project.entry-points."inspect_robots.policies"]
40
+ agent = "inspect_robots_agent.policy:agent_policy"
41
+
42
+ [project.urls]
43
+ Homepage = "https://github.com/robocurve/inspect-robots/tree/main/plugins/inspect-robots-agent"
44
+ Repository = "https://github.com/robocurve/inspect-robots"
45
+
46
+ # In the Inspect Robots monorepo this resolves the `inspect_robots` dependency
47
+ # to the in-repo core package instead of PyPI. Harmless in a standalone
48
+ # checkout (uv ignores a workspace source when there is no workspace root).
49
+ [tool.uv.sources]
50
+ inspect-robots = { workspace = true }
51
+
52
+ [tool.hatch.build.targets.wheel]
53
+ packages = ["src/inspect_robots_agent"]
54
+
55
+ [tool.hatch.build.targets.sdist]
56
+ include = ["src/inspect_robots_agent", "tests", "README.md"]
57
+
58
+ [tool.ruff]
59
+ line-length = 100
60
+
61
+ [tool.mypy]
62
+ strict = true
63
+
64
+ # PyPI readme: identical to README.md except GitHub-only alert syntax
65
+ # (e.g. `> [!NOTE]`) is rewritten to bold blockquotes, which PyPI renders.
66
+ [tool.hatch.metadata.hooks.fancy-pypi-readme]
67
+ content-type = "text/markdown"
68
+
69
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.fragments]]
70
+ path = "README.md"
71
+
72
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
73
+ pattern = '(?m)^> \[!NOTE\][ \t]*$'
74
+ replacement = '> **Note:**'
75
+
76
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
77
+ pattern = '(?m)^> \[!TIP\][ \t]*$'
78
+ replacement = '> **Tip:**'
79
+
80
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
81
+ pattern = '(?m)^> \[!IMPORTANT\][ \t]*$'
82
+ replacement = '> **Important:**'
83
+
84
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
85
+ pattern = '(?m)^> \[!WARNING\][ \t]*$'
86
+ replacement = '> **Warning:**'
87
+
88
+ [[tool.hatch.metadata.hooks.fancy-pypi-readme.substitutions]]
89
+ pattern = '(?m)^> \[!CAUTION\][ \t]*$'
90
+ replacement = '> **Caution:**'
@@ -0,0 +1,28 @@
1
+ """inspect-robots-agent — frontier LLMs as first-class Inspect Robots policies.
2
+
3
+ An LLM behind any OpenAI-compatible API (OpenRouter, OpenAI, local
4
+ vLLM/Ollama, Anthropic's compat endpoint) drives whatever embodiment it is
5
+ paired with: each tool call becomes one smooth, approver-checked action
6
+ chunk. Registered as the policy ``agent``::
7
+
8
+ inspect-robots "pick up the cube" --policy agent \
9
+ -P model=anthropic/claude-fable-5 --embodiment cubepick
10
+
11
+ or programmatically::
12
+
13
+ from inspect_robots import eval
14
+ from inspect_robots_agent import LLMAgentPolicy
15
+
16
+ eval("my-task", LLMAgentPolicy(model="openai/gpt-5.2"), "cubepick")
17
+
18
+ The policy is discovered via the ``inspect_robots.policies`` entry point, so
19
+ it shows up in ``inspect-robots list policies`` without being imported first.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from inspect_robots_agent.policy import AgentPolicyConfig, LLMAgentPolicy, agent_policy
25
+
26
+ __all__ = ["AgentPolicyConfig", "LLMAgentPolicy", "agent_policy"]
27
+
28
+ __version__ = "0.1.0"
@@ -0,0 +1,186 @@
1
+ """OpenAI-compatible chat client + provider resolution (plan 0008 §4a).
2
+
3
+ No provider SDKs: one ``httpx`` client speaking the chat-completions wire
4
+ format covers OpenRouter, OpenAI, local vLLM/Ollama, and Anthropic's
5
+ OpenAI-compat endpoint — the same "speak the protocol, don't import the
6
+ package" doctrine as the xpolicylab plugin.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import time
12
+ from collections.abc import Mapping
13
+ from dataclasses import dataclass
14
+ from typing import Any
15
+
16
+ import httpx
17
+
18
+ from inspect_robots.errors import ConfigError
19
+
20
+ ENV_MODEL = "INSPECT_ROBOTS_MODEL"
21
+
22
+ _ANTHROPIC_BASE = "https://api.anthropic.com/v1"
23
+ _OPENAI_BASE = "https://api.openai.com/v1"
24
+ _OPENROUTER_BASE = "https://openrouter.ai/api/v1"
25
+ _OPENROUTER_KEY = "OPENROUTER_API_KEY"
26
+
27
+
28
+ @dataclass(frozen=True)
29
+ class Provider:
30
+ """A resolved OpenAI-compatible endpoint: where, with which key, which model."""
31
+
32
+ base_url: str
33
+ api_key: str
34
+ model: str
35
+
36
+
37
+ def resolve_provider(
38
+ model: str | None,
39
+ base_url: str | None,
40
+ api_key_env: str | None,
41
+ env: Mapping[str, str],
42
+ ) -> Provider:
43
+ """The key/base-url ladder (plan 0008 §4a); first match wins.
44
+
45
+ 1. Explicit ``base_url`` — any OpenAI-compatible endpoint; the key comes
46
+ from ``api_key_env`` (default ``OPENROUTER_API_KEY``), and a missing
47
+ key is allowed (local vLLM/Ollama endpoints are typically keyless).
48
+ 2. ``anthropic/*`` model + ``ANTHROPIC_API_KEY`` — the Anthropic compat
49
+ endpoint (provider prefix stripped from the model id).
50
+ 3. ``openai/*`` model + ``OPENAI_API_KEY`` — OpenAI (prefix stripped).
51
+ 4. ``OPENROUTER_API_KEY`` — OpenRouter, which takes the full
52
+ ``provider/model`` string.
53
+
54
+ Anything else is a guided [`ConfigError`][inspect_robots.errors.ConfigError]
55
+ naming the fixes — never a traceback at the user.
56
+ """
57
+ if not model:
58
+ raise ConfigError(
59
+ "no model configured for the agent policy.\n"
60
+ f"fix: pass -P model=provider/model (e.g. anthropic/claude-fable-5) "
61
+ f"or set ${ENV_MODEL}"
62
+ )
63
+ if base_url:
64
+ key_env = api_key_env or _OPENROUTER_KEY
65
+ return Provider(base_url=base_url.rstrip("/"), api_key=env.get(key_env, ""), model=model)
66
+ provider_prefix, _, bare_model = model.partition("/")
67
+ if provider_prefix == "anthropic" and (key := env.get("ANTHROPIC_API_KEY")):
68
+ return Provider(base_url=_ANTHROPIC_BASE, api_key=key, model=bare_model)
69
+ if provider_prefix == "openai" and (key := env.get("OPENAI_API_KEY")):
70
+ return Provider(base_url=_OPENAI_BASE, api_key=key, model=bare_model)
71
+ if key := env.get(_OPENROUTER_KEY):
72
+ return Provider(base_url=_OPENROUTER_BASE, api_key=key, model=model)
73
+ raise ConfigError(
74
+ f"no API key found for model {model!r}.\n"
75
+ f"fix: set ${_OPENROUTER_KEY} (works for any model), or the provider's "
76
+ "key ($ANTHROPIC_API_KEY for anthropic/*, $OPENAI_API_KEY for openai/*), "
77
+ "or pass -P base_url=... (+ -P api_key_env=NAME) for a custom endpoint"
78
+ )
79
+
80
+
81
+ @dataclass(frozen=True)
82
+ class ToolCall:
83
+ """One tool invocation the model asked for; ``arguments`` is raw JSON text."""
84
+
85
+ id: str
86
+ name: str
87
+ arguments: str
88
+
89
+
90
+ @dataclass(frozen=True)
91
+ class AssistantMessage:
92
+ """The parsed ``choices[0].message`` of a chat completion."""
93
+
94
+ content: str | None
95
+ tool_calls: tuple[ToolCall, ...]
96
+
97
+ def raw(self) -> dict[str, Any]:
98
+ """The wire-format dict to append back onto the conversation."""
99
+ message: dict[str, Any] = {"role": "assistant", "content": self.content}
100
+ if self.tool_calls:
101
+ message["tool_calls"] = [
102
+ {
103
+ "id": c.id,
104
+ "type": "function",
105
+ "function": {"name": c.name, "arguments": c.arguments},
106
+ }
107
+ for c in self.tool_calls
108
+ ]
109
+ return message
110
+
111
+
112
+ class ChatClient:
113
+ """Blocking chat-completions client with bounded retry on transient failures.
114
+
115
+ Retries 429/5xx and transport errors with exponential backoff; a 4xx is
116
+ our request's fault and fails immediately. Persistent failure raises
117
+ ``RuntimeError``, which the rollout wraps as ``PolicyError``.
118
+ """
119
+
120
+ def __init__(
121
+ self,
122
+ provider: Provider,
123
+ *,
124
+ timeout_s: float = 120.0,
125
+ max_retries: int = 3,
126
+ backoff_s: float = 1.0,
127
+ transport: httpx.BaseTransport | None = None,
128
+ ):
129
+ self._provider = provider
130
+ self._max_retries = max_retries
131
+ self._backoff_s = backoff_s
132
+ headers = {}
133
+ if provider.api_key:
134
+ headers["Authorization"] = f"Bearer {provider.api_key}"
135
+ self._http = httpx.Client(
136
+ base_url=provider.base_url,
137
+ headers=headers,
138
+ timeout=timeout_s,
139
+ transport=transport,
140
+ )
141
+
142
+ def complete(
143
+ self,
144
+ messages: list[dict[str, Any]],
145
+ tools: list[dict[str, Any]],
146
+ temperature: float | None = None,
147
+ ) -> AssistantMessage:
148
+ body: dict[str, Any] = {"model": self._provider.model, "messages": messages}
149
+ if tools:
150
+ body["tools"] = tools
151
+ if temperature is not None:
152
+ body["temperature"] = temperature
153
+
154
+ last_error = "unknown error"
155
+ for attempt in range(self._max_retries):
156
+ try:
157
+ response = self._http.post("/chat/completions", json=body)
158
+ except httpx.TransportError as exc:
159
+ last_error = str(exc)
160
+ else:
161
+ if response.status_code == 200:
162
+ return _parse_message(response.json())
163
+ last_error = f"HTTP {response.status_code}: {response.text[:500]}"
164
+ if response.status_code not in (429,) and response.status_code < 500:
165
+ # A 4xx is our request's fault; retrying cannot help.
166
+ raise RuntimeError(f"LLM request rejected — {last_error}")
167
+ if attempt + 1 < self._max_retries:
168
+ time.sleep(self._backoff_s * 2**attempt)
169
+ raise RuntimeError(f"LLM request failed after {self._max_retries} attempts — {last_error}")
170
+
171
+ def close(self) -> None:
172
+ self._http.close()
173
+
174
+
175
+ def _parse_message(payload: dict[str, Any]) -> AssistantMessage:
176
+ message = payload["choices"][0]["message"]
177
+ calls = tuple(
178
+ ToolCall(
179
+ id=str(c["id"]),
180
+ name=str(c["function"]["name"]),
181
+ arguments=str(c["function"]["arguments"]),
182
+ )
183
+ for c in message.get("tool_calls") or []
184
+ )
185
+ content = message.get("content")
186
+ return AssistantMessage(content=content if content is None else str(content), tool_calls=calls)
@@ -0,0 +1,50 @@
1
+ """Minimal stdlib PNG encoder for camera frames (no Pillow dependency).
2
+
3
+ LLM APIs take images as base64 data URLs; this encodes an ``(H, W, C)``
4
+ uint8/float array as an uncompressed-filter PNG using only zlib + struct.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+ import base64
10
+ import struct
11
+ import zlib
12
+ from typing import Any
13
+
14
+ import numpy as np
15
+ import numpy.typing as npt
16
+
17
+ _COLOR_TYPE_BY_CHANNELS = {1: 0, 3: 2, 4: 6}
18
+
19
+
20
+ def _chunk(tag: bytes, data: bytes) -> bytes:
21
+ return struct.pack(">I", len(data)) + tag + data + struct.pack(">I", zlib.crc32(tag + data))
22
+
23
+
24
+ def encode_png(image: npt.NDArray[Any]) -> bytes:
25
+ """Encode an ``(H, W)`` or ``(H, W, {1,3,4})`` array as PNG bytes.
26
+
27
+ Float arrays are assumed normalized to [0, 1] and scaled; everything else
28
+ is cast to uint8.
29
+ """
30
+ arr = np.asarray(image)
31
+ if np.issubdtype(arr.dtype, np.floating):
32
+ arr = (np.clip(arr, 0.0, 1.0) * 255.0).round()
33
+ arr = np.ascontiguousarray(arr.astype(np.uint8))
34
+ if arr.ndim == 2:
35
+ arr = arr[:, :, np.newaxis]
36
+ height, width, channels = arr.shape
37
+ color_type = _COLOR_TYPE_BY_CHANNELS[channels]
38
+ header = struct.pack(">IIBBBBB", width, height, 8, color_type, 0, 0, 0)
39
+ raw = b"".join(b"\x00" + arr[row].tobytes() for row in range(height))
40
+ return (
41
+ b"\x89PNG\r\n\x1a\n"
42
+ + _chunk(b"IHDR", header)
43
+ + _chunk(b"IDAT", zlib.compress(raw))
44
+ + _chunk(b"IEND", b"")
45
+ )
46
+
47
+
48
+ def png_data_url(image: npt.NDArray[Any]) -> str:
49
+ """The ``data:image/png;base64,...`` form LLM APIs accept inline."""
50
+ return "data:image/png;base64," + base64.b64encode(encode_png(image)).decode("ascii")