bma-executor-sdk 1.0.0__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/PKG-INFO +8 -5
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/README.md +7 -4
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/examples/executor_minimal.py +1 -1
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/pyproject.toml +1 -1
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/__init__.py +4 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/client.py +77 -17
- bma_executor_sdk-1.0.2/tests/test_base_url.py +36 -0
- bma_executor_sdk-1.0.2/tests/test_reconnect.py +144 -0
- bma_executor_sdk-1.0.0/tests/test_base_url.py +0 -17
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/.gitignore +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/examples/agent_minimal.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/examples/harness_loopback.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_methods.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_npipe.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_rpc.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/agent.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/generated.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/harness.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/methods.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/py.typed +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/tools.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/transport.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/executor_helpers.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_close_codes.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_executor_conformance.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_executor_contract.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_executor_interop.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_executor_runtime.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_executor_token_refresh.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_npipe.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_review.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/test_typed_tools.py +0 -0
- {bma_executor_sdk-1.0.0 → bma_executor_sdk-1.0.2}/tests/typecheck/typed_tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bma-executor-sdk
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports.
|
|
5
5
|
Project-URL: Repository, https://github.com/beeos-ai/bma-executor-sdk
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -13,17 +13,20 @@ Description-Content-Type: text/markdown
|
|
|
13
13
|
|
|
14
14
|
[](https://pypi.org/project/bma-executor-sdk/)
|
|
15
15
|
|
|
16
|
+
Connect your machine (hands) or your own agent (brain) to BMA — the executor, harness and agent ends of the `bma.executor` protocol.
|
|
17
|
+
[`bma-sdk`](https://github.com/beeos-ai/bma-sdk) drives and uses agents (REST / façade client); `bma-executor-sdk` connects hands or brains to BMA.
|
|
18
|
+
|
|
16
19
|
asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the BMA Executor Protocol `bma.executor.v1.1`, the counterpart of the [Go `executor` module](../README.md). Protocol types are TypedDicts generated into `bma_executor_sdk/generated.py` from the contract in [`contracts/bma/executor`](../contracts/bma/executor), so messages are plain JSON dicts.
|
|
17
20
|
|
|
18
21
|
| | |
|
|
19
22
|
| --- | --- |
|
|
20
|
-
| Version | 0.
|
|
23
|
+
| Version | 1.0.0 (tag `v1.0.0`), see [CHANGELOG](../CHANGELOG.md) |
|
|
21
24
|
| Protocol | `bma.executor.v1.1` (contract 1.1.0, [`lock.json`](../contracts/bma/executor/lock.json)) |
|
|
22
25
|
| Python | 3.10 or newer; depends on `httpx` and `websockets` |
|
|
23
26
|
| Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
|
|
24
27
|
|
|
25
28
|
```sh
|
|
26
|
-
pip install bma-executor-sdk==0.
|
|
29
|
+
pip install bma-executor-sdk==1.0.2
|
|
27
30
|
```
|
|
28
31
|
|
|
29
32
|
The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
|
|
@@ -50,7 +53,7 @@ async def main() -> None:
|
|
|
50
53
|
return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
|
|
51
54
|
|
|
52
55
|
await client.start()
|
|
53
|
-
await
|
|
56
|
+
await client.wait_closed() # raises when the client ends for good
|
|
54
57
|
|
|
55
58
|
asyncio.run(main())
|
|
56
59
|
```
|
|
@@ -64,7 +67,7 @@ Behavior matches the Go SDK:
|
|
|
64
67
|
- A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
|
|
65
68
|
- `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
|
|
66
69
|
- `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
|
|
67
|
-
- Reconnect resumes with `resume.executor_id` / `active_sessions
|
|
70
|
+
- Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
|
|
68
71
|
- Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
|
|
69
72
|
- Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
|
|
70
73
|
|
|
@@ -2,17 +2,20 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://pypi.org/project/bma-executor-sdk/)
|
|
4
4
|
|
|
5
|
+
Connect your machine (hands) or your own agent (brain) to BMA — the executor, harness and agent ends of the `bma.executor` protocol.
|
|
6
|
+
[`bma-sdk`](https://github.com/beeos-ai/bma-sdk) drives and uses agents (REST / façade client); `bma-executor-sdk` connects hands or brains to BMA.
|
|
7
|
+
|
|
5
8
|
asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the BMA Executor Protocol `bma.executor.v1.1`, the counterpart of the [Go `executor` module](../README.md). Protocol types are TypedDicts generated into `bma_executor_sdk/generated.py` from the contract in [`contracts/bma/executor`](../contracts/bma/executor), so messages are plain JSON dicts.
|
|
6
9
|
|
|
7
10
|
| | |
|
|
8
11
|
| --- | --- |
|
|
9
|
-
| Version | 0.
|
|
12
|
+
| Version | 1.0.0 (tag `v1.0.0`), see [CHANGELOG](../CHANGELOG.md) |
|
|
10
13
|
| Protocol | `bma.executor.v1.1` (contract 1.1.0, [`lock.json`](../contracts/bma/executor/lock.json)) |
|
|
11
14
|
| Python | 3.10 or newer; depends on `httpx` and `websockets` |
|
|
12
15
|
| Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
|
|
13
16
|
|
|
14
17
|
```sh
|
|
15
|
-
pip install bma-executor-sdk==0.
|
|
18
|
+
pip install bma-executor-sdk==1.0.2
|
|
16
19
|
```
|
|
17
20
|
|
|
18
21
|
The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
|
|
@@ -39,7 +42,7 @@ async def main() -> None:
|
|
|
39
42
|
return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
|
|
40
43
|
|
|
41
44
|
await client.start()
|
|
42
|
-
await
|
|
45
|
+
await client.wait_closed() # raises when the client ends for good
|
|
43
46
|
|
|
44
47
|
asyncio.run(main())
|
|
45
48
|
```
|
|
@@ -53,7 +56,7 @@ Behavior matches the Go SDK:
|
|
|
53
56
|
- A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
|
|
54
57
|
- `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
|
|
55
58
|
- `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
|
|
56
|
-
- Reconnect resumes with `resume.executor_id` / `active_sessions
|
|
59
|
+
- Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
|
|
57
60
|
- Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
|
|
58
61
|
- Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
|
|
59
62
|
|
|
@@ -48,7 +48,7 @@ async def main() -> None:
|
|
|
48
48
|
registered = await client.start()
|
|
49
49
|
print("registered as", registered["executor_id"])
|
|
50
50
|
try:
|
|
51
|
-
await
|
|
51
|
+
await client.wait_closed() # raises when the client ends for good
|
|
52
52
|
finally:
|
|
53
53
|
await client.stop("shutdown")
|
|
54
54
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "bma-executor-sdk"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.2"
|
|
4
4
|
description = "BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -12,11 +12,13 @@ from .client import (
|
|
|
12
12
|
DEFAULT_BASE_URL,
|
|
13
13
|
PROTOCOL_V1,
|
|
14
14
|
PROTOCOL_V11,
|
|
15
|
+
CredentialRejectedError,
|
|
15
16
|
ExecutorClient,
|
|
16
17
|
ExecutorTokenError,
|
|
17
18
|
ExecutorTokenScopeError,
|
|
18
19
|
LogEvent,
|
|
19
20
|
MCPChannel,
|
|
21
|
+
ReconnectExhaustedError,
|
|
20
22
|
static_token,
|
|
21
23
|
)
|
|
22
24
|
from .generated import (
|
|
@@ -76,6 +78,7 @@ __all__ = [
|
|
|
76
78
|
"ChannelClosed",
|
|
77
79
|
"ClientStoppedError",
|
|
78
80
|
"ComputerActionInput",
|
|
81
|
+
"CredentialRejectedError",
|
|
79
82
|
"ComputerObserveInput",
|
|
80
83
|
"ConnectRequest",
|
|
81
84
|
"Connection",
|
|
@@ -98,6 +101,7 @@ __all__ = [
|
|
|
98
101
|
"MCPChannel",
|
|
99
102
|
"NotConnectedError",
|
|
100
103
|
"Principal",
|
|
104
|
+
"ReconnectExhaustedError",
|
|
101
105
|
"RPCError",
|
|
102
106
|
"ReadInput",
|
|
103
107
|
"Session",
|
|
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
|
|
|
16
16
|
import httpx
|
|
17
17
|
|
|
18
18
|
from ._methods import Call, Notification, Request
|
|
19
|
-
from ._rpc import ClientStoppedError, NotConnectedError, Peer, RPCError
|
|
19
|
+
from ._rpc import ClientStoppedError, ExecutorError, NotConnectedError, Peer, RPCError
|
|
20
20
|
from .generated import (
|
|
21
21
|
METHOD_EXECUTOR_CAPABILITIES_UPDATE,
|
|
22
22
|
METHOD_EXECUTOR_CLOSE,
|
|
@@ -64,8 +64,10 @@ if TYPE_CHECKING:
|
|
|
64
64
|
|
|
65
65
|
DEFAULT_BASE_URL = "https://api.bma.beeos.ai"
|
|
66
66
|
DEFAULT_REQUEST_TIMEOUT = 30.0
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
# 0 reconnects until stop(), as gRPC connection backoff and long-running
|
|
68
|
+
# agents (GitHub Actions runner, Buildkite agent) do.
|
|
69
|
+
DEFAULT_MAX_RECONNECT_ATTEMPTS = 0
|
|
70
|
+
DEFAULT_RECONNECT_BASE_DELAY = 1.0
|
|
69
71
|
DEFAULT_TOOL_RESULT_RETRY_ATTEMPTS = 5
|
|
70
72
|
DEFAULT_TOOL_RESULT_RETRY_BASE_DELAY = 0.25
|
|
71
73
|
DEFAULT_HEARTBEAT_INTERVAL = 15.0
|
|
@@ -73,7 +75,7 @@ PROTOCOL_V1 = ProtocolVersion.BmaExecutorV1.value
|
|
|
73
75
|
PROTOCOL_V11 = ProtocolVersion.BmaExecutorV11.value
|
|
74
76
|
_MAX_TRACKED_TOOL_RESULTS = 8192
|
|
75
77
|
_MAX_TOOL_RESULT_RETRY_DELAY = 10.0
|
|
76
|
-
_MAX_RECONNECT_DELAY =
|
|
78
|
+
_MAX_RECONNECT_DELAY = 60.0
|
|
77
79
|
# Fraction of a token's lifetime after which executor.token.refresh is sent.
|
|
78
80
|
_TOKEN_REFRESH_AT = 0.7
|
|
79
81
|
|
|
@@ -172,6 +174,9 @@ class ExecutorClient:
|
|
|
172
174
|
self._peer: Peer | None = None
|
|
173
175
|
self._state = ConnectionState.Disconnected
|
|
174
176
|
self._stopped = False
|
|
177
|
+
self._stop_called = False
|
|
178
|
+
self._closed = asyncio.Event()
|
|
179
|
+
self._close_error: BaseException | None = None
|
|
175
180
|
self._register_result: ExecutorRegisterResult | None = None
|
|
176
181
|
self._connection_id = ""
|
|
177
182
|
self._executor_id = ""
|
|
@@ -215,11 +220,11 @@ class ExecutorClient:
|
|
|
215
220
|
|
|
216
221
|
async def stop(self, reason: str = "") -> None:
|
|
217
222
|
"""Send executor.close, close the channel and fail pending calls."""
|
|
218
|
-
if self.
|
|
223
|
+
if self._stop_called:
|
|
219
224
|
return
|
|
220
|
-
self.
|
|
225
|
+
self._stop_called = True
|
|
226
|
+
self._end(None)
|
|
221
227
|
peer, self._peer = self._peer, None
|
|
222
|
-
self._state = ConnectionState.Closed
|
|
223
228
|
if peer is not None:
|
|
224
229
|
try:
|
|
225
230
|
await asyncio.wait_for(peer.notify(METHOD_EXECUTOR_CLOSE, {"reason": reason}), 2)
|
|
@@ -236,6 +241,25 @@ class ExecutorClient:
|
|
|
236
241
|
if self._tasks:
|
|
237
242
|
await asyncio.gather(*self._tasks, return_exceptions=True)
|
|
238
243
|
|
|
244
|
+
async def wait_closed(self) -> None:
|
|
245
|
+
"""Wait until the client stops for good (Go: ``Done`` and ``Err``).
|
|
246
|
+
Returns after stop() or executor.close; raises ChannelClosed for a
|
|
247
|
+
terminal close code (4003, 4004), CredentialRejectedError when the
|
|
248
|
+
token endpoint rejects the credential on reconnect, and
|
|
249
|
+
ReconnectExhaustedError after a positive max_reconnect_attempts."""
|
|
250
|
+
await self._closed.wait()
|
|
251
|
+
if self._close_error is not None:
|
|
252
|
+
raise self._close_error
|
|
253
|
+
|
|
254
|
+
def _end(self, error: BaseException | None) -> None:
|
|
255
|
+
"""Stop for good; wait_closed() then returns or raises error."""
|
|
256
|
+
if self._closed.is_set():
|
|
257
|
+
return
|
|
258
|
+
self._stopped = True
|
|
259
|
+
self._state = ConnectionState.Closed
|
|
260
|
+
self._close_error = error
|
|
261
|
+
self._closed.set()
|
|
262
|
+
|
|
239
263
|
@property
|
|
240
264
|
def state(self) -> ConnectionState:
|
|
241
265
|
return self._state
|
|
@@ -398,7 +422,9 @@ class ExecutorClient:
|
|
|
398
422
|
client = self._http_client or httpx.AsyncClient()
|
|
399
423
|
try:
|
|
400
424
|
response = await client.post(
|
|
401
|
-
|
|
425
|
+
# A machine daemon token (no environment id) is exchanged by
|
|
426
|
+
# bma-api, an environment key by managed-agents.
|
|
427
|
+
f"{self._base_url}{'/v1' if self._environment_id else '/api/v1'}/executors/token",
|
|
402
428
|
json={"environment_id": self._environment_id, "executor": self._executor},
|
|
403
429
|
headers={"Authorization": f"Bearer {self._environment_key}"},
|
|
404
430
|
timeout=self._request_timeout,
|
|
@@ -407,7 +433,10 @@ class ExecutorClient:
|
|
|
407
433
|
if self._http_client is None:
|
|
408
434
|
await client.aclose()
|
|
409
435
|
if response.status_code != 200:
|
|
410
|
-
|
|
436
|
+
message = f"token request failed: {response.status_code}: {response.text[:4096].strip()}"
|
|
437
|
+
if response.status_code in (401, 403):
|
|
438
|
+
raise CredentialRejectedError(message, response.status_code)
|
|
439
|
+
raise ExecutorTokenError(message)
|
|
411
440
|
token = response.json()
|
|
412
441
|
if not token.get("token") or not token.get("ws_url"):
|
|
413
442
|
raise ExecutorTokenError("token response is missing token or ws_url")
|
|
@@ -508,24 +537,27 @@ class ExecutorClient:
|
|
|
508
537
|
return
|
|
509
538
|
self._peer = None
|
|
510
539
|
if self._stopped:
|
|
511
|
-
self._state = ConnectionState.Closed
|
|
512
540
|
return
|
|
513
541
|
if closed.code in TERMINAL_CLOSE_CODES:
|
|
514
|
-
self._state = ConnectionState.Closed
|
|
515
542
|
self._log("error", f"executor channel closed with terminal code {closed.code}: {closed.reason}")
|
|
543
|
+
self._end(closed)
|
|
516
544
|
return
|
|
517
545
|
self._state = ConnectionState.Disconnected
|
|
518
546
|
self._spawn(self._reconnect_loop())
|
|
519
547
|
|
|
520
548
|
async def _reconnect_loop(self) -> None:
|
|
549
|
+
cause: BaseException | None = None
|
|
521
550
|
while not self._stopped:
|
|
522
|
-
if self.
|
|
523
|
-
|
|
524
|
-
|
|
551
|
+
if 0 < self._max_reconnect_attempts <= self._reconnect_attempts:
|
|
552
|
+
error = ReconnectExhaustedError("executor reconnect attempts exhausted")
|
|
553
|
+
error.__cause__ = cause
|
|
554
|
+
self._log("error", "executor reconnect attempts exhausted", cause)
|
|
555
|
+
self._end(error)
|
|
525
556
|
return
|
|
526
557
|
attempt = self._reconnect_attempts
|
|
527
558
|
self._reconnect_attempts += 1
|
|
528
|
-
|
|
559
|
+
# The exponent is capped: unlimited attempts must not overflow the float.
|
|
560
|
+
delay = min(self._reconnect_base_delay * (1 << min(attempt, 32)), self._reconnect_max_delay)
|
|
529
561
|
delay *= 0.5 + self._random() * 0.5
|
|
530
562
|
self._log("info", f"reconnecting executor channel in {delay:.3f}s")
|
|
531
563
|
await asyncio.sleep(delay)
|
|
@@ -536,7 +568,14 @@ class ExecutorClient:
|
|
|
536
568
|
return
|
|
537
569
|
except asyncio.CancelledError:
|
|
538
570
|
raise
|
|
571
|
+
except CredentialRejectedError as error:
|
|
572
|
+
# Retrying cannot help a revoked or invalid credential (GitHub
|
|
573
|
+
# Actions runner and Buildkite agent stop on 401/403 too).
|
|
574
|
+
self._log("error", "executor credential rejected; not reconnecting", error)
|
|
575
|
+
self._end(error)
|
|
576
|
+
return
|
|
539
577
|
except Exception as error:
|
|
578
|
+
cause = error
|
|
540
579
|
self._state = ConnectionState.Disconnected
|
|
541
580
|
self._log("warn", "executor reconnect failed", error)
|
|
542
581
|
|
|
@@ -550,10 +589,18 @@ class ExecutorClient:
|
|
|
550
589
|
load = self._heartbeat_load()
|
|
551
590
|
if load is not None:
|
|
552
591
|
params["load"] = load
|
|
592
|
+
peer = self._peer
|
|
553
593
|
try:
|
|
554
594
|
result = await self._call(METHOD_EXECUTOR_HEARTBEAT, params)
|
|
555
595
|
except (RPCError, NotConnectedError, TimeoutError) as error:
|
|
556
596
|
self._log("warn", "executor heartbeat failed", error)
|
|
597
|
+
if isinstance(error, TimeoutError) and peer is not None and peer is self._peer:
|
|
598
|
+
# No answer within request_timeout: the connection is dead
|
|
599
|
+
# (a half-open TCP connection after a network cut). Close it
|
|
600
|
+
# so the client reconnects, as gRPC keepalive closes a
|
|
601
|
+
# transport whose ping times out.
|
|
602
|
+
self._log("warn", "executor heartbeat timed out; closing the connection to reconnect")
|
|
603
|
+
await peer.conn.close(1001, "heartbeat timeout")
|
|
557
604
|
continue
|
|
558
605
|
if result.get("drain") and self._state == ConnectionState.Ready:
|
|
559
606
|
self._state = ConnectionState.Draining
|
|
@@ -596,8 +643,7 @@ class ExecutorClient:
|
|
|
596
643
|
def _dispatch_notification(self, method: str, params: Any) -> None:
|
|
597
644
|
if method == METHOD_EXECUTOR_CLOSE:
|
|
598
645
|
# The harness closed the executor deliberately: no reconnect.
|
|
599
|
-
self.
|
|
600
|
-
self._state = ConnectionState.Closed
|
|
646
|
+
self._end(None)
|
|
601
647
|
peer, self._peer = self._peer, None
|
|
602
648
|
if peer is not None:
|
|
603
649
|
peer.fail_pending(ClientStoppedError())
|
|
@@ -705,6 +751,20 @@ class ExecutorTokenError(Exception):
|
|
|
705
751
|
"""POST /v1/executors/token failed."""
|
|
706
752
|
|
|
707
753
|
|
|
754
|
+
class CredentialRejectedError(ExecutorTokenError):
|
|
755
|
+
"""The token endpoint answered 401 or 403: the environment key or machine
|
|
756
|
+
token is invalid or revoked. A reconnect does not retry it."""
|
|
757
|
+
|
|
758
|
+
def __init__(self, message: str, status_code: int) -> None:
|
|
759
|
+
super().__init__(message)
|
|
760
|
+
self.status_code = status_code
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
class ReconnectExhaustedError(ExecutorError):
|
|
764
|
+
"""A positive max_reconnect_attempts failed in a row; ``__cause__`` is the
|
|
765
|
+
last failure."""
|
|
766
|
+
|
|
767
|
+
|
|
708
768
|
class ExecutorTokenScopeError(Exception):
|
|
709
769
|
"""The executor token does not allow the requested channel."""
|
|
710
770
|
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
"""base_url is the deployment origin: a trailing /api/v1 is dropped."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import pytest
|
|
6
|
+
|
|
7
|
+
import bma_executor_sdk as bx
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@pytest.mark.parametrize("base_url", ["https://bma.test", "https://bma.test/", "https://bma.test/api/v1", "https://bma.test/api/v1/"])
|
|
11
|
+
def test_base_url_is_the_origin(base_url: str) -> None:
|
|
12
|
+
assert bx.DEFAULT_BASE_URL == "https://api.bma.beeos.ai"
|
|
13
|
+
client = bx.ExecutorClient(
|
|
14
|
+
environment_id="env", environment_key="k", base_url=base_url,
|
|
15
|
+
executor={"name": "t", "os": "linux", "arch": "arm64", "version": "0"},
|
|
16
|
+
)
|
|
17
|
+
assert client._base_url == "https://bma.test" # pyright: ignore[reportPrivateUsage]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@pytest.mark.parametrize(("environment_id", "path"), [("", "/api/v1/executors/token"), ("env", "/v1/executors/token")])
|
|
21
|
+
async def test_token_exchange_path_follows_the_credential(environment_id: str, path: str) -> None:
|
|
22
|
+
import httpx
|
|
23
|
+
|
|
24
|
+
seen: list[str] = []
|
|
25
|
+
|
|
26
|
+
def handler(request: httpx.Request) -> httpx.Response:
|
|
27
|
+
seen.append(request.url.path)
|
|
28
|
+
return httpx.Response(200, json={"executor_id": "exec_t", "token": "jwt", "expires_in": 300, "ws_url": "ws://x"})
|
|
29
|
+
|
|
30
|
+
client = bx.ExecutorClient(
|
|
31
|
+
environment_id=environment_id, environment_key="k", base_url="https://bma.test",
|
|
32
|
+
executor={"name": "t", "os": "linux", "arch": "arm64", "version": "0"},
|
|
33
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(handler)),
|
|
34
|
+
)
|
|
35
|
+
await client._fetch_token() # pyright: ignore[reportPrivateUsage]
|
|
36
|
+
assert seen == [path]
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Reconnect policy (Go reconnect_test.go): by default the client reconnects
|
|
2
|
+
until stop() through any outage; a credential the token endpoint rejects
|
|
3
|
+
(401/403) and an exhausted positive max_reconnect_attempts end the client,
|
|
4
|
+
and wait_closed() raises the reason instead of the client hanging."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
import bma_executor_sdk as bx
|
|
15
|
+
from bma_executor_sdk import client as bxc
|
|
16
|
+
|
|
17
|
+
EXECUTOR = {"name": "py-test", "os": "linux", "arch": "arm64", "version": "0.1.0"}
|
|
18
|
+
Ready = bx.generated.ConnectionState.Ready
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TokenEndpoint:
|
|
22
|
+
"""Answers the first token request, then fail_status for the next
|
|
23
|
+
fail_for requests (forever when fail_for < 0)."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, fail_status: int, fail_for: int) -> None:
|
|
26
|
+
self.requests = 0
|
|
27
|
+
self.fail_status = fail_status
|
|
28
|
+
self.fail_for = fail_for
|
|
29
|
+
|
|
30
|
+
def __call__(self, request: httpx.Request) -> httpx.Response:
|
|
31
|
+
self.requests += 1
|
|
32
|
+
failed = self.requests - 1
|
|
33
|
+
if failed > 0 and (self.fail_for < 0 or failed <= self.fail_for):
|
|
34
|
+
return httpx.Response(self.fail_status, text="upstream unavailable")
|
|
35
|
+
return httpx.Response(200, json={"executor_id": "", "token": "jwt", "expires_in": 300, "ws_url": "inprocess://harness"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def start_client(harness: bx.Harness, endpoint: TokenEndpoint, **options: Any) -> bx.ExecutorClient:
|
|
39
|
+
client = bx.ExecutorClient(
|
|
40
|
+
environment_id="env_local", executor=EXECUTOR, environment_key="env_key", base_url="http://bma.test",
|
|
41
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(endpoint)), dialer=harness.inprocess_dialer(),
|
|
42
|
+
request_timeout=2.0, reconnect_base_delay=0.001, reconnect_max_delay=0.002, **options,
|
|
43
|
+
)
|
|
44
|
+
await client.start()
|
|
45
|
+
return client
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
async def drop(client: bx.ExecutorClient) -> None:
|
|
49
|
+
await client._peer.conn.close(1012, "node replaced") # noqa: SLF001
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def test_reconnects_after_a_long_outage_by_default() -> None:
|
|
53
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
54
|
+
endpoint = TokenEndpoint(503, 25)
|
|
55
|
+
client = await start_client(harness, endpoint)
|
|
56
|
+
first = client.connection_id
|
|
57
|
+
await drop(client)
|
|
58
|
+
for _ in range(500):
|
|
59
|
+
if client.state == Ready and client.connection_id != first:
|
|
60
|
+
break
|
|
61
|
+
await asyncio.sleep(0.01)
|
|
62
|
+
assert client.state == Ready and client.connection_id != first
|
|
63
|
+
assert endpoint.requests == 27 # the start, 25 failures, the reconnect
|
|
64
|
+
await client.stop()
|
|
65
|
+
await asyncio.wait_for(client.wait_closed(), 1) # stop() ends it without an error
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.parametrize("status", [401, 403])
|
|
69
|
+
async def test_rejected_credential_ends_the_client(status: int) -> None:
|
|
70
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
71
|
+
endpoint = TokenEndpoint(status, -1)
|
|
72
|
+
client = await start_client(harness, endpoint)
|
|
73
|
+
await drop(client)
|
|
74
|
+
with pytest.raises(bx.CredentialRejectedError) as raised:
|
|
75
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
76
|
+
assert raised.value.status_code == status
|
|
77
|
+
assert endpoint.requests == 2 # no retry after the rejection
|
|
78
|
+
assert client.state == bx.generated.ConnectionState.Closed
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def test_exhausted_max_reconnect_attempts_ends_the_client() -> None:
|
|
82
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
83
|
+
endpoint = TokenEndpoint(503, -1)
|
|
84
|
+
client = await start_client(harness, endpoint, max_reconnect_attempts=3)
|
|
85
|
+
await drop(client)
|
|
86
|
+
with pytest.raises(bx.ReconnectExhaustedError) as raised:
|
|
87
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
88
|
+
assert isinstance(raised.value.__cause__, bx.ExecutorTokenError)
|
|
89
|
+
assert endpoint.requests == 4 # the start and 3 reconnect attempts
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def test_terminal_close_code_ends_the_client() -> None:
|
|
93
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
94
|
+
client = await start_client(harness, TokenEndpoint(503, 0))
|
|
95
|
+
await client._peer.conn.close(4004, "replaced") # noqa: SLF001
|
|
96
|
+
with pytest.raises(bx.ChannelClosed) as raised:
|
|
97
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
98
|
+
assert raised.value.code == 4004
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def test_heartbeat_timeout_reconnects(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
102
|
+
"""A heartbeat unanswered within request_timeout means a dead (half-open)
|
|
103
|
+
connection: the client closes it and reconnects (gRPC keepalive)."""
|
|
104
|
+
from bma_executor_sdk import harness as bxh
|
|
105
|
+
|
|
106
|
+
silent = {"on": False}
|
|
107
|
+
dispatch = bxh._HarnessConn._dispatch # noqa: SLF001
|
|
108
|
+
|
|
109
|
+
async def maybe_silent(self: Any, executor: Any, request_id: str, method: str, params: Any) -> Any:
|
|
110
|
+
if method == bx.generated.METHOD_EXECUTOR_HEARTBEAT and silent["on"]:
|
|
111
|
+
await asyncio.Event().wait()
|
|
112
|
+
return await dispatch(self, executor, request_id, method, params)
|
|
113
|
+
|
|
114
|
+
monkeypatch.setattr(bxh._HarnessConn, "_dispatch", maybe_silent) # noqa: SLF001
|
|
115
|
+
harness = bx.Harness(request_timeout=2.0, heartbeat_interval=0.02)
|
|
116
|
+
client = await start_client(harness, TokenEndpoint(503, 0))
|
|
117
|
+
client._request_timeout = 0.1 # noqa: SLF001
|
|
118
|
+
first = client.connection_id
|
|
119
|
+
silent["on"] = True
|
|
120
|
+
for _ in range(300):
|
|
121
|
+
if client.state == Ready and client.connection_id != first:
|
|
122
|
+
break
|
|
123
|
+
await asyncio.sleep(0.01)
|
|
124
|
+
assert client.state == Ready and client.connection_id != first
|
|
125
|
+
await client.stop()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_reconnect_defaults() -> None:
|
|
129
|
+
assert bxc.DEFAULT_MAX_RECONNECT_ATTEMPTS == 0
|
|
130
|
+
assert bxc.DEFAULT_RECONNECT_BASE_DELAY == 1.0
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
async def test_backoff_stays_capped_after_many_attempts() -> None:
|
|
134
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
135
|
+
client = await start_client(harness, TokenEndpoint(503, 3))
|
|
136
|
+
first = client.connection_id
|
|
137
|
+
client._reconnect_attempts = 5000 # noqa: SLF001 hours into an outage
|
|
138
|
+
await drop(client)
|
|
139
|
+
for _ in range(300):
|
|
140
|
+
if client.state == Ready and client.connection_id != first:
|
|
141
|
+
break
|
|
142
|
+
await asyncio.sleep(0.01)
|
|
143
|
+
assert client.state == Ready and client.connection_id != first # no OverflowError in the backoff
|
|
144
|
+
await client.stop()
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
"""base_url is the deployment origin: a trailing /api/v1 is dropped."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import pytest
|
|
6
|
-
|
|
7
|
-
import bma_executor_sdk as bx
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
@pytest.mark.parametrize("base_url", ["https://bma.test", "https://bma.test/", "https://bma.test/api/v1", "https://bma.test/api/v1/"])
|
|
11
|
-
def test_base_url_is_the_origin(base_url: str) -> None:
|
|
12
|
-
assert bx.DEFAULT_BASE_URL == "https://api.bma.beeos.ai"
|
|
13
|
-
client = bx.ExecutorClient(
|
|
14
|
-
environment_id="env", environment_key="k", base_url=base_url,
|
|
15
|
-
executor={"name": "t", "os": "linux", "arch": "arm64", "version": "0"},
|
|
16
|
-
)
|
|
17
|
-
assert client._base_url == "https://bma.test" # pyright: ignore[reportPrivateUsage]
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|