bma-executor-sdk 1.0.1__tar.gz → 1.0.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/PKG-INFO +4 -4
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/README.md +3 -3
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/executor_minimal.py +1 -1
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/pyproject.toml +1 -1
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/__init__.py +4 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/client.py +74 -16
- bma_executor_sdk-1.0.2/tests/test_reconnect.py +144 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/.gitignore +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/agent_minimal.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/harness_loopback.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_methods.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_npipe.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_rpc.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/agent.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/generated.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/harness.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/methods.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/py.typed +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/tools.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/transport.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/executor_helpers.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_base_url.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_close_codes.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_conformance.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_contract.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_interop.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_runtime.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_token_refresh.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_npipe.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_review.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_typed_tools.py +0 -0
- {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/typecheck/typed_tools.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: bma-executor-sdk
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.2
|
|
4
4
|
Summary: BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports.
|
|
5
5
|
Project-URL: Repository, https://github.com/beeos-ai/bma-executor-sdk
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -26,7 +26,7 @@ asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the B
|
|
|
26
26
|
| Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
|
|
27
27
|
|
|
28
28
|
```sh
|
|
29
|
-
pip install bma-executor-sdk==1.0.
|
|
29
|
+
pip install bma-executor-sdk==1.0.2
|
|
30
30
|
```
|
|
31
31
|
|
|
32
32
|
The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
|
|
@@ -53,7 +53,7 @@ async def main() -> None:
|
|
|
53
53
|
return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
|
|
54
54
|
|
|
55
55
|
await client.start()
|
|
56
|
-
await
|
|
56
|
+
await client.wait_closed() # raises when the client ends for good
|
|
57
57
|
|
|
58
58
|
asyncio.run(main())
|
|
59
59
|
```
|
|
@@ -67,7 +67,7 @@ Behavior matches the Go SDK:
|
|
|
67
67
|
- A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
|
|
68
68
|
- `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
|
|
69
69
|
- `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
|
|
70
|
-
- Reconnect resumes with `resume.executor_id` / `active_sessions
|
|
70
|
+
- Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
|
|
71
71
|
- Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
|
|
72
72
|
- Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
|
|
73
73
|
|
|
@@ -15,7 +15,7 @@ asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the B
|
|
|
15
15
|
| Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
|
|
16
16
|
|
|
17
17
|
```sh
|
|
18
|
-
pip install bma-executor-sdk==1.0.
|
|
18
|
+
pip install bma-executor-sdk==1.0.2
|
|
19
19
|
```
|
|
20
20
|
|
|
21
21
|
The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
|
|
@@ -42,7 +42,7 @@ async def main() -> None:
|
|
|
42
42
|
return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
|
|
43
43
|
|
|
44
44
|
await client.start()
|
|
45
|
-
await
|
|
45
|
+
await client.wait_closed() # raises when the client ends for good
|
|
46
46
|
|
|
47
47
|
asyncio.run(main())
|
|
48
48
|
```
|
|
@@ -56,7 +56,7 @@ Behavior matches the Go SDK:
|
|
|
56
56
|
- A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
|
|
57
57
|
- `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
|
|
58
58
|
- `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
|
|
59
|
-
- Reconnect resumes with `resume.executor_id` / `active_sessions
|
|
59
|
+
- Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
|
|
60
60
|
- Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
|
|
61
61
|
- Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
|
|
62
62
|
|
|
@@ -48,7 +48,7 @@ async def main() -> None:
|
|
|
48
48
|
registered = await client.start()
|
|
49
49
|
print("registered as", registered["executor_id"])
|
|
50
50
|
try:
|
|
51
|
-
await
|
|
51
|
+
await client.wait_closed() # raises when the client ends for good
|
|
52
52
|
finally:
|
|
53
53
|
await client.stop("shutdown")
|
|
54
54
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "bma-executor-sdk"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.2"
|
|
4
4
|
description = "BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports."
|
|
5
5
|
readme = "README.md"
|
|
6
6
|
license = "Apache-2.0"
|
|
@@ -12,11 +12,13 @@ from .client import (
|
|
|
12
12
|
DEFAULT_BASE_URL,
|
|
13
13
|
PROTOCOL_V1,
|
|
14
14
|
PROTOCOL_V11,
|
|
15
|
+
CredentialRejectedError,
|
|
15
16
|
ExecutorClient,
|
|
16
17
|
ExecutorTokenError,
|
|
17
18
|
ExecutorTokenScopeError,
|
|
18
19
|
LogEvent,
|
|
19
20
|
MCPChannel,
|
|
21
|
+
ReconnectExhaustedError,
|
|
20
22
|
static_token,
|
|
21
23
|
)
|
|
22
24
|
from .generated import (
|
|
@@ -76,6 +78,7 @@ __all__ = [
|
|
|
76
78
|
"ChannelClosed",
|
|
77
79
|
"ClientStoppedError",
|
|
78
80
|
"ComputerActionInput",
|
|
81
|
+
"CredentialRejectedError",
|
|
79
82
|
"ComputerObserveInput",
|
|
80
83
|
"ConnectRequest",
|
|
81
84
|
"Connection",
|
|
@@ -98,6 +101,7 @@ __all__ = [
|
|
|
98
101
|
"MCPChannel",
|
|
99
102
|
"NotConnectedError",
|
|
100
103
|
"Principal",
|
|
104
|
+
"ReconnectExhaustedError",
|
|
101
105
|
"RPCError",
|
|
102
106
|
"ReadInput",
|
|
103
107
|
"Session",
|
|
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
|
|
|
16
16
|
import httpx
|
|
17
17
|
|
|
18
18
|
from ._methods import Call, Notification, Request
|
|
19
|
-
from ._rpc import ClientStoppedError, NotConnectedError, Peer, RPCError
|
|
19
|
+
from ._rpc import ClientStoppedError, ExecutorError, NotConnectedError, Peer, RPCError
|
|
20
20
|
from .generated import (
|
|
21
21
|
METHOD_EXECUTOR_CAPABILITIES_UPDATE,
|
|
22
22
|
METHOD_EXECUTOR_CLOSE,
|
|
@@ -64,8 +64,10 @@ if TYPE_CHECKING:
|
|
|
64
64
|
|
|
65
65
|
DEFAULT_BASE_URL = "https://api.bma.beeos.ai"
|
|
66
66
|
DEFAULT_REQUEST_TIMEOUT = 30.0
|
|
67
|
-
|
|
68
|
-
|
|
67
|
+
# 0 reconnects until stop(), as gRPC connection backoff and long-running
|
|
68
|
+
# agents (GitHub Actions runner, Buildkite agent) do.
|
|
69
|
+
DEFAULT_MAX_RECONNECT_ATTEMPTS = 0
|
|
70
|
+
DEFAULT_RECONNECT_BASE_DELAY = 1.0
|
|
69
71
|
DEFAULT_TOOL_RESULT_RETRY_ATTEMPTS = 5
|
|
70
72
|
DEFAULT_TOOL_RESULT_RETRY_BASE_DELAY = 0.25
|
|
71
73
|
DEFAULT_HEARTBEAT_INTERVAL = 15.0
|
|
@@ -73,7 +75,7 @@ PROTOCOL_V1 = ProtocolVersion.BmaExecutorV1.value
|
|
|
73
75
|
PROTOCOL_V11 = ProtocolVersion.BmaExecutorV11.value
|
|
74
76
|
_MAX_TRACKED_TOOL_RESULTS = 8192
|
|
75
77
|
_MAX_TOOL_RESULT_RETRY_DELAY = 10.0
|
|
76
|
-
_MAX_RECONNECT_DELAY =
|
|
78
|
+
_MAX_RECONNECT_DELAY = 60.0
|
|
77
79
|
# Fraction of a token's lifetime after which executor.token.refresh is sent.
|
|
78
80
|
_TOKEN_REFRESH_AT = 0.7
|
|
79
81
|
|
|
@@ -172,6 +174,9 @@ class ExecutorClient:
|
|
|
172
174
|
self._peer: Peer | None = None
|
|
173
175
|
self._state = ConnectionState.Disconnected
|
|
174
176
|
self._stopped = False
|
|
177
|
+
self._stop_called = False
|
|
178
|
+
self._closed = asyncio.Event()
|
|
179
|
+
self._close_error: BaseException | None = None
|
|
175
180
|
self._register_result: ExecutorRegisterResult | None = None
|
|
176
181
|
self._connection_id = ""
|
|
177
182
|
self._executor_id = ""
|
|
@@ -215,11 +220,11 @@ class ExecutorClient:
|
|
|
215
220
|
|
|
216
221
|
async def stop(self, reason: str = "") -> None:
|
|
217
222
|
"""Send executor.close, close the channel and fail pending calls."""
|
|
218
|
-
if self.
|
|
223
|
+
if self._stop_called:
|
|
219
224
|
return
|
|
220
|
-
self.
|
|
225
|
+
self._stop_called = True
|
|
226
|
+
self._end(None)
|
|
221
227
|
peer, self._peer = self._peer, None
|
|
222
|
-
self._state = ConnectionState.Closed
|
|
223
228
|
if peer is not None:
|
|
224
229
|
try:
|
|
225
230
|
await asyncio.wait_for(peer.notify(METHOD_EXECUTOR_CLOSE, {"reason": reason}), 2)
|
|
@@ -236,6 +241,25 @@ class ExecutorClient:
|
|
|
236
241
|
if self._tasks:
|
|
237
242
|
await asyncio.gather(*self._tasks, return_exceptions=True)
|
|
238
243
|
|
|
244
|
+
async def wait_closed(self) -> None:
|
|
245
|
+
"""Wait until the client stops for good (Go: ``Done`` and ``Err``).
|
|
246
|
+
Returns after stop() or executor.close; raises ChannelClosed for a
|
|
247
|
+
terminal close code (4003, 4004), CredentialRejectedError when the
|
|
248
|
+
token endpoint rejects the credential on reconnect, and
|
|
249
|
+
ReconnectExhaustedError after a positive max_reconnect_attempts."""
|
|
250
|
+
await self._closed.wait()
|
|
251
|
+
if self._close_error is not None:
|
|
252
|
+
raise self._close_error
|
|
253
|
+
|
|
254
|
+
def _end(self, error: BaseException | None) -> None:
|
|
255
|
+
"""Stop for good; wait_closed() then returns or raises error."""
|
|
256
|
+
if self._closed.is_set():
|
|
257
|
+
return
|
|
258
|
+
self._stopped = True
|
|
259
|
+
self._state = ConnectionState.Closed
|
|
260
|
+
self._close_error = error
|
|
261
|
+
self._closed.set()
|
|
262
|
+
|
|
239
263
|
@property
|
|
240
264
|
def state(self) -> ConnectionState:
|
|
241
265
|
return self._state
|
|
@@ -409,7 +433,10 @@ class ExecutorClient:
|
|
|
409
433
|
if self._http_client is None:
|
|
410
434
|
await client.aclose()
|
|
411
435
|
if response.status_code != 200:
|
|
412
|
-
|
|
436
|
+
message = f"token request failed: {response.status_code}: {response.text[:4096].strip()}"
|
|
437
|
+
if response.status_code in (401, 403):
|
|
438
|
+
raise CredentialRejectedError(message, response.status_code)
|
|
439
|
+
raise ExecutorTokenError(message)
|
|
413
440
|
token = response.json()
|
|
414
441
|
if not token.get("token") or not token.get("ws_url"):
|
|
415
442
|
raise ExecutorTokenError("token response is missing token or ws_url")
|
|
@@ -510,24 +537,27 @@ class ExecutorClient:
|
|
|
510
537
|
return
|
|
511
538
|
self._peer = None
|
|
512
539
|
if self._stopped:
|
|
513
|
-
self._state = ConnectionState.Closed
|
|
514
540
|
return
|
|
515
541
|
if closed.code in TERMINAL_CLOSE_CODES:
|
|
516
|
-
self._state = ConnectionState.Closed
|
|
517
542
|
self._log("error", f"executor channel closed with terminal code {closed.code}: {closed.reason}")
|
|
543
|
+
self._end(closed)
|
|
518
544
|
return
|
|
519
545
|
self._state = ConnectionState.Disconnected
|
|
520
546
|
self._spawn(self._reconnect_loop())
|
|
521
547
|
|
|
522
548
|
async def _reconnect_loop(self) -> None:
|
|
549
|
+
cause: BaseException | None = None
|
|
523
550
|
while not self._stopped:
|
|
524
|
-
if self.
|
|
525
|
-
|
|
526
|
-
|
|
551
|
+
if 0 < self._max_reconnect_attempts <= self._reconnect_attempts:
|
|
552
|
+
error = ReconnectExhaustedError("executor reconnect attempts exhausted")
|
|
553
|
+
error.__cause__ = cause
|
|
554
|
+
self._log("error", "executor reconnect attempts exhausted", cause)
|
|
555
|
+
self._end(error)
|
|
527
556
|
return
|
|
528
557
|
attempt = self._reconnect_attempts
|
|
529
558
|
self._reconnect_attempts += 1
|
|
530
|
-
|
|
559
|
+
# The exponent is capped: unlimited attempts must not overflow the float.
|
|
560
|
+
delay = min(self._reconnect_base_delay * (1 << min(attempt, 32)), self._reconnect_max_delay)
|
|
531
561
|
delay *= 0.5 + self._random() * 0.5
|
|
532
562
|
self._log("info", f"reconnecting executor channel in {delay:.3f}s")
|
|
533
563
|
await asyncio.sleep(delay)
|
|
@@ -538,7 +568,14 @@ class ExecutorClient:
|
|
|
538
568
|
return
|
|
539
569
|
except asyncio.CancelledError:
|
|
540
570
|
raise
|
|
571
|
+
except CredentialRejectedError as error:
|
|
572
|
+
# Retrying cannot help a revoked or invalid credential (GitHub
|
|
573
|
+
# Actions runner and Buildkite agent stop on 401/403 too).
|
|
574
|
+
self._log("error", "executor credential rejected; not reconnecting", error)
|
|
575
|
+
self._end(error)
|
|
576
|
+
return
|
|
541
577
|
except Exception as error:
|
|
578
|
+
cause = error
|
|
542
579
|
self._state = ConnectionState.Disconnected
|
|
543
580
|
self._log("warn", "executor reconnect failed", error)
|
|
544
581
|
|
|
@@ -552,10 +589,18 @@ class ExecutorClient:
|
|
|
552
589
|
load = self._heartbeat_load()
|
|
553
590
|
if load is not None:
|
|
554
591
|
params["load"] = load
|
|
592
|
+
peer = self._peer
|
|
555
593
|
try:
|
|
556
594
|
result = await self._call(METHOD_EXECUTOR_HEARTBEAT, params)
|
|
557
595
|
except (RPCError, NotConnectedError, TimeoutError) as error:
|
|
558
596
|
self._log("warn", "executor heartbeat failed", error)
|
|
597
|
+
if isinstance(error, TimeoutError) and peer is not None and peer is self._peer:
|
|
598
|
+
# No answer within request_timeout: the connection is dead
|
|
599
|
+
# (a half-open TCP connection after a network cut). Close it
|
|
600
|
+
# so the client reconnects, as gRPC keepalive closes a
|
|
601
|
+
# transport whose ping times out.
|
|
602
|
+
self._log("warn", "executor heartbeat timed out; closing the connection to reconnect")
|
|
603
|
+
await peer.conn.close(1001, "heartbeat timeout")
|
|
559
604
|
continue
|
|
560
605
|
if result.get("drain") and self._state == ConnectionState.Ready:
|
|
561
606
|
self._state = ConnectionState.Draining
|
|
@@ -598,8 +643,7 @@ class ExecutorClient:
|
|
|
598
643
|
def _dispatch_notification(self, method: str, params: Any) -> None:
|
|
599
644
|
if method == METHOD_EXECUTOR_CLOSE:
|
|
600
645
|
# The harness closed the executor deliberately: no reconnect.
|
|
601
|
-
self.
|
|
602
|
-
self._state = ConnectionState.Closed
|
|
646
|
+
self._end(None)
|
|
603
647
|
peer, self._peer = self._peer, None
|
|
604
648
|
if peer is not None:
|
|
605
649
|
peer.fail_pending(ClientStoppedError())
|
|
@@ -707,6 +751,20 @@ class ExecutorTokenError(Exception):
|
|
|
707
751
|
"""POST /v1/executors/token failed."""
|
|
708
752
|
|
|
709
753
|
|
|
754
|
+
class CredentialRejectedError(ExecutorTokenError):
|
|
755
|
+
"""The token endpoint answered 401 or 403: the environment key or machine
|
|
756
|
+
token is invalid or revoked. A reconnect does not retry it."""
|
|
757
|
+
|
|
758
|
+
def __init__(self, message: str, status_code: int) -> None:
|
|
759
|
+
super().__init__(message)
|
|
760
|
+
self.status_code = status_code
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
class ReconnectExhaustedError(ExecutorError):
|
|
764
|
+
"""A positive max_reconnect_attempts failed in a row; ``__cause__`` is the
|
|
765
|
+
last failure."""
|
|
766
|
+
|
|
767
|
+
|
|
710
768
|
class ExecutorTokenScopeError(Exception):
|
|
711
769
|
"""The executor token does not allow the requested channel."""
|
|
712
770
|
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
"""Reconnect policy (Go reconnect_test.go): by default the client reconnects
|
|
2
|
+
until stop() through any outage; a credential the token endpoint rejects
|
|
3
|
+
(401/403) and an exhausted positive max_reconnect_attempts end the client,
|
|
4
|
+
and wait_closed() raises the reason instead of the client hanging."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
from typing import Any
|
|
10
|
+
|
|
11
|
+
import httpx
|
|
12
|
+
import pytest
|
|
13
|
+
|
|
14
|
+
import bma_executor_sdk as bx
|
|
15
|
+
from bma_executor_sdk import client as bxc
|
|
16
|
+
|
|
17
|
+
EXECUTOR = {"name": "py-test", "os": "linux", "arch": "arm64", "version": "0.1.0"}
|
|
18
|
+
Ready = bx.generated.ConnectionState.Ready
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class TokenEndpoint:
|
|
22
|
+
"""Answers the first token request, then fail_status for the next
|
|
23
|
+
fail_for requests (forever when fail_for < 0)."""
|
|
24
|
+
|
|
25
|
+
def __init__(self, fail_status: int, fail_for: int) -> None:
|
|
26
|
+
self.requests = 0
|
|
27
|
+
self.fail_status = fail_status
|
|
28
|
+
self.fail_for = fail_for
|
|
29
|
+
|
|
30
|
+
def __call__(self, request: httpx.Request) -> httpx.Response:
|
|
31
|
+
self.requests += 1
|
|
32
|
+
failed = self.requests - 1
|
|
33
|
+
if failed > 0 and (self.fail_for < 0 or failed <= self.fail_for):
|
|
34
|
+
return httpx.Response(self.fail_status, text="upstream unavailable")
|
|
35
|
+
return httpx.Response(200, json={"executor_id": "", "token": "jwt", "expires_in": 300, "ws_url": "inprocess://harness"})
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
async def start_client(harness: bx.Harness, endpoint: TokenEndpoint, **options: Any) -> bx.ExecutorClient:
|
|
39
|
+
client = bx.ExecutorClient(
|
|
40
|
+
environment_id="env_local", executor=EXECUTOR, environment_key="env_key", base_url="http://bma.test",
|
|
41
|
+
http_client=httpx.AsyncClient(transport=httpx.MockTransport(endpoint)), dialer=harness.inprocess_dialer(),
|
|
42
|
+
request_timeout=2.0, reconnect_base_delay=0.001, reconnect_max_delay=0.002, **options,
|
|
43
|
+
)
|
|
44
|
+
await client.start()
|
|
45
|
+
return client
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
async def drop(client: bx.ExecutorClient) -> None:
|
|
49
|
+
await client._peer.conn.close(1012, "node replaced") # noqa: SLF001
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def test_reconnects_after_a_long_outage_by_default() -> None:
|
|
53
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
54
|
+
endpoint = TokenEndpoint(503, 25)
|
|
55
|
+
client = await start_client(harness, endpoint)
|
|
56
|
+
first = client.connection_id
|
|
57
|
+
await drop(client)
|
|
58
|
+
for _ in range(500):
|
|
59
|
+
if client.state == Ready and client.connection_id != first:
|
|
60
|
+
break
|
|
61
|
+
await asyncio.sleep(0.01)
|
|
62
|
+
assert client.state == Ready and client.connection_id != first
|
|
63
|
+
assert endpoint.requests == 27 # the start, 25 failures, the reconnect
|
|
64
|
+
await client.stop()
|
|
65
|
+
await asyncio.wait_for(client.wait_closed(), 1) # stop() ends it without an error
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@pytest.mark.parametrize("status", [401, 403])
|
|
69
|
+
async def test_rejected_credential_ends_the_client(status: int) -> None:
|
|
70
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
71
|
+
endpoint = TokenEndpoint(status, -1)
|
|
72
|
+
client = await start_client(harness, endpoint)
|
|
73
|
+
await drop(client)
|
|
74
|
+
with pytest.raises(bx.CredentialRejectedError) as raised:
|
|
75
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
76
|
+
assert raised.value.status_code == status
|
|
77
|
+
assert endpoint.requests == 2 # no retry after the rejection
|
|
78
|
+
assert client.state == bx.generated.ConnectionState.Closed
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
async def test_exhausted_max_reconnect_attempts_ends_the_client() -> None:
|
|
82
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
83
|
+
endpoint = TokenEndpoint(503, -1)
|
|
84
|
+
client = await start_client(harness, endpoint, max_reconnect_attempts=3)
|
|
85
|
+
await drop(client)
|
|
86
|
+
with pytest.raises(bx.ReconnectExhaustedError) as raised:
|
|
87
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
88
|
+
assert isinstance(raised.value.__cause__, bx.ExecutorTokenError)
|
|
89
|
+
assert endpoint.requests == 4 # the start and 3 reconnect attempts
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
async def test_terminal_close_code_ends_the_client() -> None:
|
|
93
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
94
|
+
client = await start_client(harness, TokenEndpoint(503, 0))
|
|
95
|
+
await client._peer.conn.close(4004, "replaced") # noqa: SLF001
|
|
96
|
+
with pytest.raises(bx.ChannelClosed) as raised:
|
|
97
|
+
await asyncio.wait_for(client.wait_closed(), 2)
|
|
98
|
+
assert raised.value.code == 4004
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
async def test_heartbeat_timeout_reconnects(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
102
|
+
"""A heartbeat unanswered within request_timeout means a dead (half-open)
|
|
103
|
+
connection: the client closes it and reconnects (gRPC keepalive)."""
|
|
104
|
+
from bma_executor_sdk import harness as bxh
|
|
105
|
+
|
|
106
|
+
silent = {"on": False}
|
|
107
|
+
dispatch = bxh._HarnessConn._dispatch # noqa: SLF001
|
|
108
|
+
|
|
109
|
+
async def maybe_silent(self: Any, executor: Any, request_id: str, method: str, params: Any) -> Any:
|
|
110
|
+
if method == bx.generated.METHOD_EXECUTOR_HEARTBEAT and silent["on"]:
|
|
111
|
+
await asyncio.Event().wait()
|
|
112
|
+
return await dispatch(self, executor, request_id, method, params)
|
|
113
|
+
|
|
114
|
+
monkeypatch.setattr(bxh._HarnessConn, "_dispatch", maybe_silent) # noqa: SLF001
|
|
115
|
+
harness = bx.Harness(request_timeout=2.0, heartbeat_interval=0.02)
|
|
116
|
+
client = await start_client(harness, TokenEndpoint(503, 0))
|
|
117
|
+
client._request_timeout = 0.1 # noqa: SLF001
|
|
118
|
+
first = client.connection_id
|
|
119
|
+
silent["on"] = True
|
|
120
|
+
for _ in range(300):
|
|
121
|
+
if client.state == Ready and client.connection_id != first:
|
|
122
|
+
break
|
|
123
|
+
await asyncio.sleep(0.01)
|
|
124
|
+
assert client.state == Ready and client.connection_id != first
|
|
125
|
+
await client.stop()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_reconnect_defaults() -> None:
|
|
129
|
+
assert bxc.DEFAULT_MAX_RECONNECT_ATTEMPTS == 0
|
|
130
|
+
assert bxc.DEFAULT_RECONNECT_BASE_DELAY == 1.0
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
async def test_backoff_stays_capped_after_many_attempts() -> None:
|
|
134
|
+
harness = bx.Harness(request_timeout=2.0)
|
|
135
|
+
client = await start_client(harness, TokenEndpoint(503, 3))
|
|
136
|
+
first = client.connection_id
|
|
137
|
+
client._reconnect_attempts = 5000 # noqa: SLF001 hours into an outage
|
|
138
|
+
await drop(client)
|
|
139
|
+
for _ in range(300):
|
|
140
|
+
if client.state == Ready and client.connection_id != first:
|
|
141
|
+
break
|
|
142
|
+
await asyncio.sleep(0.01)
|
|
143
|
+
assert client.state == Ready and client.connection_id != first # no OverflowError in the backoff
|
|
144
|
+
await client.stop()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|