bma-executor-sdk 1.0.1__tar.gz → 1.0.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/PKG-INFO +4 -4
  2. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/README.md +3 -3
  3. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/executor_minimal.py +1 -1
  4. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/pyproject.toml +1 -1
  5. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/__init__.py +4 -0
  6. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/client.py +74 -16
  7. bma_executor_sdk-1.0.2/tests/test_reconnect.py +144 -0
  8. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/.gitignore +0 -0
  9. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/agent_minimal.py +0 -0
  10. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/examples/harness_loopback.py +0 -0
  11. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_methods.py +0 -0
  12. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_npipe.py +0 -0
  13. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/_rpc.py +0 -0
  14. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/agent.py +0 -0
  15. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/generated.py +0 -0
  16. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/harness.py +0 -0
  17. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/methods.py +0 -0
  18. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/py.typed +0 -0
  19. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/tools.py +0 -0
  20. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/src/bma_executor_sdk/transport.py +0 -0
  21. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/executor_helpers.py +0 -0
  22. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_base_url.py +0 -0
  23. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_close_codes.py +0 -0
  24. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_conformance.py +0 -0
  25. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_contract.py +0 -0
  26. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_interop.py +0 -0
  27. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_runtime.py +0 -0
  28. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_executor_token_refresh.py +0 -0
  29. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_npipe.py +0 -0
  30. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_review.py +0 -0
  31. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/test_typed_tools.py +0 -0
  32. {bma_executor_sdk-1.0.1 → bma_executor_sdk-1.0.2}/tests/typecheck/typed_tools.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: bma-executor-sdk
3
- Version: 1.0.1
3
+ Version: 1.0.2
4
4
  Summary: BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports.
5
5
  Project-URL: Repository, https://github.com/beeos-ai/bma-executor-sdk
6
6
  License-Expression: Apache-2.0
@@ -26,7 +26,7 @@ asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the B
26
26
  | Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
27
27
 
28
28
  ```sh
29
- pip install bma-executor-sdk==1.0.1
29
+ pip install bma-executor-sdk==1.0.2
30
30
  ```
31
31
 
32
32
  The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
@@ -53,7 +53,7 @@ async def main() -> None:
53
53
  return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
54
54
 
55
55
  await client.start()
56
- await asyncio.Event().wait()
56
+ await client.wait_closed() # raises when the client ends for good
57
57
 
58
58
  asyncio.run(main())
59
59
  ```
@@ -67,7 +67,7 @@ Behavior matches the Go SDK:
67
67
  - A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
68
68
  - `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
69
69
  - `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
70
- - Reconnect resumes with `resume.executor_id` / `active_sessions`; close codes 1008, 4002, 4003 and 4004 stop the client (4001 and 4005 reconnect).
70
+ - Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
71
71
  - Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
72
72
  - Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
73
73
 
@@ -15,7 +15,7 @@ asyncio executor (`ExecutorClient`), harness (`Harness`) and agent ends of the B
15
15
  | Integration guide | [docs/executor](../docs/executor/README.md) ([中文](../docs/executor/README.zh-CN.md)) |
16
16
 
17
17
  ```sh
18
- pip install bma-executor-sdk==1.0.1
18
+ pip install bma-executor-sdk==1.0.2
19
19
  ```
20
20
 
21
21
  The REST / façade client is the separate package [`bma-sdk`](https://github.com/beeos-ai/bma-sdk/tree/main/python).
@@ -42,7 +42,7 @@ async def main() -> None:
42
42
  return BashResult(stdout=out.decode(), stderr=err.decode(), exit_code=proc.returncode or 0)
43
43
 
44
44
  await client.start()
45
- await asyncio.Event().wait()
45
+ await client.wait_closed() # raises when the client ends for good
46
46
 
47
47
  asyncio.run(main())
48
48
  ```
@@ -56,7 +56,7 @@ Behavior matches the Go SDK:
56
56
  - A handler returns a string, content blocks, `ToolResult(content, is_error, artifacts)` or (bash) `BashResult`, rendered like the Go `Core`; `raise ToolError(text)` reports a tool-level error (`is_error: true`), any other exception is `outcome: "failed"`. The harness calls tools as `await session.tools.bash.call({"command": "ls"})`.
57
57
  - `tool.cancel` cancels the handler's task and reports `outcome: "canceled"`; duplicate `tool.call` ids run once.
58
58
  - `tool.result` retries `-32009` / `-32004` / `retryable` errors and hands a lost result to `on_tool_result_lost`.
59
- - Reconnect resumes with `resume.executor_id` / `active_sessions`; close codes 1008, 4002, 4003 and 4004 stop the client (4001 and 4005 reconnect).
59
+ - Reconnect resumes with `resume.executor_id` / `active_sessions`, until `stop()` by default (backoff 1 s doubling to 60 s, jitter); only close codes 4003 and 4004 stop the client. A positive `max_reconnect_attempts` bounds the attempts. `await client.wait_closed()` returns after `stop()` / `executor.close` and raises `ChannelClosed` (4003/4004), `CredentialRejectedError` (the token endpoint answered 401/403 on reconnect: revoked or invalid credential, not retried) or `ReconnectExhaustedError`; let it end the process so a supervisor can restart it.
60
60
  - Call `session_ready` after setup and `session_ended` after cleanup. `client.on_request(methods.PROCESS_START, handler)` serves `process.*`, `terminal.*`, `skill.install` or `workspace.backup` yourself with the contract's TypedDicts; `await client.call(methods.SESSION_EVENT, params)` / `await client.notify(methods.TOOL_PROGRESS, params)` send typed methods (`bma_executor_sdk.methods`). There is no execution `Core` in Python.
61
61
  - Input missing a required member fails the call before the handler runs; a `tool.call` for a tool without a handler is refused with `-32012` before the ack; `shell` / `files` are enabled while a handler of the group is registered unless set `enabled: False`, and a handler registered after `start()` sends `executor.capabilities.update`. `BashResult` beyond `INLINE_RESULT_BYTES` (or `truncated=True`) is cut to Go Core's preview. The `computer_*` tools get slots once the contract defines their input.
62
62
 
@@ -48,7 +48,7 @@ async def main() -> None:
48
48
  registered = await client.start()
49
49
  print("registered as", registered["executor_id"])
50
50
  try:
51
- await asyncio.Event().wait()
51
+ await client.wait_closed() # raises when the client ends for good
52
52
  finally:
53
53
  await client.stop("shutdown")
54
54
 
@@ -1,6 +1,6 @@
1
1
  [project]
2
2
  name = "bma-executor-sdk"
3
- version = "1.0.1"
3
+ version = "1.0.2"
4
4
  description = "BMA executor protocol (bma.executor.v1.1): executor, harness and agent ends on asyncio with WebSocket, Unix socket and in-process transports."
5
5
  readme = "README.md"
6
6
  license = "Apache-2.0"
@@ -12,11 +12,13 @@ from .client import (
12
12
  DEFAULT_BASE_URL,
13
13
  PROTOCOL_V1,
14
14
  PROTOCOL_V11,
15
+ CredentialRejectedError,
15
16
  ExecutorClient,
16
17
  ExecutorTokenError,
17
18
  ExecutorTokenScopeError,
18
19
  LogEvent,
19
20
  MCPChannel,
21
+ ReconnectExhaustedError,
20
22
  static_token,
21
23
  )
22
24
  from .generated import (
@@ -76,6 +78,7 @@ __all__ = [
76
78
  "ChannelClosed",
77
79
  "ClientStoppedError",
78
80
  "ComputerActionInput",
81
+ "CredentialRejectedError",
79
82
  "ComputerObserveInput",
80
83
  "ConnectRequest",
81
84
  "Connection",
@@ -98,6 +101,7 @@ __all__ = [
98
101
  "MCPChannel",
99
102
  "NotConnectedError",
100
103
  "Principal",
104
+ "ReconnectExhaustedError",
101
105
  "RPCError",
102
106
  "ReadInput",
103
107
  "Session",
@@ -16,7 +16,7 @@ from typing import TYPE_CHECKING, Any, TypeVar
16
16
  import httpx
17
17
 
18
18
  from ._methods import Call, Notification, Request
19
- from ._rpc import ClientStoppedError, NotConnectedError, Peer, RPCError
19
+ from ._rpc import ClientStoppedError, ExecutorError, NotConnectedError, Peer, RPCError
20
20
  from .generated import (
21
21
  METHOD_EXECUTOR_CAPABILITIES_UPDATE,
22
22
  METHOD_EXECUTOR_CLOSE,
@@ -64,8 +64,10 @@ if TYPE_CHECKING:
64
64
 
65
65
  DEFAULT_BASE_URL = "https://api.bma.beeos.ai"
66
66
  DEFAULT_REQUEST_TIMEOUT = 30.0
67
- DEFAULT_MAX_RECONNECT_ATTEMPTS = 10
68
- DEFAULT_RECONNECT_BASE_DELAY = 0.5
67
+ # 0 reconnects until stop(), as gRPC connection backoff and long-running
68
+ # agents (GitHub Actions runner, Buildkite agent) do.
69
+ DEFAULT_MAX_RECONNECT_ATTEMPTS = 0
70
+ DEFAULT_RECONNECT_BASE_DELAY = 1.0
69
71
  DEFAULT_TOOL_RESULT_RETRY_ATTEMPTS = 5
70
72
  DEFAULT_TOOL_RESULT_RETRY_BASE_DELAY = 0.25
71
73
  DEFAULT_HEARTBEAT_INTERVAL = 15.0
@@ -73,7 +75,7 @@ PROTOCOL_V1 = ProtocolVersion.BmaExecutorV1.value
73
75
  PROTOCOL_V11 = ProtocolVersion.BmaExecutorV11.value
74
76
  _MAX_TRACKED_TOOL_RESULTS = 8192
75
77
  _MAX_TOOL_RESULT_RETRY_DELAY = 10.0
76
- _MAX_RECONNECT_DELAY = 30.0
78
+ _MAX_RECONNECT_DELAY = 60.0
77
79
  # Fraction of a token's lifetime after which executor.token.refresh is sent.
78
80
  _TOKEN_REFRESH_AT = 0.7
79
81
 
@@ -172,6 +174,9 @@ class ExecutorClient:
172
174
  self._peer: Peer | None = None
173
175
  self._state = ConnectionState.Disconnected
174
176
  self._stopped = False
177
+ self._stop_called = False
178
+ self._closed = asyncio.Event()
179
+ self._close_error: BaseException | None = None
175
180
  self._register_result: ExecutorRegisterResult | None = None
176
181
  self._connection_id = ""
177
182
  self._executor_id = ""
@@ -215,11 +220,11 @@ class ExecutorClient:
215
220
 
216
221
  async def stop(self, reason: str = "") -> None:
217
222
  """Send executor.close, close the channel and fail pending calls."""
218
- if self._stopped:
223
+ if self._stop_called:
219
224
  return
220
- self._stopped = True
225
+ self._stop_called = True
226
+ self._end(None)
221
227
  peer, self._peer = self._peer, None
222
- self._state = ConnectionState.Closed
223
228
  if peer is not None:
224
229
  try:
225
230
  await asyncio.wait_for(peer.notify(METHOD_EXECUTOR_CLOSE, {"reason": reason}), 2)
@@ -236,6 +241,25 @@ class ExecutorClient:
236
241
  if self._tasks:
237
242
  await asyncio.gather(*self._tasks, return_exceptions=True)
238
243
 
244
+ async def wait_closed(self) -> None:
245
+ """Wait until the client stops for good (Go: ``Done`` and ``Err``).
246
+ Returns after stop() or executor.close; raises ChannelClosed for a
247
+ terminal close code (4003, 4004), CredentialRejectedError when the
248
+ token endpoint rejects the credential on reconnect, and
249
+ ReconnectExhaustedError after a positive max_reconnect_attempts."""
250
+ await self._closed.wait()
251
+ if self._close_error is not None:
252
+ raise self._close_error
253
+
254
+ def _end(self, error: BaseException | None) -> None:
255
+ """Stop for good; wait_closed() then returns or raises error."""
256
+ if self._closed.is_set():
257
+ return
258
+ self._stopped = True
259
+ self._state = ConnectionState.Closed
260
+ self._close_error = error
261
+ self._closed.set()
262
+
239
263
  @property
240
264
  def state(self) -> ConnectionState:
241
265
  return self._state
@@ -409,7 +433,10 @@ class ExecutorClient:
409
433
  if self._http_client is None:
410
434
  await client.aclose()
411
435
  if response.status_code != 200:
412
- raise ExecutorTokenError(f"token request failed: {response.status_code}: {response.text[:4096].strip()}")
436
+ message = f"token request failed: {response.status_code}: {response.text[:4096].strip()}"
437
+ if response.status_code in (401, 403):
438
+ raise CredentialRejectedError(message, response.status_code)
439
+ raise ExecutorTokenError(message)
413
440
  token = response.json()
414
441
  if not token.get("token") or not token.get("ws_url"):
415
442
  raise ExecutorTokenError("token response is missing token or ws_url")
@@ -510,24 +537,27 @@ class ExecutorClient:
510
537
  return
511
538
  self._peer = None
512
539
  if self._stopped:
513
- self._state = ConnectionState.Closed
514
540
  return
515
541
  if closed.code in TERMINAL_CLOSE_CODES:
516
- self._state = ConnectionState.Closed
517
542
  self._log("error", f"executor channel closed with terminal code {closed.code}: {closed.reason}")
543
+ self._end(closed)
518
544
  return
519
545
  self._state = ConnectionState.Disconnected
520
546
  self._spawn(self._reconnect_loop())
521
547
 
522
548
  async def _reconnect_loop(self) -> None:
549
+ cause: BaseException | None = None
523
550
  while not self._stopped:
524
- if self._reconnect_attempts >= self._max_reconnect_attempts:
525
- self._state = ConnectionState.Closed
526
- self._log("error", "executor reconnect attempts exhausted")
551
+ if 0 < self._max_reconnect_attempts <= self._reconnect_attempts:
552
+ error = ReconnectExhaustedError("executor reconnect attempts exhausted")
553
+ error.__cause__ = cause
554
+ self._log("error", "executor reconnect attempts exhausted", cause)
555
+ self._end(error)
527
556
  return
528
557
  attempt = self._reconnect_attempts
529
558
  self._reconnect_attempts += 1
530
- delay = min(self._reconnect_base_delay * (1 << attempt), self._reconnect_max_delay)
559
+ # The exponent is capped: unlimited attempts must not overflow the float.
560
+ delay = min(self._reconnect_base_delay * (1 << min(attempt, 32)), self._reconnect_max_delay)
531
561
  delay *= 0.5 + self._random() * 0.5
532
562
  self._log("info", f"reconnecting executor channel in {delay:.3f}s")
533
563
  await asyncio.sleep(delay)
@@ -538,7 +568,14 @@ class ExecutorClient:
538
568
  return
539
569
  except asyncio.CancelledError:
540
570
  raise
571
+ except CredentialRejectedError as error:
572
+ # Retrying cannot help a revoked or invalid credential (GitHub
573
+ # Actions runner and Buildkite agent stop on 401/403 too).
574
+ self._log("error", "executor credential rejected; not reconnecting", error)
575
+ self._end(error)
576
+ return
541
577
  except Exception as error:
578
+ cause = error
542
579
  self._state = ConnectionState.Disconnected
543
580
  self._log("warn", "executor reconnect failed", error)
544
581
 
@@ -552,10 +589,18 @@ class ExecutorClient:
552
589
  load = self._heartbeat_load()
553
590
  if load is not None:
554
591
  params["load"] = load
592
+ peer = self._peer
555
593
  try:
556
594
  result = await self._call(METHOD_EXECUTOR_HEARTBEAT, params)
557
595
  except (RPCError, NotConnectedError, TimeoutError) as error:
558
596
  self._log("warn", "executor heartbeat failed", error)
597
+ if isinstance(error, TimeoutError) and peer is not None and peer is self._peer:
598
+ # No answer within request_timeout: the connection is dead
599
+ # (a half-open TCP connection after a network cut). Close it
600
+ # so the client reconnects, as gRPC keepalive closes a
601
+ # transport whose ping times out.
602
+ self._log("warn", "executor heartbeat timed out; closing the connection to reconnect")
603
+ await peer.conn.close(1001, "heartbeat timeout")
559
604
  continue
560
605
  if result.get("drain") and self._state == ConnectionState.Ready:
561
606
  self._state = ConnectionState.Draining
@@ -598,8 +643,7 @@ class ExecutorClient:
598
643
  def _dispatch_notification(self, method: str, params: Any) -> None:
599
644
  if method == METHOD_EXECUTOR_CLOSE:
600
645
  # The harness closed the executor deliberately: no reconnect.
601
- self._stopped = True
602
- self._state = ConnectionState.Closed
646
+ self._end(None)
603
647
  peer, self._peer = self._peer, None
604
648
  if peer is not None:
605
649
  peer.fail_pending(ClientStoppedError())
@@ -707,6 +751,20 @@ class ExecutorTokenError(Exception):
707
751
  """POST /v1/executors/token failed."""
708
752
 
709
753
 
754
+ class CredentialRejectedError(ExecutorTokenError):
755
+ """The token endpoint answered 401 or 403: the environment key or machine
756
+ token is invalid or revoked. A reconnect does not retry it."""
757
+
758
+ def __init__(self, message: str, status_code: int) -> None:
759
+ super().__init__(message)
760
+ self.status_code = status_code
761
+
762
+
763
+ class ReconnectExhaustedError(ExecutorError):
764
+ """A positive max_reconnect_attempts failed in a row; ``__cause__`` is the
765
+ last failure."""
766
+
767
+
710
768
  class ExecutorTokenScopeError(Exception):
711
769
  """The executor token does not allow the requested channel."""
712
770
 
@@ -0,0 +1,144 @@
1
+ """Reconnect policy (Go reconnect_test.go): by default the client reconnects
2
+ until stop() through any outage; a credential the token endpoint rejects
3
+ (401/403) and an exhausted positive max_reconnect_attempts end the client,
4
+ and wait_closed() raises the reason instead of the client hanging."""
5
+
6
+ from __future__ import annotations
7
+
8
+ import asyncio
9
+ from typing import Any
10
+
11
+ import httpx
12
+ import pytest
13
+
14
+ import bma_executor_sdk as bx
15
+ from bma_executor_sdk import client as bxc
16
+
17
+ EXECUTOR = {"name": "py-test", "os": "linux", "arch": "arm64", "version": "0.1.0"}
18
+ Ready = bx.generated.ConnectionState.Ready
19
+
20
+
21
+ class TokenEndpoint:
22
+ """Answers the first token request, then fail_status for the next
23
+ fail_for requests (forever when fail_for < 0)."""
24
+
25
+ def __init__(self, fail_status: int, fail_for: int) -> None:
26
+ self.requests = 0
27
+ self.fail_status = fail_status
28
+ self.fail_for = fail_for
29
+
30
+ def __call__(self, request: httpx.Request) -> httpx.Response:
31
+ self.requests += 1
32
+ failed = self.requests - 1
33
+ if failed > 0 and (self.fail_for < 0 or failed <= self.fail_for):
34
+ return httpx.Response(self.fail_status, text="upstream unavailable")
35
+ return httpx.Response(200, json={"executor_id": "", "token": "jwt", "expires_in": 300, "ws_url": "inprocess://harness"})
36
+
37
+
38
+ async def start_client(harness: bx.Harness, endpoint: TokenEndpoint, **options: Any) -> bx.ExecutorClient:
39
+ client = bx.ExecutorClient(
40
+ environment_id="env_local", executor=EXECUTOR, environment_key="env_key", base_url="http://bma.test",
41
+ http_client=httpx.AsyncClient(transport=httpx.MockTransport(endpoint)), dialer=harness.inprocess_dialer(),
42
+ request_timeout=2.0, reconnect_base_delay=0.001, reconnect_max_delay=0.002, **options,
43
+ )
44
+ await client.start()
45
+ return client
46
+
47
+
48
+ async def drop(client: bx.ExecutorClient) -> None:
49
+ await client._peer.conn.close(1012, "node replaced") # noqa: SLF001
50
+
51
+
52
+ async def test_reconnects_after_a_long_outage_by_default() -> None:
53
+ harness = bx.Harness(request_timeout=2.0)
54
+ endpoint = TokenEndpoint(503, 25)
55
+ client = await start_client(harness, endpoint)
56
+ first = client.connection_id
57
+ await drop(client)
58
+ for _ in range(500):
59
+ if client.state == Ready and client.connection_id != first:
60
+ break
61
+ await asyncio.sleep(0.01)
62
+ assert client.state == Ready and client.connection_id != first
63
+ assert endpoint.requests == 27 # the start, 25 failures, the reconnect
64
+ await client.stop()
65
+ await asyncio.wait_for(client.wait_closed(), 1) # stop() ends it without an error
66
+
67
+
68
+ @pytest.mark.parametrize("status", [401, 403])
69
+ async def test_rejected_credential_ends_the_client(status: int) -> None:
70
+ harness = bx.Harness(request_timeout=2.0)
71
+ endpoint = TokenEndpoint(status, -1)
72
+ client = await start_client(harness, endpoint)
73
+ await drop(client)
74
+ with pytest.raises(bx.CredentialRejectedError) as raised:
75
+ await asyncio.wait_for(client.wait_closed(), 2)
76
+ assert raised.value.status_code == status
77
+ assert endpoint.requests == 2 # no retry after the rejection
78
+ assert client.state == bx.generated.ConnectionState.Closed
79
+
80
+
81
+ async def test_exhausted_max_reconnect_attempts_ends_the_client() -> None:
82
+ harness = bx.Harness(request_timeout=2.0)
83
+ endpoint = TokenEndpoint(503, -1)
84
+ client = await start_client(harness, endpoint, max_reconnect_attempts=3)
85
+ await drop(client)
86
+ with pytest.raises(bx.ReconnectExhaustedError) as raised:
87
+ await asyncio.wait_for(client.wait_closed(), 2)
88
+ assert isinstance(raised.value.__cause__, bx.ExecutorTokenError)
89
+ assert endpoint.requests == 4 # the start and 3 reconnect attempts
90
+
91
+
92
+ async def test_terminal_close_code_ends_the_client() -> None:
93
+ harness = bx.Harness(request_timeout=2.0)
94
+ client = await start_client(harness, TokenEndpoint(503, 0))
95
+ await client._peer.conn.close(4004, "replaced") # noqa: SLF001
96
+ with pytest.raises(bx.ChannelClosed) as raised:
97
+ await asyncio.wait_for(client.wait_closed(), 2)
98
+ assert raised.value.code == 4004
99
+
100
+
101
+ async def test_heartbeat_timeout_reconnects(monkeypatch: pytest.MonkeyPatch) -> None:
102
+ """A heartbeat unanswered within request_timeout means a dead (half-open)
103
+ connection: the client closes it and reconnects (gRPC keepalive)."""
104
+ from bma_executor_sdk import harness as bxh
105
+
106
+ silent = {"on": False}
107
+ dispatch = bxh._HarnessConn._dispatch # noqa: SLF001
108
+
109
+ async def maybe_silent(self: Any, executor: Any, request_id: str, method: str, params: Any) -> Any:
110
+ if method == bx.generated.METHOD_EXECUTOR_HEARTBEAT and silent["on"]:
111
+ await asyncio.Event().wait()
112
+ return await dispatch(self, executor, request_id, method, params)
113
+
114
+ monkeypatch.setattr(bxh._HarnessConn, "_dispatch", maybe_silent) # noqa: SLF001
115
+ harness = bx.Harness(request_timeout=2.0, heartbeat_interval=0.02)
116
+ client = await start_client(harness, TokenEndpoint(503, 0))
117
+ client._request_timeout = 0.1 # noqa: SLF001
118
+ first = client.connection_id
119
+ silent["on"] = True
120
+ for _ in range(300):
121
+ if client.state == Ready and client.connection_id != first:
122
+ break
123
+ await asyncio.sleep(0.01)
124
+ assert client.state == Ready and client.connection_id != first
125
+ await client.stop()
126
+
127
+
128
+ def test_reconnect_defaults() -> None:
129
+ assert bxc.DEFAULT_MAX_RECONNECT_ATTEMPTS == 0
130
+ assert bxc.DEFAULT_RECONNECT_BASE_DELAY == 1.0
131
+
132
+
133
+ async def test_backoff_stays_capped_after_many_attempts() -> None:
134
+ harness = bx.Harness(request_timeout=2.0)
135
+ client = await start_client(harness, TokenEndpoint(503, 3))
136
+ first = client.connection_id
137
+ client._reconnect_attempts = 5000 # noqa: SLF001 hours into an outage
138
+ await drop(client)
139
+ for _ in range(300):
140
+ if client.state == Ready and client.connection_id != first:
141
+ break
142
+ await asyncio.sleep(0.01)
143
+ assert client.state == Ready and client.connection_id != first # no OverflowError in the backoff
144
+ await client.stop()