relay-codex 0.7.3 → 0.7.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/publishing.md +2 -2
- package/docs/troubleshooting.md +12 -3
- package/package.json +1 -1
- package/python/broker.py +5 -0
- package/python/execution.py +1 -1
- package/python/pool.py +7 -0
- package/python/router.py +209 -21
- package/python/service.py +9 -3
- package/tests/fake_native.py +34 -0
- package/tests/test_recovery_boundary.py +422 -0
package/docs/publishing.md
CHANGED
|
@@ -8,8 +8,8 @@ Friends can run `npx relay-codex` or install it with `npm install -g relay-codex
|
|
|
8
8
|
The GitHub repository is enough for friends to install or try Relay:
|
|
9
9
|
|
|
10
10
|
```sh
|
|
11
|
-
npm install -g github:blackspider-ops/relay#v0.7.
|
|
12
|
-
npx --yes github:blackspider-ops/relay#v0.7.
|
|
11
|
+
npm install -g github:blackspider-ops/relay#v0.7.4
|
|
12
|
+
npx --yes github:blackspider-ops/relay#v0.7.4 --version
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
`npx` runs packages; it is not a separate publishing service. Publishing
|
package/docs/troubleshooting.md
CHANGED
|
@@ -51,13 +51,22 @@ visible. Use Restart for a stopped worker rather than sending it live controls.
|
|
|
51
51
|
|
|
52
52
|
**A usage-limit panel remains open and Relay says “waiting for activity.”**
|
|
53
53
|
|
|
54
|
-
Update Relay if you are using a version older than 0.7.
|
|
55
|
-
|
|
56
|
-
native history
|
|
54
|
+
Update Relay if you are using a version older than 0.7.4. Background terminals
|
|
55
|
+
and missed tool notifications could leave a finished turn marked as active.
|
|
56
|
+
Relay checks native history and the current backend's status. Commands explicitly
|
|
57
|
+
listed by Codex as background terminals can keep running while accounts change
|
|
58
|
+
after the model turn ends. The usage panel does not need to be clicked. Relay keeps
|
|
57
59
|
waiting when work is still running or completion cannot be verified; approvals
|
|
58
60
|
and paused recovery still need attention. Existing workers retain the code they
|
|
59
61
|
started with, so an update takes effect after that worker is relaunched.
|
|
60
62
|
|
|
63
|
+
**Codex says “auth refresh request failed: code=-32000.”**
|
|
64
|
+
|
|
65
|
+
Relay retries authentication on its normal retry schedule, then tries another
|
|
66
|
+
available account. If Codex reports that a refresh token was revoked, sign in
|
|
67
|
+
again with `relay account reauth LABEL`. Switch any sessions using that account
|
|
68
|
+
first. A revoked login cannot be repaired by retrying the same token.
|
|
69
|
+
|
|
61
70
|
**Tab completion is missing.**
|
|
62
71
|
|
|
63
72
|
Load `source <(relay completion zsh)` after `compinit`, or use
|
package/package.json
CHANGED
package/python/broker.py
CHANGED
|
@@ -225,6 +225,11 @@ async def serve(state, binary):
|
|
|
225
225
|
):
|
|
226
226
|
value = candidate
|
|
227
227
|
if value is None:
|
|
228
|
+
if request.get("refresh"):
|
|
229
|
+
# A retained native manager can hold a refresh
|
|
230
|
+
# grant rotated by another process. Reload the
|
|
231
|
+
# Keychain under this account's shared lock.
|
|
232
|
+
await account.close()
|
|
228
233
|
await account.start()
|
|
229
234
|
result = await account.rpc.call(
|
|
230
235
|
"account/read",
|
package/python/execution.py
CHANGED
|
@@ -350,7 +350,7 @@ async def _run(args, state, binary):
|
|
|
350
350
|
controller.launch_overrides = overrides
|
|
351
351
|
controller.client = output
|
|
352
352
|
output.controller = controller
|
|
353
|
-
backend.handler = controller.event
|
|
353
|
+
backend.handler = lambda message: controller.event(message, origin=backend)
|
|
354
354
|
init = await backend.initialize()
|
|
355
355
|
heartbeat = asyncio.create_task(controller.maintain())
|
|
356
356
|
model = values.get("--model")
|
package/python/pool.py
CHANGED
|
@@ -545,6 +545,13 @@ CAPACITY_MESSAGE = "Selected model is at capacity. Please try a different model.
|
|
|
545
545
|
|
|
546
546
|
|
|
547
547
|
def failure_kind(info, message=None):
|
|
548
|
+
if (
|
|
549
|
+
info in (None, "other", "unauthorized")
|
|
550
|
+
and isinstance(message, str)
|
|
551
|
+
and message.strip().removeprefix("■").strip().lower()
|
|
552
|
+
== "auth refresh request failed: code=-32000"
|
|
553
|
+
):
|
|
554
|
+
return "auth"
|
|
548
555
|
if info == "policyCapacityUnavailable":
|
|
549
556
|
return "capacity"
|
|
550
557
|
# Older/generic error envelopes can omit the enum. Match only the known
|
package/python/router.py
CHANGED
|
@@ -24,7 +24,7 @@ from pool import Pool, blocked, reset_time, failure_kind, file_lock
|
|
|
24
24
|
from launch import translate, RESUME_FIELDS
|
|
25
25
|
import settings
|
|
26
26
|
|
|
27
|
-
VERSION = "0.7.
|
|
27
|
+
VERSION = "0.7.4"
|
|
28
28
|
_support = Path.home() / "Library/Application Support"
|
|
29
29
|
_legacy = _support / "Codex Accounts"
|
|
30
30
|
_default_state = _legacy if (_legacy / "accounts").exists() else _support / "Relay"
|
|
@@ -95,6 +95,17 @@ def clean_env(home: Path) -> dict:
|
|
|
95
95
|
return env
|
|
96
96
|
|
|
97
97
|
|
|
98
|
+
def rejected_failure_kind(payload):
|
|
99
|
+
if payload.get("code") == -32001:
|
|
100
|
+
return "transient"
|
|
101
|
+
if (
|
|
102
|
+
payload.get("code") in {-32000, -32603}
|
|
103
|
+
and failure_kind("other", payload.get("message")) == "auth"
|
|
104
|
+
):
|
|
105
|
+
return "auth"
|
|
106
|
+
return None
|
|
107
|
+
|
|
108
|
+
|
|
98
109
|
def codex_path() -> str:
|
|
99
110
|
path = shutil.which("codex")
|
|
100
111
|
if not path:
|
|
@@ -193,6 +204,9 @@ class RPC:
|
|
|
193
204
|
self.reader = asyncio.create_task(self.read_loop())
|
|
194
205
|
self.startup_guard = None
|
|
195
206
|
|
|
207
|
+
def disconnected(self):
|
|
208
|
+
return self.proc.returncode is not None or self.reader.done()
|
|
209
|
+
|
|
196
210
|
@classmethod
|
|
197
211
|
async def start(
|
|
198
212
|
cls, binary: str, home: Path, *, helper=False, handler=None, extra_args=None
|
|
@@ -330,7 +344,7 @@ class Account:
|
|
|
330
344
|
self.lock = asyncio.Lock()
|
|
331
345
|
|
|
332
346
|
async def start(self):
|
|
333
|
-
if self.rpc and self.rpc.
|
|
347
|
+
if self.rpc and self.rpc.disconnected():
|
|
334
348
|
await self.rpc.close()
|
|
335
349
|
self.rpc = None
|
|
336
350
|
if not self.rpc:
|
|
@@ -384,6 +398,7 @@ class Account:
|
|
|
384
398
|
async def close(self):
|
|
385
399
|
if self.rpc:
|
|
386
400
|
await self.rpc.close()
|
|
401
|
+
self.rpc = None
|
|
387
402
|
|
|
388
403
|
|
|
389
404
|
class Controller:
|
|
@@ -433,9 +448,11 @@ class Controller:
|
|
|
433
448
|
self.turn_ids = {}
|
|
434
449
|
self.recovery_jobs = {}
|
|
435
450
|
self.pending_intents = {}
|
|
451
|
+
self.rejected_turns = {}
|
|
436
452
|
self.active_items = set()
|
|
437
453
|
self.active_item_threads = {}
|
|
438
454
|
self.item_revision = 0
|
|
455
|
+
self.auth_account = None
|
|
439
456
|
self.backend_factory = None
|
|
440
457
|
self.restart_lock = asyncio.Lock()
|
|
441
458
|
self.restart_task = None
|
|
@@ -675,6 +692,7 @@ class Controller:
|
|
|
675
692
|
model=next((self.model_for(t) for t in self.last_turn), None),
|
|
676
693
|
paused=self.paused,
|
|
677
694
|
background=self.background,
|
|
695
|
+
activity=self.activity_snapshot(),
|
|
678
696
|
)
|
|
679
697
|
last_heartbeat = now
|
|
680
698
|
new_revision = self.pool.revision()
|
|
@@ -743,7 +761,7 @@ class Controller:
|
|
|
743
761
|
pass
|
|
744
762
|
if now - last_watch >= self.preferences["watchdog_interval"]:
|
|
745
763
|
last_watch = now
|
|
746
|
-
if self.
|
|
764
|
+
if self.backend_disconnected() and self.backend_factory:
|
|
747
765
|
if not self.restart_task or self.restart_task.done():
|
|
748
766
|
self.restart_task = asyncio.create_task(self.restart_backend())
|
|
749
767
|
elif self.active_threads:
|
|
@@ -805,6 +823,7 @@ class Controller:
|
|
|
805
823
|
)
|
|
806
824
|
if index is None:
|
|
807
825
|
raise ControlRejected("unknown_account")
|
|
826
|
+
await self.reconcile_active_items()
|
|
808
827
|
if self.active_threads or self.active_items or self.server_requests:
|
|
809
828
|
raise ControlRejected("busy")
|
|
810
829
|
options = self.pool.options(label) if self.pool else {}
|
|
@@ -859,20 +878,24 @@ class Controller:
|
|
|
859
878
|
|
|
860
879
|
async def restart_backend(self):
|
|
861
880
|
async with self.restart_lock:
|
|
862
|
-
if self.
|
|
881
|
+
if not self.backend_disconnected():
|
|
863
882
|
return
|
|
864
883
|
self.note("restarting")
|
|
865
884
|
in_flight = bool(self.active_items or self.server_requests)
|
|
866
885
|
old = self.backend
|
|
886
|
+
old_auth_account = self.auth_account
|
|
867
887
|
candidate = None
|
|
868
888
|
try:
|
|
869
889
|
candidate = await self.backend_factory()
|
|
870
|
-
candidate.handler = self.event
|
|
890
|
+
candidate.handler = lambda message: self.event(
|
|
891
|
+
message, origin=candidate
|
|
892
|
+
)
|
|
871
893
|
await candidate.initialize()
|
|
872
894
|
self.backend = candidate
|
|
873
895
|
await self.login(self.index)
|
|
874
896
|
except Exception:
|
|
875
897
|
self.backend = old
|
|
898
|
+
self.auth_account = old_auth_account
|
|
876
899
|
if candidate:
|
|
877
900
|
await candidate.close()
|
|
878
901
|
self.note("backend_restart_failed", reason="startup_or_login_failed")
|
|
@@ -966,9 +989,17 @@ class Controller:
|
|
|
966
989
|
)
|
|
967
990
|
):
|
|
968
991
|
raise ControlRejected("account_unavailable")
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
992
|
+
# Native refresh callbacks can arrive before the login response.
|
|
993
|
+
# Bind them to the credentials being installed, not the old index.
|
|
994
|
+
previous_binding = self.auth_account
|
|
995
|
+
self.auth_account = self.accounts[index]
|
|
996
|
+
try:
|
|
997
|
+
await self.backend.call(
|
|
998
|
+
"account/login/start", {"type": "chatgptAuthTokens", **credentials}
|
|
999
|
+
)
|
|
1000
|
+
except RemoteError:
|
|
1001
|
+
self.auth_account = previous_binding
|
|
1002
|
+
raise
|
|
972
1003
|
self.index = index
|
|
973
1004
|
# Acceptance of token syntax alone isn't a verified, usable login.
|
|
974
1005
|
result = await self.backend.call("account/read", {"refreshToken": False})
|
|
@@ -984,25 +1015,34 @@ class Controller:
|
|
|
984
1015
|
except Exception:
|
|
985
1016
|
pass
|
|
986
1017
|
|
|
987
|
-
async def event(self, message):
|
|
1018
|
+
async def event(self, message, *, origin=None):
|
|
1019
|
+
if origin is not None and origin is not self.backend:
|
|
1020
|
+
return
|
|
988
1021
|
method, params = message.get("method"), message.get("params", {})
|
|
989
1022
|
if method == "account/chatgptAuthTokens/refresh":
|
|
1023
|
+
account = self.auth_account or self.account
|
|
1024
|
+
transport = self.backend
|
|
990
1025
|
try:
|
|
991
1026
|
# Never return another identity in response to a refresh request.
|
|
992
|
-
credentials = await
|
|
1027
|
+
credentials = await account.credentials(refresh=True)
|
|
993
1028
|
previous = params.get("previousAccountId")
|
|
994
1029
|
if previous and previous != credentials["chatgptAccountId"]:
|
|
995
1030
|
raise RouterError(
|
|
996
1031
|
"Refresh account mismatch; re-enroll the account."
|
|
997
1032
|
)
|
|
998
|
-
|
|
1033
|
+
if transport is not self.backend:
|
|
1034
|
+
return
|
|
1035
|
+
await transport.send({"id": message["id"], "result": credentials})
|
|
999
1036
|
except (RouterError, asyncio.TimeoutError):
|
|
1000
|
-
|
|
1037
|
+
if transport is not self.backend:
|
|
1038
|
+
return
|
|
1039
|
+
self.note("auth_refresh_failed", account=account.label)
|
|
1040
|
+
await transport.send(
|
|
1001
1041
|
{
|
|
1002
1042
|
"id": message["id"],
|
|
1003
1043
|
"error": {
|
|
1004
1044
|
"code": -32000,
|
|
1005
|
-
"message": "Account
|
|
1045
|
+
"message": "Account credentials could not be refreshed. Relay will retry after this turn ends.",
|
|
1006
1046
|
},
|
|
1007
1047
|
}
|
|
1008
1048
|
)
|
|
@@ -1024,6 +1064,7 @@ class Controller:
|
|
|
1024
1064
|
if self.pool and thread.get("id"):
|
|
1025
1065
|
self.pool.claim(thread["id"])
|
|
1026
1066
|
if method == "turn/started":
|
|
1067
|
+
self.rejected_turns.pop(params["threadId"], None)
|
|
1027
1068
|
self.active_threads.add(params["threadId"])
|
|
1028
1069
|
self.turn_ids[params["threadId"]] = params.get("turn", {}).get("id")
|
|
1029
1070
|
if params["threadId"] in self.pending_intents:
|
|
@@ -1034,6 +1075,7 @@ class Controller:
|
|
|
1034
1075
|
accepted=True,
|
|
1035
1076
|
turn=params.get("turn", {}).get("id"),
|
|
1036
1077
|
state="working",
|
|
1078
|
+
rejected=False,
|
|
1037
1079
|
)
|
|
1038
1080
|
elif method == "item/started" and params.get("item", {}).get("type") in {
|
|
1039
1081
|
"commandExecution",
|
|
@@ -1206,7 +1248,9 @@ class Controller:
|
|
|
1206
1248
|
await self.reconcile(thread)
|
|
1207
1249
|
if time.monotonic() - began > self.preferences["idle_warning_after"]:
|
|
1208
1250
|
self.note(
|
|
1209
|
-
"waiting_for_activity",
|
|
1251
|
+
"waiting_for_activity",
|
|
1252
|
+
reason="actions_or_approvals_not_idle",
|
|
1253
|
+
activity=self.activity_snapshot(),
|
|
1210
1254
|
)
|
|
1211
1255
|
if self.paused:
|
|
1212
1256
|
self.wake.clear()
|
|
@@ -1217,7 +1261,7 @@ class Controller:
|
|
|
1217
1261
|
return generation == self.user_generation and self.attached()
|
|
1218
1262
|
|
|
1219
1263
|
async def reconcile_active_items(self):
|
|
1220
|
-
"""
|
|
1264
|
+
"""Reconcile finished actions and native background jobs at idle boundaries."""
|
|
1221
1265
|
owners = dict(self.active_item_threads)
|
|
1222
1266
|
revision = self.item_revision
|
|
1223
1267
|
for thread in set(owners.values()):
|
|
@@ -1226,6 +1270,10 @@ class Controller:
|
|
|
1226
1270
|
for item, owner in owners.items()
|
|
1227
1271
|
if owner == thread and item in self.active_items
|
|
1228
1272
|
}
|
|
1273
|
+
if not pending:
|
|
1274
|
+
continue
|
|
1275
|
+
await self.reconcile_background_activity(thread, pending, revision)
|
|
1276
|
+
pending &= self.active_items
|
|
1229
1277
|
if not pending:
|
|
1230
1278
|
continue
|
|
1231
1279
|
cursor = None
|
|
@@ -1279,6 +1327,103 @@ class Controller:
|
|
|
1279
1327
|
break
|
|
1280
1328
|
visited.add(cursor)
|
|
1281
1329
|
|
|
1330
|
+
def backend_disconnected(self):
|
|
1331
|
+
check = getattr(self.backend, "disconnected", None)
|
|
1332
|
+
return (
|
|
1333
|
+
check()
|
|
1334
|
+
if check
|
|
1335
|
+
else getattr(getattr(self.backend, "proc", None), "returncode", None)
|
|
1336
|
+
is not None
|
|
1337
|
+
)
|
|
1338
|
+
|
|
1339
|
+
def activity_snapshot(self):
|
|
1340
|
+
"""Private diagnostic metadata; never tool arguments or credentials."""
|
|
1341
|
+
return {
|
|
1342
|
+
"threads": sorted(self.active_threads),
|
|
1343
|
+
"items": sorted(self.active_items),
|
|
1344
|
+
"owners": dict(self.active_item_threads),
|
|
1345
|
+
"requests": sorted(
|
|
1346
|
+
{
|
|
1347
|
+
m.get("method", "unknown")
|
|
1348
|
+
for m in self.pending_server_requests.values()
|
|
1349
|
+
}
|
|
1350
|
+
),
|
|
1351
|
+
"request_count": len(self.server_requests),
|
|
1352
|
+
"paused": self.paused,
|
|
1353
|
+
"disconnected": self.backend_disconnected(),
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
async def reconcile_background_activity(self, thread, pending, revision):
|
|
1357
|
+
"""After a turn ends, known background terminals need not block login.
|
|
1358
|
+
|
|
1359
|
+
These jobs remain alive in the same backend. Unknown tools and pending
|
|
1360
|
+
approvals still block recovery; a terminal turn alone cannot clear them.
|
|
1361
|
+
"""
|
|
1362
|
+
turn_id = self.turn_ids.get(thread)
|
|
1363
|
+
generation = self.user_generation
|
|
1364
|
+
if not turn_id or self.server_requests:
|
|
1365
|
+
return
|
|
1366
|
+
try:
|
|
1367
|
+
turn = await self.latest(thread) or {}
|
|
1368
|
+
if turn.get("id") != turn_id or turn.get("status") not in {
|
|
1369
|
+
"completed",
|
|
1370
|
+
"failed",
|
|
1371
|
+
"interrupted",
|
|
1372
|
+
}:
|
|
1373
|
+
return
|
|
1374
|
+
background = set()
|
|
1375
|
+
cursor = None
|
|
1376
|
+
visited = set()
|
|
1377
|
+
for _ in range(8):
|
|
1378
|
+
params = {"threadId": thread, "limit": 100}
|
|
1379
|
+
if cursor:
|
|
1380
|
+
params["cursor"] = cursor
|
|
1381
|
+
result = await self.backend.call(
|
|
1382
|
+
"thread/backgroundTerminals/list", params, timeout=15
|
|
1383
|
+
)
|
|
1384
|
+
background.update(item.get("itemId") for item in result.get("data", []))
|
|
1385
|
+
cursor = result.get("nextCursor")
|
|
1386
|
+
if not cursor or cursor in visited or pending <= background:
|
|
1387
|
+
break
|
|
1388
|
+
visited.add(cursor)
|
|
1389
|
+
if not pending & background:
|
|
1390
|
+
return
|
|
1391
|
+
info = await self.backend.call(
|
|
1392
|
+
"thread/read", {"threadId": thread}, timeout=15
|
|
1393
|
+
)
|
|
1394
|
+
except (RouterError, asyncio.TimeoutError):
|
|
1395
|
+
return
|
|
1396
|
+
if (
|
|
1397
|
+
revision != self.item_revision
|
|
1398
|
+
or generation != self.user_generation
|
|
1399
|
+
or self.turn_ids.get(thread) != turn_id
|
|
1400
|
+
or self.server_requests
|
|
1401
|
+
or info.get("thread", {}).get("status", {}).get("type")
|
|
1402
|
+
not in {"idle", "systemError"}
|
|
1403
|
+
or info.get("thread", {}).get("canAcceptDirectInput") is False
|
|
1404
|
+
):
|
|
1405
|
+
return
|
|
1406
|
+
cleared = {
|
|
1407
|
+
item
|
|
1408
|
+
for item in pending & background
|
|
1409
|
+
if self.active_item_threads.get(item) == thread
|
|
1410
|
+
and item in self.active_items
|
|
1411
|
+
}
|
|
1412
|
+
for item in cleared:
|
|
1413
|
+
self.active_items.discard(item)
|
|
1414
|
+
self.active_item_threads.pop(item, None)
|
|
1415
|
+
if cleared:
|
|
1416
|
+
self.active_threads.discard(thread)
|
|
1417
|
+
self.persist(thread, active_items=bool(self.active_items))
|
|
1418
|
+
if self.pool:
|
|
1419
|
+
self.pool.record(
|
|
1420
|
+
"background_activity_reconciled",
|
|
1421
|
+
count=len(cleared),
|
|
1422
|
+
thread=thread,
|
|
1423
|
+
turn=turn_id,
|
|
1424
|
+
)
|
|
1425
|
+
self.status_changed.set()
|
|
1426
|
+
|
|
1282
1427
|
async def continue_thread(self, thread, generation):
|
|
1283
1428
|
if not self.can_continue(thread, generation):
|
|
1284
1429
|
return
|
|
@@ -1295,6 +1440,19 @@ class Controller:
|
|
|
1295
1440
|
params[key] = self.thread_settings[thread][key]
|
|
1296
1441
|
params["clientUserMessageId"] = str(uuid.uuid4())
|
|
1297
1442
|
baseline = self.turn_ids.get(thread) or "__unknown_before_submission__"
|
|
1443
|
+
rejected = self.rejected_turns.get(thread)
|
|
1444
|
+
if rejected:
|
|
1445
|
+
baseline = rejected["baseline"]
|
|
1446
|
+
try:
|
|
1447
|
+
latest = await self.latest(thread)
|
|
1448
|
+
except (RouterError, asyncio.TimeoutError):
|
|
1449
|
+
self.hold_for_review(thread, "rejected_request_history_unavailable")
|
|
1450
|
+
return
|
|
1451
|
+
observed = (latest or {}).get("id") or "__unknown_before_submission__"
|
|
1452
|
+
if observed != baseline:
|
|
1453
|
+
self.hold_for_review(thread, "rejected_request_history_changed")
|
|
1454
|
+
return
|
|
1455
|
+
params = dict(rejected["params"])
|
|
1298
1456
|
try:
|
|
1299
1457
|
while True:
|
|
1300
1458
|
async with self.login_lock:
|
|
@@ -1312,6 +1470,7 @@ class Controller:
|
|
|
1312
1470
|
accepted=False,
|
|
1313
1471
|
state="submitting",
|
|
1314
1472
|
baseline=baseline,
|
|
1473
|
+
rejected=bool(rejected),
|
|
1315
1474
|
)
|
|
1316
1475
|
result = await self.backend.call("turn/start", params)
|
|
1317
1476
|
break
|
|
@@ -1319,6 +1478,7 @@ class Controller:
|
|
|
1319
1478
|
return
|
|
1320
1479
|
turn = result.get("turn", {})
|
|
1321
1480
|
if turn.get("id"):
|
|
1481
|
+
self.rejected_turns.pop(thread, None)
|
|
1322
1482
|
self.turn_ids[thread] = turn["id"]
|
|
1323
1483
|
self.pending_intents[thread]["accepted"] = True
|
|
1324
1484
|
self.persist(
|
|
@@ -1326,18 +1486,35 @@ class Controller:
|
|
|
1326
1486
|
accepted=True,
|
|
1327
1487
|
turn=turn["id"],
|
|
1328
1488
|
state=turn.get("status", "working"),
|
|
1489
|
+
rejected=False,
|
|
1329
1490
|
)
|
|
1330
1491
|
except RemoteError as error:
|
|
1331
|
-
|
|
1332
|
-
|
|
1492
|
+
kind = rejected_failure_kind(error.payload)
|
|
1493
|
+
if kind:
|
|
1494
|
+
if self.pending_intents.get(thread, {}).get("accepted"):
|
|
1495
|
+
return await self.uncertain(thread, generation, baseline)
|
|
1496
|
+
self.remember_rejected(thread, params, baseline)
|
|
1497
|
+
return await self.recover(thread, generation, kind)
|
|
1333
1498
|
raise
|
|
1334
1499
|
except (RouterError, asyncio.TimeoutError):
|
|
1335
1500
|
return await self.uncertain(thread, generation, baseline)
|
|
1336
1501
|
self.report(f"Continued the saved thread with {self.account_name()}.")
|
|
1337
1502
|
|
|
1503
|
+
def remember_rejected(self, thread, params, baseline):
|
|
1504
|
+
# Only explicit pre-acceptance rejections qualify. Keep the original
|
|
1505
|
+
# input in memory; a worker restart requires resubmission, not a guess.
|
|
1506
|
+
self.rejected_turns[thread] = {
|
|
1507
|
+
"params": dict(params),
|
|
1508
|
+
"baseline": baseline or "__unknown_before_submission__",
|
|
1509
|
+
}
|
|
1510
|
+
self.persist(thread, rejected=True)
|
|
1511
|
+
|
|
1338
1512
|
async def restore_recovery(self, thread, persisted):
|
|
1339
1513
|
"""Honor an accepted retry's original deadline after a process restart."""
|
|
1340
1514
|
generation = self.user_generation
|
|
1515
|
+
if persisted.get("rejected"):
|
|
1516
|
+
self.hold_for_review(thread, "rejected_request_requires_resubmission")
|
|
1517
|
+
return
|
|
1341
1518
|
if persisted.get("review_required"):
|
|
1342
1519
|
self.hold_for_review(
|
|
1343
1520
|
thread, persisted.get("review_reason") or "review_required"
|
|
@@ -1703,6 +1880,7 @@ class Controller:
|
|
|
1703
1880
|
self.fallback_tried.pop(message["params"]["threadId"], None)
|
|
1704
1881
|
self.status_changed.set()
|
|
1705
1882
|
if method == "turn/start":
|
|
1883
|
+
self.rejected_turns.pop(message["params"]["threadId"], None)
|
|
1706
1884
|
self.clear_review(message["params"]["threadId"])
|
|
1707
1885
|
self.last_turn[message["params"]["threadId"]] = message["params"]
|
|
1708
1886
|
self.merge_settings(message["params"]["threadId"], message["params"])
|
|
@@ -1768,6 +1946,7 @@ class Controller:
|
|
|
1768
1946
|
accepted=False,
|
|
1769
1947
|
baseline=self.turn_ids.get(thread),
|
|
1770
1948
|
state="submitting",
|
|
1949
|
+
rejected=False,
|
|
1771
1950
|
)
|
|
1772
1951
|
async with guard:
|
|
1773
1952
|
result = await self.backend.call(
|
|
@@ -1834,13 +2013,22 @@ class Controller:
|
|
|
1834
2013
|
await self.emit({"id": request_id, "error": error.payload})
|
|
1835
2014
|
# Ingress overload rejects a turn before a turn/completed event
|
|
1836
2015
|
# exists. Its documented code is explicit, not message matching.
|
|
1837
|
-
|
|
2016
|
+
kind = rejected_failure_kind(error.payload)
|
|
2017
|
+
if method == "turn/start" and kind:
|
|
1838
2018
|
if self.recovery and not self.recovery.done():
|
|
1839
2019
|
self.recovery.cancel()
|
|
1840
2020
|
thread = message["params"]["threadId"]
|
|
1841
|
-
|
|
1842
|
-
|
|
1843
|
-
)
|
|
2021
|
+
intent = self.pending_intents.get(thread, {})
|
|
2022
|
+
baseline = intent.get("baseline", "__unknown_before_submission__")
|
|
2023
|
+
if intent.get("accepted"):
|
|
2024
|
+
self.recovery = asyncio.create_task(
|
|
2025
|
+
self.uncertain(thread, self.user_generation, baseline)
|
|
2026
|
+
)
|
|
2027
|
+
else:
|
|
2028
|
+
self.remember_rejected(thread, message["params"], baseline)
|
|
2029
|
+
self.recovery = asyncio.create_task(
|
|
2030
|
+
self.recover(thread, self.user_generation, kind)
|
|
2031
|
+
)
|
|
1844
2032
|
except (RouterError, asyncio.TimeoutError):
|
|
1845
2033
|
await self.emit(
|
|
1846
2034
|
{
|
package/python/service.py
CHANGED
|
@@ -376,7 +376,12 @@ def restart(state, prefix):
|
|
|
376
376
|
spec = json.loads(path.read_text())
|
|
377
377
|
if not Path(spec["cwd"]).is_dir():
|
|
378
378
|
raise RouterError("The project folder no longer exists")
|
|
379
|
-
spec.update(
|
|
379
|
+
spec.update(
|
|
380
|
+
state="starting",
|
|
381
|
+
background_launch=True,
|
|
382
|
+
foreground_owner=None,
|
|
383
|
+
relay_runtime=str(Path(__file__).parent.resolve()),
|
|
384
|
+
)
|
|
380
385
|
spec.pop("last_error", None)
|
|
381
386
|
atomic_json(path, spec)
|
|
382
387
|
process = spawn(
|
|
@@ -534,7 +539,7 @@ async def _serve(state, ident, binary):
|
|
|
534
539
|
controller.retry_interval = controller.preferences["retry_interval"]
|
|
535
540
|
controller.retry_attempts = controller.preferences["retry_attempts"]
|
|
536
541
|
controller.interval = controller.preferences["unknown_reset_interval"]
|
|
537
|
-
backend.handler = controller.event
|
|
542
|
+
backend.handler = lambda message: controller.event(message, origin=backend)
|
|
538
543
|
heartbeat = asyncio.create_task(controller.maintain())
|
|
539
544
|
connection = root / (ident + ".connection.json")
|
|
540
545
|
try:
|
|
@@ -717,7 +722,7 @@ async def _serve(state, ident, binary):
|
|
|
717
722
|
if not persisted.get("accepted")
|
|
718
723
|
else None,
|
|
719
724
|
)
|
|
720
|
-
if state_name == "completed":
|
|
725
|
+
if state_name == "completed" and not persisted.get("rejected"):
|
|
721
726
|
controller.persist(
|
|
722
727
|
thread_id,
|
|
723
728
|
state="completed",
|
|
@@ -729,6 +734,7 @@ async def _serve(state, ident, binary):
|
|
|
729
734
|
)
|
|
730
735
|
elif (
|
|
731
736
|
persisted.get("review_required")
|
|
737
|
+
or persisted.get("rejected")
|
|
732
738
|
or persisted.get("active_items")
|
|
733
739
|
or not persisted.get("accepted")
|
|
734
740
|
):
|
package/tests/fake_native.py
CHANGED
|
@@ -128,6 +128,12 @@ for raw in sys.stdin:
|
|
|
128
128
|
result = {"account": {"type": "chatgpt", "planType": "plus"}}
|
|
129
129
|
elif method == "account/login/start":
|
|
130
130
|
result = {"type": params["type"]}
|
|
131
|
+
send(
|
|
132
|
+
{
|
|
133
|
+
"method": "account/updated",
|
|
134
|
+
"params": {"authMode": "chatgptAuthTokens", "planType": "plus"},
|
|
135
|
+
}
|
|
136
|
+
)
|
|
131
137
|
elif method == "account/rateLimits/read":
|
|
132
138
|
reset_count = change(lambda d: d.get("reset_credit_count", 1))
|
|
133
139
|
weekly = change(
|
|
@@ -215,6 +221,12 @@ for raw in sys.stdin:
|
|
|
215
221
|
value["itemsView"] = params.get("itemsView", "summary")
|
|
216
222
|
elif method == "thread/loaded/list":
|
|
217
223
|
result = {"data": change(lambda d: list(d["threads"]))}
|
|
224
|
+
elif method == "thread/backgroundTerminals/list":
|
|
225
|
+
result = {
|
|
226
|
+
"data": change(
|
|
227
|
+
lambda d: d.get("backgrounds", {}).get(params["threadId"], [])
|
|
228
|
+
)
|
|
229
|
+
}
|
|
218
230
|
elif method in {"turn/start", "review/start"}:
|
|
219
231
|
thread = params["threadId"]
|
|
220
232
|
turn = str(uuid.uuid4())
|
|
@@ -263,6 +275,28 @@ for raw in sys.stdin:
|
|
|
263
275
|
"params": {"threadId": thread, "item": tool},
|
|
264
276
|
}
|
|
265
277
|
)
|
|
278
|
+
if count <= fault.get("background_activity_turns", 0):
|
|
279
|
+
item_id = "background-" + turn
|
|
280
|
+
change(
|
|
281
|
+
lambda d: (
|
|
282
|
+
d.setdefault("backgrounds", {})
|
|
283
|
+
.setdefault(thread, [])
|
|
284
|
+
.append({"itemId": item_id, "processId": "test-background-job"})
|
|
285
|
+
)
|
|
286
|
+
)
|
|
287
|
+
send(
|
|
288
|
+
{
|
|
289
|
+
"method": "item/started",
|
|
290
|
+
"params": {
|
|
291
|
+
"threadId": thread,
|
|
292
|
+
"item": {
|
|
293
|
+
"id": item_id,
|
|
294
|
+
"type": "commandExecution",
|
|
295
|
+
"status": "inProgress",
|
|
296
|
+
},
|
|
297
|
+
},
|
|
298
|
+
}
|
|
299
|
+
)
|
|
266
300
|
if fault.get("external"):
|
|
267
301
|
send(
|
|
268
302
|
{
|
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"""Quota recovery at native idle boundaries and account refresh races."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
import tempfile
|
|
7
|
+
import unittest
|
|
8
|
+
from unittest.mock import patch
|
|
9
|
+
import sys
|
|
10
|
+
|
|
11
|
+
from pool import failure_kind
|
|
12
|
+
from router import Controller, RemoteError, RouterError, RPC
|
|
13
|
+
from settings import atomic_json
|
|
14
|
+
from test_router import FakeAccount, FakeBackend, FakeClient, completed
|
|
15
|
+
from test_services import fake_executable, wait_for
|
|
16
|
+
from test_pool import Clock
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class BoundaryTests(unittest.IsolatedAsyncioTestCase):
|
|
20
|
+
def controller(
|
|
21
|
+
self,
|
|
22
|
+
*,
|
|
23
|
+
status="systemError",
|
|
24
|
+
turn_status="failed",
|
|
25
|
+
tool_status="completed",
|
|
26
|
+
backgrounds=True,
|
|
27
|
+
):
|
|
28
|
+
c = Controller(
|
|
29
|
+
[FakeAccount("one"), FakeAccount("two")],
|
|
30
|
+
FakeBackend(),
|
|
31
|
+
report=lambda _: None,
|
|
32
|
+
)
|
|
33
|
+
c.client = FakeClient()
|
|
34
|
+
c.last_turn["thread-1"] = {"threadId": "thread-1"}
|
|
35
|
+
c.turn_ids["thread-1"] = "turn-1"
|
|
36
|
+
c.active_items.add("orphaned-lifecycle-id")
|
|
37
|
+
c.active_item_threads["orphaned-lifecycle-id"] = "thread-1"
|
|
38
|
+
original = c.backend.call
|
|
39
|
+
|
|
40
|
+
async def call(method, params=None, **kw):
|
|
41
|
+
if method == "thread/backgroundTerminals/list":
|
|
42
|
+
return {
|
|
43
|
+
"data": [
|
|
44
|
+
{
|
|
45
|
+
"itemId": "orphaned-lifecycle-id",
|
|
46
|
+
"processId": "background-job",
|
|
47
|
+
}
|
|
48
|
+
]
|
|
49
|
+
if backgrounds
|
|
50
|
+
else []
|
|
51
|
+
}
|
|
52
|
+
if method == "thread/turns/list":
|
|
53
|
+
return {
|
|
54
|
+
"data": [
|
|
55
|
+
{
|
|
56
|
+
"id": "turn-1",
|
|
57
|
+
"status": turn_status,
|
|
58
|
+
"itemsView": "full",
|
|
59
|
+
"items": [
|
|
60
|
+
{
|
|
61
|
+
"id": "persisted-action",
|
|
62
|
+
"type": "commandExecution",
|
|
63
|
+
"status": tool_status,
|
|
64
|
+
}
|
|
65
|
+
],
|
|
66
|
+
}
|
|
67
|
+
]
|
|
68
|
+
}
|
|
69
|
+
if method == "thread/read":
|
|
70
|
+
return {
|
|
71
|
+
"thread": {"status": {"type": status}, "canAcceptDirectInput": True}
|
|
72
|
+
}
|
|
73
|
+
return await original(method, params, **kw)
|
|
74
|
+
|
|
75
|
+
c.backend.call = call
|
|
76
|
+
return c
|
|
77
|
+
|
|
78
|
+
async def test_known_background_terminal_unblocks_quota_without_replaying_request(
|
|
79
|
+
self,
|
|
80
|
+
):
|
|
81
|
+
c = self.controller()
|
|
82
|
+
await c.event(completed())
|
|
83
|
+
await asyncio.wait_for(c.recovery, 1)
|
|
84
|
+
self.assertFalse(c.active_items)
|
|
85
|
+
self.assertEqual(c.account.label, "two")
|
|
86
|
+
starts = [p for m, p in c.backend.calls if m == "turn/start"]
|
|
87
|
+
self.assertEqual(len(starts), 1)
|
|
88
|
+
self.assertEqual(starts[0]["threadId"], "thread-1")
|
|
89
|
+
|
|
90
|
+
async def test_active_or_incomplete_native_work_still_blocks(self):
|
|
91
|
+
for options in (
|
|
92
|
+
{"status": "active"},
|
|
93
|
+
{"status": "notLoaded"},
|
|
94
|
+
{"turn_status": "inProgress"},
|
|
95
|
+
{"tool_status": "inProgress", "backgrounds": False},
|
|
96
|
+
{"tool_status": None, "backgrounds": False},
|
|
97
|
+
{"backgrounds": False},
|
|
98
|
+
):
|
|
99
|
+
with self.subTest(options=options):
|
|
100
|
+
c = self.controller(**options)
|
|
101
|
+
await c.reconcile_active_items()
|
|
102
|
+
self.assertTrue(c.active_items)
|
|
103
|
+
|
|
104
|
+
async def test_manual_switch_keeps_known_background_job_without_new_turn(self):
|
|
105
|
+
c = self.controller()
|
|
106
|
+
await c.control("switch", {"account": "two"})
|
|
107
|
+
self.assertEqual(c.account.label, "two")
|
|
108
|
+
self.assertFalse(c.active_items)
|
|
109
|
+
self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
|
|
110
|
+
|
|
111
|
+
async def test_pending_approval_and_different_current_turn_still_block(self):
|
|
112
|
+
for blocker in ("approval", "different-turn"):
|
|
113
|
+
c = self.controller()
|
|
114
|
+
if blocker == "approval":
|
|
115
|
+
c.server_requests.add(99)
|
|
116
|
+
else:
|
|
117
|
+
c.turn_ids["thread-1"] = "newer-turn"
|
|
118
|
+
await c.reconcile_active_items()
|
|
119
|
+
self.assertTrue(c.active_items)
|
|
120
|
+
|
|
121
|
+
async def test_new_user_work_during_status_read_invalidates_boundary(self):
|
|
122
|
+
c = self.controller()
|
|
123
|
+
original = c.backend.call
|
|
124
|
+
|
|
125
|
+
async def call(method, params=None, **kw):
|
|
126
|
+
result = await original(method, params, **kw)
|
|
127
|
+
if method == "thread/read":
|
|
128
|
+
c.user_generation += 1
|
|
129
|
+
return result
|
|
130
|
+
|
|
131
|
+
c.backend.call = call
|
|
132
|
+
await c.reconcile_active_items()
|
|
133
|
+
self.assertTrue(c.active_items)
|
|
134
|
+
|
|
135
|
+
async def test_refresh_callback_during_login_uses_new_account(self):
|
|
136
|
+
c = self.controller()
|
|
137
|
+
original = c.backend.call
|
|
138
|
+
|
|
139
|
+
async def call(method, params=None, **kw):
|
|
140
|
+
if method == "account/login/start":
|
|
141
|
+
await c.event(
|
|
142
|
+
{
|
|
143
|
+
"id": "refresh",
|
|
144
|
+
"method": "account/chatgptAuthTokens/refresh",
|
|
145
|
+
"params": {"previousAccountId": "two"},
|
|
146
|
+
}
|
|
147
|
+
)
|
|
148
|
+
return await original(method, params, **kw)
|
|
149
|
+
|
|
150
|
+
c.backend.call = call
|
|
151
|
+
await asyncio.wait_for(c.login(1), 1)
|
|
152
|
+
self.assertEqual(c.backend.sent[-1]["result"]["chatgptAccountId"], "two")
|
|
153
|
+
self.assertFalse(c.client.messages)
|
|
154
|
+
|
|
155
|
+
async def test_failed_refresh_is_private_and_classified_for_auth_retry(self):
|
|
156
|
+
c = self.controller()
|
|
157
|
+
|
|
158
|
+
async def failed(refresh=False):
|
|
159
|
+
raise RouterError("PRIVATE_EXCEPTION_DETAIL")
|
|
160
|
+
|
|
161
|
+
c.accounts[0].credentials = failed
|
|
162
|
+
await c.event(
|
|
163
|
+
{"id": 3, "method": "account/chatgptAuthTokens/refresh", "params": {}}
|
|
164
|
+
)
|
|
165
|
+
self.assertEqual(c.backend.sent[-1]["error"]["code"], -32000)
|
|
166
|
+
self.assertNotIn("PRIVATE_EXCEPTION_DETAIL", json.dumps(c.backend.sent))
|
|
167
|
+
self.assertFalse(c.client.messages)
|
|
168
|
+
self.assertEqual(
|
|
169
|
+
failure_kind("other", "Auth refresh request failed: code=-32000"), "auth"
|
|
170
|
+
)
|
|
171
|
+
self.assertIsNone(
|
|
172
|
+
failure_kind("badRequest", "Auth refresh request failed: code=-32000")
|
|
173
|
+
)
|
|
174
|
+
self.assertIsNone(
|
|
175
|
+
failure_kind("other", "unrelated request failed: code=-32000")
|
|
176
|
+
)
|
|
177
|
+
|
|
178
|
+
async def test_refresh_from_replaced_backend_cannot_reach_new_connection(self):
|
|
179
|
+
c = self.controller()
|
|
180
|
+
old = c.backend
|
|
181
|
+
|
|
182
|
+
async def rotating(refresh=False):
|
|
183
|
+
c.backend = FakeBackend()
|
|
184
|
+
return {"accessToken": "FAKE", "chatgptAccountId": "one"}
|
|
185
|
+
|
|
186
|
+
c.account.credentials = rotating
|
|
187
|
+
await c.event(
|
|
188
|
+
{"id": 12, "method": "account/chatgptAuthTokens/refresh", "params": {}},
|
|
189
|
+
origin=old,
|
|
190
|
+
)
|
|
191
|
+
self.assertFalse(c.backend.sent)
|
|
192
|
+
self.assertFalse(old.sent)
|
|
193
|
+
await c.event(completed(), origin=old)
|
|
194
|
+
self.assertIsNone(c.recovery)
|
|
195
|
+
|
|
196
|
+
async def test_auth_rejection_before_acceptance_retries_original_input_once(self):
|
|
197
|
+
c = self.controller()
|
|
198
|
+
c.active_items.clear()
|
|
199
|
+
c.active_item_threads.clear()
|
|
200
|
+
clock = Clock()
|
|
201
|
+
c.clock, c.sleep = clock, clock.sleep
|
|
202
|
+
c.retry_interval = 300
|
|
203
|
+
original = c.backend.call
|
|
204
|
+
requests = []
|
|
205
|
+
|
|
206
|
+
async def call(method, params=None, **kw):
|
|
207
|
+
if method == "turn/start":
|
|
208
|
+
requests.append(dict(params))
|
|
209
|
+
if len(requests) == 1:
|
|
210
|
+
raise RemoteError(
|
|
211
|
+
{
|
|
212
|
+
"code": -32603,
|
|
213
|
+
"message": "auth refresh request failed: code=-32000",
|
|
214
|
+
}
|
|
215
|
+
)
|
|
216
|
+
return {"turn": {"id": "accepted-turn", "status": "inProgress"}}
|
|
217
|
+
return await original(method, params, **kw)
|
|
218
|
+
|
|
219
|
+
c.backend.call = call
|
|
220
|
+
await c.forward(
|
|
221
|
+
{
|
|
222
|
+
"id": 1,
|
|
223
|
+
"method": "turn/start",
|
|
224
|
+
"params": {
|
|
225
|
+
"threadId": "thread-1",
|
|
226
|
+
"input": [
|
|
227
|
+
{
|
|
228
|
+
"type": "text",
|
|
229
|
+
"text": "UNIQUE_NEW_REQUEST",
|
|
230
|
+
"text_elements": [],
|
|
231
|
+
}
|
|
232
|
+
],
|
|
233
|
+
},
|
|
234
|
+
},
|
|
235
|
+
{},
|
|
236
|
+
)
|
|
237
|
+
await asyncio.wait_for(c.recovery, 1)
|
|
238
|
+
self.assertEqual(len(requests), 2)
|
|
239
|
+
self.assertEqual(requests[0]["input"], requests[1]["input"])
|
|
240
|
+
self.assertEqual(
|
|
241
|
+
requests[0]["clientUserMessageId"], requests[1]["clientUserMessageId"]
|
|
242
|
+
)
|
|
243
|
+
self.assertEqual(c.account.label, "one")
|
|
244
|
+
self.assertFalse(c.rejected_turns)
|
|
245
|
+
self.assertEqual(c.turn_ids["thread-1"], "accepted-turn")
|
|
246
|
+
|
|
247
|
+
async def test_changed_history_prevents_replay_of_rejected_input(self):
|
|
248
|
+
c = self.controller()
|
|
249
|
+
c.active_items.clear()
|
|
250
|
+
c.remember_rejected(
|
|
251
|
+
"thread-1", {"threadId": "thread-1", "input": []}, "older-turn"
|
|
252
|
+
)
|
|
253
|
+
await c.continue_thread("thread-1", 0)
|
|
254
|
+
self.assertIn("thread-1", c.paused_uncertain)
|
|
255
|
+
self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
|
|
256
|
+
|
|
257
|
+
async def test_worker_restart_does_not_guess_rejected_prompt(self):
|
|
258
|
+
c = self.controller()
|
|
259
|
+
await c.restore_recovery(
|
|
260
|
+
"thread-1",
|
|
261
|
+
{"rejected": True, "accepted": False, "state": "retry_scheduled"},
|
|
262
|
+
)
|
|
263
|
+
self.assertIn("thread-1", c.paused_uncertain)
|
|
264
|
+
self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
|
|
265
|
+
|
|
266
|
+
async def test_watchdog_detects_eof_while_native_process_is_still_alive(self):
|
|
267
|
+
proc = await asyncio.create_subprocess_exec(
|
|
268
|
+
sys.executable,
|
|
269
|
+
"-c",
|
|
270
|
+
"import os,time; os.close(1); time.sleep(30)",
|
|
271
|
+
stdin=asyncio.subprocess.PIPE,
|
|
272
|
+
stdout=asyncio.subprocess.PIPE,
|
|
273
|
+
)
|
|
274
|
+
rpc = RPC(proc)
|
|
275
|
+
c = Controller([FakeAccount("one")], rpc, report=lambda _: None)
|
|
276
|
+
c.backend_factory = lambda: None
|
|
277
|
+
restarted = []
|
|
278
|
+
|
|
279
|
+
async def restart():
|
|
280
|
+
restarted.append(True)
|
|
281
|
+
c.stopping.set()
|
|
282
|
+
|
|
283
|
+
c.restart_backend = restart
|
|
284
|
+
try:
|
|
285
|
+
await asyncio.wait_for(asyncio.shield(rpc.reader), 3)
|
|
286
|
+
self.assertIsNone(proc.returncode)
|
|
287
|
+
await asyncio.wait_for(c.maintain(), 3)
|
|
288
|
+
await c.restart_task
|
|
289
|
+
self.assertTrue(restarted)
|
|
290
|
+
finally:
|
|
291
|
+
await rpc.close()
|
|
292
|
+
|
|
293
|
+
async def test_broker_refresh_reloads_persisted_grant_under_shared_lock(self):
|
|
294
|
+
import base64
|
|
295
|
+
import time
|
|
296
|
+
import broker
|
|
297
|
+
|
|
298
|
+
with tempfile.TemporaryDirectory() as folder:
|
|
299
|
+
state = Path(folder)
|
|
300
|
+
home = state / "accounts" / "one"
|
|
301
|
+
home.mkdir(parents=True)
|
|
302
|
+
(home / "enrolled").write_text("{}")
|
|
303
|
+
events = []
|
|
304
|
+
|
|
305
|
+
class Account:
|
|
306
|
+
def __init__(self, label, home, binary, pool):
|
|
307
|
+
self.home = home
|
|
308
|
+
self.rpc = FakeBackend()
|
|
309
|
+
|
|
310
|
+
async def start(self):
|
|
311
|
+
events.append("start")
|
|
312
|
+
|
|
313
|
+
async def close(self):
|
|
314
|
+
events.append("close")
|
|
315
|
+
|
|
316
|
+
part = (
|
|
317
|
+
base64.urlsafe_b64encode(
|
|
318
|
+
json.dumps({"exp": time.time() + 3600}).encode()
|
|
319
|
+
)
|
|
320
|
+
.decode()
|
|
321
|
+
.rstrip("=")
|
|
322
|
+
)
|
|
323
|
+
credentials = {
|
|
324
|
+
"accessToken": "fake." + part + ".fake",
|
|
325
|
+
"chatgptAccountId": "one",
|
|
326
|
+
}
|
|
327
|
+
with (
|
|
328
|
+
patch("router.Account", Account),
|
|
329
|
+
patch("router.read_keychain", return_value=credentials),
|
|
330
|
+
):
|
|
331
|
+
server = asyncio.create_task(broker.serve(state, "unused"))
|
|
332
|
+
try:
|
|
333
|
+
await wait_for(lambda: broker.paths(state)[1].exists())
|
|
334
|
+
await broker.exchange(state, {"op": "usage", "label": "one"})
|
|
335
|
+
value = await broker.exchange(
|
|
336
|
+
state, {"op": "credentials", "label": "one", "refresh": True}
|
|
337
|
+
)
|
|
338
|
+
self.assertEqual(value, credentials)
|
|
339
|
+
self.assertEqual(events[:3], ["start", "close", "start"])
|
|
340
|
+
finally:
|
|
341
|
+
await broker.exchange(state, {"op": "stop"})
|
|
342
|
+
await asyncio.wait_for(server, 3)
|
|
343
|
+
|
|
344
|
+
async def test_background_activity_over_real_jsonl_switches_without_restart(self):
|
|
345
|
+
class ResetPromptClient(FakeClient):
|
|
346
|
+
def __init__(self):
|
|
347
|
+
super().__init__()
|
|
348
|
+
self.blocked = False
|
|
349
|
+
self.saw_reset_prompt = False
|
|
350
|
+
|
|
351
|
+
async def send(self, raw):
|
|
352
|
+
message = json.loads(raw)
|
|
353
|
+
params = message.get("params", {})
|
|
354
|
+
error = params.get("error") or params.get("turn", {}).get("error") or {}
|
|
355
|
+
if (
|
|
356
|
+
message.get("method") in {"error", "turn/completed"}
|
|
357
|
+
and error.get("codexErrorInfo") == "usageLimitExceeded"
|
|
358
|
+
):
|
|
359
|
+
self.blocked = True
|
|
360
|
+
self.saw_reset_prompt = True
|
|
361
|
+
if message.get("method") == "account/updated":
|
|
362
|
+
self.blocked = False
|
|
363
|
+
await super().send(raw)
|
|
364
|
+
|
|
365
|
+
with tempfile.TemporaryDirectory() as folder:
|
|
366
|
+
root = Path(folder)
|
|
367
|
+
atomic_json(
|
|
368
|
+
root / "fault.json", {"quota_turns": 1, "background_activity_turns": 1}
|
|
369
|
+
)
|
|
370
|
+
rpc = await RPC.start(fake_executable(root), root)
|
|
371
|
+
c = Controller(
|
|
372
|
+
[FakeAccount("one"), FakeAccount("two")], rpc, report=lambda _: None
|
|
373
|
+
)
|
|
374
|
+
c.client = ResetPromptClient()
|
|
375
|
+
rpc.handler = c.event
|
|
376
|
+
try:
|
|
377
|
+
pid = rpc.proc.pid
|
|
378
|
+
await rpc.initialize()
|
|
379
|
+
await c.forward(
|
|
380
|
+
{"id": "thread", "method": "thread/start", "params": {}}, {}
|
|
381
|
+
)
|
|
382
|
+
thread = next(iter(c.last_turn))
|
|
383
|
+
await c.forward(
|
|
384
|
+
{
|
|
385
|
+
"id": "turn",
|
|
386
|
+
"method": "turn/start",
|
|
387
|
+
"params": {
|
|
388
|
+
"threadId": thread,
|
|
389
|
+
"input": [
|
|
390
|
+
{"type": "text", "text": "work", "text_elements": []}
|
|
391
|
+
],
|
|
392
|
+
},
|
|
393
|
+
},
|
|
394
|
+
{},
|
|
395
|
+
)
|
|
396
|
+
|
|
397
|
+
def recovered():
|
|
398
|
+
turns = json.loads((root / "fake-history.json").read_text())[
|
|
399
|
+
"threads"
|
|
400
|
+
][thread]["turns"]
|
|
401
|
+
return len(turns) == 2 and turns[-1]["status"] == "completed"
|
|
402
|
+
|
|
403
|
+
await wait_for(recovered)
|
|
404
|
+
self.assertEqual(c.account.label, "two")
|
|
405
|
+
self.assertFalse(c.active_items)
|
|
406
|
+
self.assertEqual(rpc.proc.pid, pid)
|
|
407
|
+
history = json.loads((root / "fake-history.json").read_text())
|
|
408
|
+
self.assertTrue(history["backgrounds"][thread])
|
|
409
|
+
self.assertTrue(c.client.saw_reset_prompt)
|
|
410
|
+
self.assertFalse(c.client.blocked)
|
|
411
|
+
self.assertFalse(c.server_requests)
|
|
412
|
+
self.assertFalse(
|
|
413
|
+
any(
|
|
414
|
+
c["method"] == "account/rateLimitResetCredit/consume"
|
|
415
|
+
for c in history["calls"]
|
|
416
|
+
)
|
|
417
|
+
)
|
|
418
|
+
finally:
|
|
419
|
+
if c.recovery:
|
|
420
|
+
c.recovery.cancel()
|
|
421
|
+
await asyncio.gather(c.recovery, return_exceptions=True)
|
|
422
|
+
await rpc.close()
|