relay-codex 0.7.3 → 0.7.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -8,8 +8,8 @@ Friends can run `npx relay-codex` or install it with `npm install -g relay-codex
8
8
  The GitHub repository is enough for friends to install or try Relay:
9
9
 
10
10
  ```sh
11
- npm install -g github:blackspider-ops/relay#v0.7.3
12
- npx --yes github:blackspider-ops/relay#v0.7.3 --version
11
+ npm install -g github:blackspider-ops/relay#v0.7.4
12
+ npx --yes github:blackspider-ops/relay#v0.7.4 --version
13
13
  ```
14
14
 
15
15
  `npx` runs packages; it is not a separate publishing service. Publishing
@@ -51,13 +51,22 @@ visible. Use Restart for a stopped worker rather than sending it live controls.
51
51
 
52
52
  **A usage-limit panel remains open and Relay says “waiting for activity.”**
53
53
 
54
- Update Relay if you are using a version older than 0.7.3. A missed tool-completion
55
- notification could leave a finished tool marked as active. Relay now checks full
56
- native history for the matching tool before continuing account recovery. It keeps
54
+ Update Relay if you are using a version older than 0.7.4. Background terminals
55
+ and missed tool notifications could leave a finished turn marked as active.
56
+ Relay checks native history and the current backend's status. Commands explicitly
57
+ listed by Codex as background terminals can keep running while accounts change
58
+ after the model turn ends. The usage panel does not need to be clicked. Relay keeps
57
59
  waiting when work is still running or completion cannot be verified; approvals
58
60
  and paused recovery still need attention. Existing workers retain the code they
59
61
  started with, so an update takes effect after that worker is relaunched.
60
62
 
63
+ **Codex says “auth refresh request failed: code=-32000.”**
64
+
65
+ Relay retries authentication on its normal retry schedule, then tries another
66
+ available account. If Codex reports that a refresh token was revoked, sign in
67
+ again with `relay account reauth LABEL`. Switch any sessions using that account
68
+ first. A revoked login cannot be repaired by retrying the same token.
69
+
61
70
  **Tab completion is missing.**
62
71
 
63
72
  Load `source <(relay completion zsh)` after `compinit`, or use
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "relay-codex",
3
- "version": "0.7.3",
3
+ "version": "0.7.4",
4
4
  "description": "Keep your Codex sessions moving between your own accounts.",
5
5
  "license": "MIT",
6
6
  "author": "blackspider-ops",
package/python/broker.py CHANGED
@@ -225,6 +225,11 @@ async def serve(state, binary):
225
225
  ):
226
226
  value = candidate
227
227
  if value is None:
228
+ if request.get("refresh"):
229
+ # A retained native manager can hold a refresh
230
+ # grant rotated by another process. Reload the
231
+ # Keychain under this account's shared lock.
232
+ await account.close()
228
233
  await account.start()
229
234
  result = await account.rpc.call(
230
235
  "account/read",
@@ -350,7 +350,7 @@ async def _run(args, state, binary):
350
350
  controller.launch_overrides = overrides
351
351
  controller.client = output
352
352
  output.controller = controller
353
- backend.handler = controller.event
353
+ backend.handler = lambda message: controller.event(message, origin=backend)
354
354
  init = await backend.initialize()
355
355
  heartbeat = asyncio.create_task(controller.maintain())
356
356
  model = values.get("--model")
package/python/pool.py CHANGED
@@ -545,6 +545,13 @@ CAPACITY_MESSAGE = "Selected model is at capacity. Please try a different model.
545
545
 
546
546
 
547
547
  def failure_kind(info, message=None):
548
+ if (
549
+ info in (None, "other", "unauthorized")
550
+ and isinstance(message, str)
551
+ and message.strip().removeprefix("■").strip().lower()
552
+ == "auth refresh request failed: code=-32000"
553
+ ):
554
+ return "auth"
548
555
  if info == "policyCapacityUnavailable":
549
556
  return "capacity"
550
557
  # Older/generic error envelopes can omit the enum. Match only the known
package/python/router.py CHANGED
@@ -24,7 +24,7 @@ from pool import Pool, blocked, reset_time, failure_kind, file_lock
24
24
  from launch import translate, RESUME_FIELDS
25
25
  import settings
26
26
 
27
- VERSION = "0.7.3"
27
+ VERSION = "0.7.4"
28
28
  _support = Path.home() / "Library/Application Support"
29
29
  _legacy = _support / "Codex Accounts"
30
30
  _default_state = _legacy if (_legacy / "accounts").exists() else _support / "Relay"
@@ -95,6 +95,17 @@ def clean_env(home: Path) -> dict:
95
95
  return env
96
96
 
97
97
 
98
+ def rejected_failure_kind(payload):
99
+ if payload.get("code") == -32001:
100
+ return "transient"
101
+ if (
102
+ payload.get("code") in {-32000, -32603}
103
+ and failure_kind("other", payload.get("message")) == "auth"
104
+ ):
105
+ return "auth"
106
+ return None
107
+
108
+
98
109
  def codex_path() -> str:
99
110
  path = shutil.which("codex")
100
111
  if not path:
@@ -193,6 +204,9 @@ class RPC:
193
204
  self.reader = asyncio.create_task(self.read_loop())
194
205
  self.startup_guard = None
195
206
 
207
+ def disconnected(self):
208
+ return self.proc.returncode is not None or self.reader.done()
209
+
196
210
  @classmethod
197
211
  async def start(
198
212
  cls, binary: str, home: Path, *, helper=False, handler=None, extra_args=None
@@ -330,7 +344,7 @@ class Account:
330
344
  self.lock = asyncio.Lock()
331
345
 
332
346
  async def start(self):
333
- if self.rpc and self.rpc.proc.returncode is not None:
347
+ if self.rpc and self.rpc.disconnected():
334
348
  await self.rpc.close()
335
349
  self.rpc = None
336
350
  if not self.rpc:
@@ -384,6 +398,7 @@ class Account:
384
398
  async def close(self):
385
399
  if self.rpc:
386
400
  await self.rpc.close()
401
+ self.rpc = None
387
402
 
388
403
 
389
404
  class Controller:
@@ -433,9 +448,11 @@ class Controller:
433
448
  self.turn_ids = {}
434
449
  self.recovery_jobs = {}
435
450
  self.pending_intents = {}
451
+ self.rejected_turns = {}
436
452
  self.active_items = set()
437
453
  self.active_item_threads = {}
438
454
  self.item_revision = 0
455
+ self.auth_account = None
439
456
  self.backend_factory = None
440
457
  self.restart_lock = asyncio.Lock()
441
458
  self.restart_task = None
@@ -675,6 +692,7 @@ class Controller:
675
692
  model=next((self.model_for(t) for t in self.last_turn), None),
676
693
  paused=self.paused,
677
694
  background=self.background,
695
+ activity=self.activity_snapshot(),
678
696
  )
679
697
  last_heartbeat = now
680
698
  new_revision = self.pool.revision()
@@ -743,7 +761,7 @@ class Controller:
743
761
  pass
744
762
  if now - last_watch >= self.preferences["watchdog_interval"]:
745
763
  last_watch = now
746
- if self.backend.proc.returncode is not None and self.backend_factory:
764
+ if self.backend_disconnected() and self.backend_factory:
747
765
  if not self.restart_task or self.restart_task.done():
748
766
  self.restart_task = asyncio.create_task(self.restart_backend())
749
767
  elif self.active_threads:
@@ -805,6 +823,7 @@ class Controller:
805
823
  )
806
824
  if index is None:
807
825
  raise ControlRejected("unknown_account")
826
+ await self.reconcile_active_items()
808
827
  if self.active_threads or self.active_items or self.server_requests:
809
828
  raise ControlRejected("busy")
810
829
  options = self.pool.options(label) if self.pool else {}
@@ -859,20 +878,24 @@ class Controller:
859
878
 
860
879
  async def restart_backend(self):
861
880
  async with self.restart_lock:
862
- if self.backend.proc.returncode is None:
881
+ if not self.backend_disconnected():
863
882
  return
864
883
  self.note("restarting")
865
884
  in_flight = bool(self.active_items or self.server_requests)
866
885
  old = self.backend
886
+ old_auth_account = self.auth_account
867
887
  candidate = None
868
888
  try:
869
889
  candidate = await self.backend_factory()
870
- candidate.handler = self.event
890
+ candidate.handler = lambda message: self.event(
891
+ message, origin=candidate
892
+ )
871
893
  await candidate.initialize()
872
894
  self.backend = candidate
873
895
  await self.login(self.index)
874
896
  except Exception:
875
897
  self.backend = old
898
+ self.auth_account = old_auth_account
876
899
  if candidate:
877
900
  await candidate.close()
878
901
  self.note("backend_restart_failed", reason="startup_or_login_failed")
@@ -966,9 +989,17 @@ class Controller:
966
989
  )
967
990
  ):
968
991
  raise ControlRejected("account_unavailable")
969
- await self.backend.call(
970
- "account/login/start", {"type": "chatgptAuthTokens", **credentials}
971
- )
992
+ # Native refresh callbacks can arrive before the login response.
993
+ # Bind them to the credentials being installed, not the old index.
994
+ previous_binding = self.auth_account
995
+ self.auth_account = self.accounts[index]
996
+ try:
997
+ await self.backend.call(
998
+ "account/login/start", {"type": "chatgptAuthTokens", **credentials}
999
+ )
1000
+ except RemoteError:
1001
+ self.auth_account = previous_binding
1002
+ raise
972
1003
  self.index = index
973
1004
  # Acceptance of token syntax alone isn't a verified, usable login.
974
1005
  result = await self.backend.call("account/read", {"refreshToken": False})
@@ -984,25 +1015,34 @@ class Controller:
984
1015
  except Exception:
985
1016
  pass
986
1017
 
987
- async def event(self, message):
1018
+ async def event(self, message, *, origin=None):
1019
+ if origin is not None and origin is not self.backend:
1020
+ return
988
1021
  method, params = message.get("method"), message.get("params", {})
989
1022
  if method == "account/chatgptAuthTokens/refresh":
1023
+ account = self.auth_account or self.account
1024
+ transport = self.backend
990
1025
  try:
991
1026
  # Never return another identity in response to a refresh request.
992
- credentials = await self.account.credentials(refresh=True)
1027
+ credentials = await account.credentials(refresh=True)
993
1028
  previous = params.get("previousAccountId")
994
1029
  if previous and previous != credentials["chatgptAccountId"]:
995
1030
  raise RouterError(
996
1031
  "Refresh account mismatch; re-enroll the account."
997
1032
  )
998
- await self.backend.send({"id": message["id"], "result": credentials})
1033
+ if transport is not self.backend:
1034
+ return
1035
+ await transport.send({"id": message["id"], "result": credentials})
999
1036
  except (RouterError, asyncio.TimeoutError):
1000
- await self.backend.send(
1037
+ if transport is not self.backend:
1038
+ return
1039
+ self.note("auth_refresh_failed", account=account.label)
1040
+ await transport.send(
1001
1041
  {
1002
1042
  "id": message["id"],
1003
1043
  "error": {
1004
1044
  "code": -32000,
1005
- "message": "Account requires sign-in.",
1045
+ "message": "Account credentials could not be refreshed. Relay will retry after this turn ends.",
1006
1046
  },
1007
1047
  }
1008
1048
  )
@@ -1024,6 +1064,7 @@ class Controller:
1024
1064
  if self.pool and thread.get("id"):
1025
1065
  self.pool.claim(thread["id"])
1026
1066
  if method == "turn/started":
1067
+ self.rejected_turns.pop(params["threadId"], None)
1027
1068
  self.active_threads.add(params["threadId"])
1028
1069
  self.turn_ids[params["threadId"]] = params.get("turn", {}).get("id")
1029
1070
  if params["threadId"] in self.pending_intents:
@@ -1034,6 +1075,7 @@ class Controller:
1034
1075
  accepted=True,
1035
1076
  turn=params.get("turn", {}).get("id"),
1036
1077
  state="working",
1078
+ rejected=False,
1037
1079
  )
1038
1080
  elif method == "item/started" and params.get("item", {}).get("type") in {
1039
1081
  "commandExecution",
@@ -1206,7 +1248,9 @@ class Controller:
1206
1248
  await self.reconcile(thread)
1207
1249
  if time.monotonic() - began > self.preferences["idle_warning_after"]:
1208
1250
  self.note(
1209
- "waiting_for_activity", reason="actions_or_approvals_not_idle"
1251
+ "waiting_for_activity",
1252
+ reason="actions_or_approvals_not_idle",
1253
+ activity=self.activity_snapshot(),
1210
1254
  )
1211
1255
  if self.paused:
1212
1256
  self.wake.clear()
@@ -1217,7 +1261,7 @@ class Controller:
1217
1261
  return generation == self.user_generation and self.attached()
1218
1262
 
1219
1263
  async def reconcile_active_items(self):
1220
- """Retire missed completion events only from matching, full native history."""
1264
+ """Reconcile finished actions and native background jobs at idle boundaries."""
1221
1265
  owners = dict(self.active_item_threads)
1222
1266
  revision = self.item_revision
1223
1267
  for thread in set(owners.values()):
@@ -1226,6 +1270,10 @@ class Controller:
1226
1270
  for item, owner in owners.items()
1227
1271
  if owner == thread and item in self.active_items
1228
1272
  }
1273
+ if not pending:
1274
+ continue
1275
+ await self.reconcile_background_activity(thread, pending, revision)
1276
+ pending &= self.active_items
1229
1277
  if not pending:
1230
1278
  continue
1231
1279
  cursor = None
@@ -1279,6 +1327,103 @@ class Controller:
1279
1327
  break
1280
1328
  visited.add(cursor)
1281
1329
 
1330
+ def backend_disconnected(self):
1331
+ check = getattr(self.backend, "disconnected", None)
1332
+ return (
1333
+ check()
1334
+ if check
1335
+ else getattr(getattr(self.backend, "proc", None), "returncode", None)
1336
+ is not None
1337
+ )
1338
+
1339
+ def activity_snapshot(self):
1340
+ """Private diagnostic metadata; never tool arguments or credentials."""
1341
+ return {
1342
+ "threads": sorted(self.active_threads),
1343
+ "items": sorted(self.active_items),
1344
+ "owners": dict(self.active_item_threads),
1345
+ "requests": sorted(
1346
+ {
1347
+ m.get("method", "unknown")
1348
+ for m in self.pending_server_requests.values()
1349
+ }
1350
+ ),
1351
+ "request_count": len(self.server_requests),
1352
+ "paused": self.paused,
1353
+ "disconnected": self.backend_disconnected(),
1354
+ }
1355
+
1356
+ async def reconcile_background_activity(self, thread, pending, revision):
1357
+ """After a turn ends, known background terminals need not block login.
1358
+
1359
+ These jobs remain alive in the same backend. Unknown tools and pending
1360
+ approvals still block recovery; a terminal turn alone cannot clear them.
1361
+ """
1362
+ turn_id = self.turn_ids.get(thread)
1363
+ generation = self.user_generation
1364
+ if not turn_id or self.server_requests:
1365
+ return
1366
+ try:
1367
+ turn = await self.latest(thread) or {}
1368
+ if turn.get("id") != turn_id or turn.get("status") not in {
1369
+ "completed",
1370
+ "failed",
1371
+ "interrupted",
1372
+ }:
1373
+ return
1374
+ background = set()
1375
+ cursor = None
1376
+ visited = set()
1377
+ for _ in range(8):
1378
+ params = {"threadId": thread, "limit": 100}
1379
+ if cursor:
1380
+ params["cursor"] = cursor
1381
+ result = await self.backend.call(
1382
+ "thread/backgroundTerminals/list", params, timeout=15
1383
+ )
1384
+ background.update(item.get("itemId") for item in result.get("data", []))
1385
+ cursor = result.get("nextCursor")
1386
+ if not cursor or cursor in visited or pending <= background:
1387
+ break
1388
+ visited.add(cursor)
1389
+ if not pending & background:
1390
+ return
1391
+ info = await self.backend.call(
1392
+ "thread/read", {"threadId": thread}, timeout=15
1393
+ )
1394
+ except (RouterError, asyncio.TimeoutError):
1395
+ return
1396
+ if (
1397
+ revision != self.item_revision
1398
+ or generation != self.user_generation
1399
+ or self.turn_ids.get(thread) != turn_id
1400
+ or self.server_requests
1401
+ or info.get("thread", {}).get("status", {}).get("type")
1402
+ not in {"idle", "systemError"}
1403
+ or info.get("thread", {}).get("canAcceptDirectInput") is False
1404
+ ):
1405
+ return
1406
+ cleared = {
1407
+ item
1408
+ for item in pending & background
1409
+ if self.active_item_threads.get(item) == thread
1410
+ and item in self.active_items
1411
+ }
1412
+ for item in cleared:
1413
+ self.active_items.discard(item)
1414
+ self.active_item_threads.pop(item, None)
1415
+ if cleared:
1416
+ self.active_threads.discard(thread)
1417
+ self.persist(thread, active_items=bool(self.active_items))
1418
+ if self.pool:
1419
+ self.pool.record(
1420
+ "background_activity_reconciled",
1421
+ count=len(cleared),
1422
+ thread=thread,
1423
+ turn=turn_id,
1424
+ )
1425
+ self.status_changed.set()
1426
+
1282
1427
  async def continue_thread(self, thread, generation):
1283
1428
  if not self.can_continue(thread, generation):
1284
1429
  return
@@ -1295,6 +1440,19 @@ class Controller:
1295
1440
  params[key] = self.thread_settings[thread][key]
1296
1441
  params["clientUserMessageId"] = str(uuid.uuid4())
1297
1442
  baseline = self.turn_ids.get(thread) or "__unknown_before_submission__"
1443
+ rejected = self.rejected_turns.get(thread)
1444
+ if rejected:
1445
+ baseline = rejected["baseline"]
1446
+ try:
1447
+ latest = await self.latest(thread)
1448
+ except (RouterError, asyncio.TimeoutError):
1449
+ self.hold_for_review(thread, "rejected_request_history_unavailable")
1450
+ return
1451
+ observed = (latest or {}).get("id") or "__unknown_before_submission__"
1452
+ if observed != baseline:
1453
+ self.hold_for_review(thread, "rejected_request_history_changed")
1454
+ return
1455
+ params = dict(rejected["params"])
1298
1456
  try:
1299
1457
  while True:
1300
1458
  async with self.login_lock:
@@ -1312,6 +1470,7 @@ class Controller:
1312
1470
  accepted=False,
1313
1471
  state="submitting",
1314
1472
  baseline=baseline,
1473
+ rejected=bool(rejected),
1315
1474
  )
1316
1475
  result = await self.backend.call("turn/start", params)
1317
1476
  break
@@ -1319,6 +1478,7 @@ class Controller:
1319
1478
  return
1320
1479
  turn = result.get("turn", {})
1321
1480
  if turn.get("id"):
1481
+ self.rejected_turns.pop(thread, None)
1322
1482
  self.turn_ids[thread] = turn["id"]
1323
1483
  self.pending_intents[thread]["accepted"] = True
1324
1484
  self.persist(
@@ -1326,18 +1486,35 @@ class Controller:
1326
1486
  accepted=True,
1327
1487
  turn=turn["id"],
1328
1488
  state=turn.get("status", "working"),
1489
+ rejected=False,
1329
1490
  )
1330
1491
  except RemoteError as error:
1331
- if error.payload.get("code") == -32001:
1332
- return await self.recover(thread, generation, "transient")
1492
+ kind = rejected_failure_kind(error.payload)
1493
+ if kind:
1494
+ if self.pending_intents.get(thread, {}).get("accepted"):
1495
+ return await self.uncertain(thread, generation, baseline)
1496
+ self.remember_rejected(thread, params, baseline)
1497
+ return await self.recover(thread, generation, kind)
1333
1498
  raise
1334
1499
  except (RouterError, asyncio.TimeoutError):
1335
1500
  return await self.uncertain(thread, generation, baseline)
1336
1501
  self.report(f"Continued the saved thread with {self.account_name()}.")
1337
1502
 
1503
+ def remember_rejected(self, thread, params, baseline):
1504
+ # Only explicit pre-acceptance rejections qualify. Keep the original
1505
+ # input in memory; a worker restart requires resubmission, not a guess.
1506
+ self.rejected_turns[thread] = {
1507
+ "params": dict(params),
1508
+ "baseline": baseline or "__unknown_before_submission__",
1509
+ }
1510
+ self.persist(thread, rejected=True)
1511
+
1338
1512
  async def restore_recovery(self, thread, persisted):
1339
1513
  """Honor an accepted retry's original deadline after a process restart."""
1340
1514
  generation = self.user_generation
1515
+ if persisted.get("rejected"):
1516
+ self.hold_for_review(thread, "rejected_request_requires_resubmission")
1517
+ return
1341
1518
  if persisted.get("review_required"):
1342
1519
  self.hold_for_review(
1343
1520
  thread, persisted.get("review_reason") or "review_required"
@@ -1703,6 +1880,7 @@ class Controller:
1703
1880
  self.fallback_tried.pop(message["params"]["threadId"], None)
1704
1881
  self.status_changed.set()
1705
1882
  if method == "turn/start":
1883
+ self.rejected_turns.pop(message["params"]["threadId"], None)
1706
1884
  self.clear_review(message["params"]["threadId"])
1707
1885
  self.last_turn[message["params"]["threadId"]] = message["params"]
1708
1886
  self.merge_settings(message["params"]["threadId"], message["params"])
@@ -1768,6 +1946,7 @@ class Controller:
1768
1946
  accepted=False,
1769
1947
  baseline=self.turn_ids.get(thread),
1770
1948
  state="submitting",
1949
+ rejected=False,
1771
1950
  )
1772
1951
  async with guard:
1773
1952
  result = await self.backend.call(
@@ -1834,13 +2013,22 @@ class Controller:
1834
2013
  await self.emit({"id": request_id, "error": error.payload})
1835
2014
  # Ingress overload rejects a turn before a turn/completed event
1836
2015
  # exists. Its documented code is explicit, not message matching.
1837
- if method == "turn/start" and error.payload.get("code") == -32001:
2016
+ kind = rejected_failure_kind(error.payload)
2017
+ if method == "turn/start" and kind:
1838
2018
  if self.recovery and not self.recovery.done():
1839
2019
  self.recovery.cancel()
1840
2020
  thread = message["params"]["threadId"]
1841
- self.recovery = asyncio.create_task(
1842
- self.recover(thread, self.user_generation, "transient")
1843
- )
2021
+ intent = self.pending_intents.get(thread, {})
2022
+ baseline = intent.get("baseline", "__unknown_before_submission__")
2023
+ if intent.get("accepted"):
2024
+ self.recovery = asyncio.create_task(
2025
+ self.uncertain(thread, self.user_generation, baseline)
2026
+ )
2027
+ else:
2028
+ self.remember_rejected(thread, message["params"], baseline)
2029
+ self.recovery = asyncio.create_task(
2030
+ self.recover(thread, self.user_generation, kind)
2031
+ )
1844
2032
  except (RouterError, asyncio.TimeoutError):
1845
2033
  await self.emit(
1846
2034
  {
package/python/service.py CHANGED
@@ -376,7 +376,12 @@ def restart(state, prefix):
376
376
  spec = json.loads(path.read_text())
377
377
  if not Path(spec["cwd"]).is_dir():
378
378
  raise RouterError("The project folder no longer exists")
379
- spec.update(state="starting", background_launch=True, foreground_owner=None)
379
+ spec.update(
380
+ state="starting",
381
+ background_launch=True,
382
+ foreground_owner=None,
383
+ relay_runtime=str(Path(__file__).parent.resolve()),
384
+ )
380
385
  spec.pop("last_error", None)
381
386
  atomic_json(path, spec)
382
387
  process = spawn(
@@ -534,7 +539,7 @@ async def _serve(state, ident, binary):
534
539
  controller.retry_interval = controller.preferences["retry_interval"]
535
540
  controller.retry_attempts = controller.preferences["retry_attempts"]
536
541
  controller.interval = controller.preferences["unknown_reset_interval"]
537
- backend.handler = controller.event
542
+ backend.handler = lambda message: controller.event(message, origin=backend)
538
543
  heartbeat = asyncio.create_task(controller.maintain())
539
544
  connection = root / (ident + ".connection.json")
540
545
  try:
@@ -717,7 +722,7 @@ async def _serve(state, ident, binary):
717
722
  if not persisted.get("accepted")
718
723
  else None,
719
724
  )
720
- if state_name == "completed":
725
+ if state_name == "completed" and not persisted.get("rejected"):
721
726
  controller.persist(
722
727
  thread_id,
723
728
  state="completed",
@@ -729,6 +734,7 @@ async def _serve(state, ident, binary):
729
734
  )
730
735
  elif (
731
736
  persisted.get("review_required")
737
+ or persisted.get("rejected")
732
738
  or persisted.get("active_items")
733
739
  or not persisted.get("accepted")
734
740
  ):
@@ -128,6 +128,12 @@ for raw in sys.stdin:
128
128
  result = {"account": {"type": "chatgpt", "planType": "plus"}}
129
129
  elif method == "account/login/start":
130
130
  result = {"type": params["type"]}
131
+ send(
132
+ {
133
+ "method": "account/updated",
134
+ "params": {"authMode": "chatgptAuthTokens", "planType": "plus"},
135
+ }
136
+ )
131
137
  elif method == "account/rateLimits/read":
132
138
  reset_count = change(lambda d: d.get("reset_credit_count", 1))
133
139
  weekly = change(
@@ -215,6 +221,12 @@ for raw in sys.stdin:
215
221
  value["itemsView"] = params.get("itemsView", "summary")
216
222
  elif method == "thread/loaded/list":
217
223
  result = {"data": change(lambda d: list(d["threads"]))}
224
+ elif method == "thread/backgroundTerminals/list":
225
+ result = {
226
+ "data": change(
227
+ lambda d: d.get("backgrounds", {}).get(params["threadId"], [])
228
+ )
229
+ }
218
230
  elif method in {"turn/start", "review/start"}:
219
231
  thread = params["threadId"]
220
232
  turn = str(uuid.uuid4())
@@ -263,6 +275,28 @@ for raw in sys.stdin:
263
275
  "params": {"threadId": thread, "item": tool},
264
276
  }
265
277
  )
278
+ if count <= fault.get("background_activity_turns", 0):
279
+ item_id = "background-" + turn
280
+ change(
281
+ lambda d: (
282
+ d.setdefault("backgrounds", {})
283
+ .setdefault(thread, [])
284
+ .append({"itemId": item_id, "processId": "test-background-job"})
285
+ )
286
+ )
287
+ send(
288
+ {
289
+ "method": "item/started",
290
+ "params": {
291
+ "threadId": thread,
292
+ "item": {
293
+ "id": item_id,
294
+ "type": "commandExecution",
295
+ "status": "inProgress",
296
+ },
297
+ },
298
+ }
299
+ )
266
300
  if fault.get("external"):
267
301
  send(
268
302
  {
@@ -0,0 +1,422 @@
1
+ """Quota recovery at native idle boundaries and account refresh races."""
2
+
3
+ import asyncio
4
+ import json
5
+ from pathlib import Path
6
+ import tempfile
7
+ import unittest
8
+ from unittest.mock import patch
9
+ import sys
10
+
11
+ from pool import failure_kind
12
+ from router import Controller, RemoteError, RouterError, RPC
13
+ from settings import atomic_json
14
+ from test_router import FakeAccount, FakeBackend, FakeClient, completed
15
+ from test_services import fake_executable, wait_for
16
+ from test_pool import Clock
17
+
18
+
19
+ class BoundaryTests(unittest.IsolatedAsyncioTestCase):
20
+ def controller(
21
+ self,
22
+ *,
23
+ status="systemError",
24
+ turn_status="failed",
25
+ tool_status="completed",
26
+ backgrounds=True,
27
+ ):
28
+ c = Controller(
29
+ [FakeAccount("one"), FakeAccount("two")],
30
+ FakeBackend(),
31
+ report=lambda _: None,
32
+ )
33
+ c.client = FakeClient()
34
+ c.last_turn["thread-1"] = {"threadId": "thread-1"}
35
+ c.turn_ids["thread-1"] = "turn-1"
36
+ c.active_items.add("orphaned-lifecycle-id")
37
+ c.active_item_threads["orphaned-lifecycle-id"] = "thread-1"
38
+ original = c.backend.call
39
+
40
+ async def call(method, params=None, **kw):
41
+ if method == "thread/backgroundTerminals/list":
42
+ return {
43
+ "data": [
44
+ {
45
+ "itemId": "orphaned-lifecycle-id",
46
+ "processId": "background-job",
47
+ }
48
+ ]
49
+ if backgrounds
50
+ else []
51
+ }
52
+ if method == "thread/turns/list":
53
+ return {
54
+ "data": [
55
+ {
56
+ "id": "turn-1",
57
+ "status": turn_status,
58
+ "itemsView": "full",
59
+ "items": [
60
+ {
61
+ "id": "persisted-action",
62
+ "type": "commandExecution",
63
+ "status": tool_status,
64
+ }
65
+ ],
66
+ }
67
+ ]
68
+ }
69
+ if method == "thread/read":
70
+ return {
71
+ "thread": {"status": {"type": status}, "canAcceptDirectInput": True}
72
+ }
73
+ return await original(method, params, **kw)
74
+
75
+ c.backend.call = call
76
+ return c
77
+
78
+ async def test_known_background_terminal_unblocks_quota_without_replaying_request(
79
+ self,
80
+ ):
81
+ c = self.controller()
82
+ await c.event(completed())
83
+ await asyncio.wait_for(c.recovery, 1)
84
+ self.assertFalse(c.active_items)
85
+ self.assertEqual(c.account.label, "two")
86
+ starts = [p for m, p in c.backend.calls if m == "turn/start"]
87
+ self.assertEqual(len(starts), 1)
88
+ self.assertEqual(starts[0]["threadId"], "thread-1")
89
+
90
+ async def test_active_or_incomplete_native_work_still_blocks(self):
91
+ for options in (
92
+ {"status": "active"},
93
+ {"status": "notLoaded"},
94
+ {"turn_status": "inProgress"},
95
+ {"tool_status": "inProgress", "backgrounds": False},
96
+ {"tool_status": None, "backgrounds": False},
97
+ {"backgrounds": False},
98
+ ):
99
+ with self.subTest(options=options):
100
+ c = self.controller(**options)
101
+ await c.reconcile_active_items()
102
+ self.assertTrue(c.active_items)
103
+
104
+ async def test_manual_switch_keeps_known_background_job_without_new_turn(self):
105
+ c = self.controller()
106
+ await c.control("switch", {"account": "two"})
107
+ self.assertEqual(c.account.label, "two")
108
+ self.assertFalse(c.active_items)
109
+ self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
110
+
111
+ async def test_pending_approval_and_different_current_turn_still_block(self):
112
+ for blocker in ("approval", "different-turn"):
113
+ c = self.controller()
114
+ if blocker == "approval":
115
+ c.server_requests.add(99)
116
+ else:
117
+ c.turn_ids["thread-1"] = "newer-turn"
118
+ await c.reconcile_active_items()
119
+ self.assertTrue(c.active_items)
120
+
121
+ async def test_new_user_work_during_status_read_invalidates_boundary(self):
122
+ c = self.controller()
123
+ original = c.backend.call
124
+
125
+ async def call(method, params=None, **kw):
126
+ result = await original(method, params, **kw)
127
+ if method == "thread/read":
128
+ c.user_generation += 1
129
+ return result
130
+
131
+ c.backend.call = call
132
+ await c.reconcile_active_items()
133
+ self.assertTrue(c.active_items)
134
+
135
+ async def test_refresh_callback_during_login_uses_new_account(self):
136
+ c = self.controller()
137
+ original = c.backend.call
138
+
139
+ async def call(method, params=None, **kw):
140
+ if method == "account/login/start":
141
+ await c.event(
142
+ {
143
+ "id": "refresh",
144
+ "method": "account/chatgptAuthTokens/refresh",
145
+ "params": {"previousAccountId": "two"},
146
+ }
147
+ )
148
+ return await original(method, params, **kw)
149
+
150
+ c.backend.call = call
151
+ await asyncio.wait_for(c.login(1), 1)
152
+ self.assertEqual(c.backend.sent[-1]["result"]["chatgptAccountId"], "two")
153
+ self.assertFalse(c.client.messages)
154
+
155
+ async def test_failed_refresh_is_private_and_classified_for_auth_retry(self):
156
+ c = self.controller()
157
+
158
+ async def failed(refresh=False):
159
+ raise RouterError("PRIVATE_EXCEPTION_DETAIL")
160
+
161
+ c.accounts[0].credentials = failed
162
+ await c.event(
163
+ {"id": 3, "method": "account/chatgptAuthTokens/refresh", "params": {}}
164
+ )
165
+ self.assertEqual(c.backend.sent[-1]["error"]["code"], -32000)
166
+ self.assertNotIn("PRIVATE_EXCEPTION_DETAIL", json.dumps(c.backend.sent))
167
+ self.assertFalse(c.client.messages)
168
+ self.assertEqual(
169
+ failure_kind("other", "Auth refresh request failed: code=-32000"), "auth"
170
+ )
171
+ self.assertIsNone(
172
+ failure_kind("badRequest", "Auth refresh request failed: code=-32000")
173
+ )
174
+ self.assertIsNone(
175
+ failure_kind("other", "unrelated request failed: code=-32000")
176
+ )
177
+
178
+ async def test_refresh_from_replaced_backend_cannot_reach_new_connection(self):
179
+ c = self.controller()
180
+ old = c.backend
181
+
182
+ async def rotating(refresh=False):
183
+ c.backend = FakeBackend()
184
+ return {"accessToken": "FAKE", "chatgptAccountId": "one"}
185
+
186
+ c.account.credentials = rotating
187
+ await c.event(
188
+ {"id": 12, "method": "account/chatgptAuthTokens/refresh", "params": {}},
189
+ origin=old,
190
+ )
191
+ self.assertFalse(c.backend.sent)
192
+ self.assertFalse(old.sent)
193
+ await c.event(completed(), origin=old)
194
+ self.assertIsNone(c.recovery)
195
+
196
+ async def test_auth_rejection_before_acceptance_retries_original_input_once(self):
197
+ c = self.controller()
198
+ c.active_items.clear()
199
+ c.active_item_threads.clear()
200
+ clock = Clock()
201
+ c.clock, c.sleep = clock, clock.sleep
202
+ c.retry_interval = 300
203
+ original = c.backend.call
204
+ requests = []
205
+
206
+ async def call(method, params=None, **kw):
207
+ if method == "turn/start":
208
+ requests.append(dict(params))
209
+ if len(requests) == 1:
210
+ raise RemoteError(
211
+ {
212
+ "code": -32603,
213
+ "message": "auth refresh request failed: code=-32000",
214
+ }
215
+ )
216
+ return {"turn": {"id": "accepted-turn", "status": "inProgress"}}
217
+ return await original(method, params, **kw)
218
+
219
+ c.backend.call = call
220
+ await c.forward(
221
+ {
222
+ "id": 1,
223
+ "method": "turn/start",
224
+ "params": {
225
+ "threadId": "thread-1",
226
+ "input": [
227
+ {
228
+ "type": "text",
229
+ "text": "UNIQUE_NEW_REQUEST",
230
+ "text_elements": [],
231
+ }
232
+ ],
233
+ },
234
+ },
235
+ {},
236
+ )
237
+ await asyncio.wait_for(c.recovery, 1)
238
+ self.assertEqual(len(requests), 2)
239
+ self.assertEqual(requests[0]["input"], requests[1]["input"])
240
+ self.assertEqual(
241
+ requests[0]["clientUserMessageId"], requests[1]["clientUserMessageId"]
242
+ )
243
+ self.assertEqual(c.account.label, "one")
244
+ self.assertFalse(c.rejected_turns)
245
+ self.assertEqual(c.turn_ids["thread-1"], "accepted-turn")
246
+
247
+ async def test_changed_history_prevents_replay_of_rejected_input(self):
248
+ c = self.controller()
249
+ c.active_items.clear()
250
+ c.remember_rejected(
251
+ "thread-1", {"threadId": "thread-1", "input": []}, "older-turn"
252
+ )
253
+ await c.continue_thread("thread-1", 0)
254
+ self.assertIn("thread-1", c.paused_uncertain)
255
+ self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
256
+
257
+ async def test_worker_restart_does_not_guess_rejected_prompt(self):
258
+ c = self.controller()
259
+ await c.restore_recovery(
260
+ "thread-1",
261
+ {"rejected": True, "accepted": False, "state": "retry_scheduled"},
262
+ )
263
+ self.assertIn("thread-1", c.paused_uncertain)
264
+ self.assertFalse(any(m == "turn/start" for m, _ in c.backend.calls))
265
+
266
+ async def test_watchdog_detects_eof_while_native_process_is_still_alive(self):
267
+ proc = await asyncio.create_subprocess_exec(
268
+ sys.executable,
269
+ "-c",
270
+ "import os,time; os.close(1); time.sleep(30)",
271
+ stdin=asyncio.subprocess.PIPE,
272
+ stdout=asyncio.subprocess.PIPE,
273
+ )
274
+ rpc = RPC(proc)
275
+ c = Controller([FakeAccount("one")], rpc, report=lambda _: None)
276
+ c.backend_factory = lambda: None
277
+ restarted = []
278
+
279
+ async def restart():
280
+ restarted.append(True)
281
+ c.stopping.set()
282
+
283
+ c.restart_backend = restart
284
+ try:
285
+ await asyncio.wait_for(asyncio.shield(rpc.reader), 3)
286
+ self.assertIsNone(proc.returncode)
287
+ await asyncio.wait_for(c.maintain(), 3)
288
+ await c.restart_task
289
+ self.assertTrue(restarted)
290
+ finally:
291
+ await rpc.close()
292
+
293
+ async def test_broker_refresh_reloads_persisted_grant_under_shared_lock(self):
294
+ import base64
295
+ import time
296
+ import broker
297
+
298
+ with tempfile.TemporaryDirectory() as folder:
299
+ state = Path(folder)
300
+ home = state / "accounts" / "one"
301
+ home.mkdir(parents=True)
302
+ (home / "enrolled").write_text("{}")
303
+ events = []
304
+
305
+ class Account:
306
+ def __init__(self, label, home, binary, pool):
307
+ self.home = home
308
+ self.rpc = FakeBackend()
309
+
310
+ async def start(self):
311
+ events.append("start")
312
+
313
+ async def close(self):
314
+ events.append("close")
315
+
316
+ part = (
317
+ base64.urlsafe_b64encode(
318
+ json.dumps({"exp": time.time() + 3600}).encode()
319
+ )
320
+ .decode()
321
+ .rstrip("=")
322
+ )
323
+ credentials = {
324
+ "accessToken": "fake." + part + ".fake",
325
+ "chatgptAccountId": "one",
326
+ }
327
+ with (
328
+ patch("router.Account", Account),
329
+ patch("router.read_keychain", return_value=credentials),
330
+ ):
331
+ server = asyncio.create_task(broker.serve(state, "unused"))
332
+ try:
333
+ await wait_for(lambda: broker.paths(state)[1].exists())
334
+ await broker.exchange(state, {"op": "usage", "label": "one"})
335
+ value = await broker.exchange(
336
+ state, {"op": "credentials", "label": "one", "refresh": True}
337
+ )
338
+ self.assertEqual(value, credentials)
339
+ self.assertEqual(events[:3], ["start", "close", "start"])
340
+ finally:
341
+ await broker.exchange(state, {"op": "stop"})
342
+ await asyncio.wait_for(server, 3)
343
+
344
+ async def test_background_activity_over_real_jsonl_switches_without_restart(self):
345
+ class ResetPromptClient(FakeClient):
346
+ def __init__(self):
347
+ super().__init__()
348
+ self.blocked = False
349
+ self.saw_reset_prompt = False
350
+
351
+ async def send(self, raw):
352
+ message = json.loads(raw)
353
+ params = message.get("params", {})
354
+ error = params.get("error") or params.get("turn", {}).get("error") or {}
355
+ if (
356
+ message.get("method") in {"error", "turn/completed"}
357
+ and error.get("codexErrorInfo") == "usageLimitExceeded"
358
+ ):
359
+ self.blocked = True
360
+ self.saw_reset_prompt = True
361
+ if message.get("method") == "account/updated":
362
+ self.blocked = False
363
+ await super().send(raw)
364
+
365
+ with tempfile.TemporaryDirectory() as folder:
366
+ root = Path(folder)
367
+ atomic_json(
368
+ root / "fault.json", {"quota_turns": 1, "background_activity_turns": 1}
369
+ )
370
+ rpc = await RPC.start(fake_executable(root), root)
371
+ c = Controller(
372
+ [FakeAccount("one"), FakeAccount("two")], rpc, report=lambda _: None
373
+ )
374
+ c.client = ResetPromptClient()
375
+ rpc.handler = c.event
376
+ try:
377
+ pid = rpc.proc.pid
378
+ await rpc.initialize()
379
+ await c.forward(
380
+ {"id": "thread", "method": "thread/start", "params": {}}, {}
381
+ )
382
+ thread = next(iter(c.last_turn))
383
+ await c.forward(
384
+ {
385
+ "id": "turn",
386
+ "method": "turn/start",
387
+ "params": {
388
+ "threadId": thread,
389
+ "input": [
390
+ {"type": "text", "text": "work", "text_elements": []}
391
+ ],
392
+ },
393
+ },
394
+ {},
395
+ )
396
+
397
+ def recovered():
398
+ turns = json.loads((root / "fake-history.json").read_text())[
399
+ "threads"
400
+ ][thread]["turns"]
401
+ return len(turns) == 2 and turns[-1]["status"] == "completed"
402
+
403
+ await wait_for(recovered)
404
+ self.assertEqual(c.account.label, "two")
405
+ self.assertFalse(c.active_items)
406
+ self.assertEqual(rpc.proc.pid, pid)
407
+ history = json.loads((root / "fake-history.json").read_text())
408
+ self.assertTrue(history["backgrounds"][thread])
409
+ self.assertTrue(c.client.saw_reset_prompt)
410
+ self.assertFalse(c.client.blocked)
411
+ self.assertFalse(c.server_requests)
412
+ self.assertFalse(
413
+ any(
414
+ c["method"] == "account/rateLimitResetCredit/consume"
415
+ for c in history["calls"]
416
+ )
417
+ )
418
+ finally:
419
+ if c.recovery:
420
+ c.recovery.cancel()
421
+ await asyncio.gather(c.recovery, return_exceptions=True)
422
+ await rpc.close()