codex-autorunner 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (227) hide show
  1. codex_autorunner/__init__.py +12 -1
  2. codex_autorunner/agents/codex/harness.py +1 -1
  3. codex_autorunner/agents/opencode/client.py +113 -4
  4. codex_autorunner/agents/opencode/constants.py +3 -0
  5. codex_autorunner/agents/opencode/harness.py +6 -1
  6. codex_autorunner/agents/opencode/runtime.py +59 -18
  7. codex_autorunner/agents/opencode/supervisor.py +4 -0
  8. codex_autorunner/agents/registry.py +36 -7
  9. codex_autorunner/bootstrap.py +226 -4
  10. codex_autorunner/cli.py +5 -1174
  11. codex_autorunner/codex_cli.py +20 -84
  12. codex_autorunner/core/__init__.py +20 -0
  13. codex_autorunner/core/about_car.py +119 -1
  14. codex_autorunner/core/app_server_ids.py +59 -0
  15. codex_autorunner/core/app_server_threads.py +17 -2
  16. codex_autorunner/core/app_server_utils.py +165 -0
  17. codex_autorunner/core/archive.py +349 -0
  18. codex_autorunner/core/codex_runner.py +6 -2
  19. codex_autorunner/core/config.py +433 -4
  20. codex_autorunner/core/context_awareness.py +38 -0
  21. codex_autorunner/core/docs.py +0 -122
  22. codex_autorunner/core/drafts.py +58 -4
  23. codex_autorunner/core/exceptions.py +4 -0
  24. codex_autorunner/core/filebox.py +265 -0
  25. codex_autorunner/core/flows/controller.py +96 -2
  26. codex_autorunner/core/flows/models.py +13 -0
  27. codex_autorunner/core/flows/reasons.py +52 -0
  28. codex_autorunner/core/flows/reconciler.py +134 -0
  29. codex_autorunner/core/flows/runtime.py +57 -4
  30. codex_autorunner/core/flows/store.py +142 -7
  31. codex_autorunner/core/flows/transition.py +27 -15
  32. codex_autorunner/core/flows/ux_helpers.py +272 -0
  33. codex_autorunner/core/flows/worker_process.py +32 -6
  34. codex_autorunner/core/git_utils.py +62 -0
  35. codex_autorunner/core/hub.py +291 -20
  36. codex_autorunner/core/lifecycle_events.py +253 -0
  37. codex_autorunner/core/notifications.py +14 -2
  38. codex_autorunner/core/path_utils.py +2 -1
  39. codex_autorunner/core/pma_audit.py +224 -0
  40. codex_autorunner/core/pma_context.py +496 -0
  41. codex_autorunner/core/pma_dispatch_interceptor.py +284 -0
  42. codex_autorunner/core/pma_lifecycle.py +527 -0
  43. codex_autorunner/core/pma_queue.py +367 -0
  44. codex_autorunner/core/pma_safety.py +221 -0
  45. codex_autorunner/core/pma_state.py +115 -0
  46. codex_autorunner/core/ports/__init__.py +28 -0
  47. codex_autorunner/{integrations/agents → core/ports}/agent_backend.py +13 -8
  48. codex_autorunner/core/ports/backend_orchestrator.py +41 -0
  49. codex_autorunner/{integrations/agents → core/ports}/run_event.py +23 -6
  50. codex_autorunner/core/prompt.py +0 -80
  51. codex_autorunner/core/prompts.py +56 -172
  52. codex_autorunner/core/redaction.py +0 -4
  53. codex_autorunner/core/review_context.py +11 -9
  54. codex_autorunner/core/runner_controller.py +35 -33
  55. codex_autorunner/core/runner_state.py +147 -0
  56. codex_autorunner/core/runtime.py +829 -0
  57. codex_autorunner/core/sqlite_utils.py +13 -4
  58. codex_autorunner/core/state.py +7 -10
  59. codex_autorunner/core/state_roots.py +62 -0
  60. codex_autorunner/core/supervisor_protocol.py +15 -0
  61. codex_autorunner/core/templates/__init__.py +39 -0
  62. codex_autorunner/core/templates/git_mirror.py +234 -0
  63. codex_autorunner/core/templates/provenance.py +56 -0
  64. codex_autorunner/core/templates/scan_cache.py +120 -0
  65. codex_autorunner/core/text_delta_coalescer.py +54 -0
  66. codex_autorunner/core/ticket_linter_cli.py +218 -0
  67. codex_autorunner/core/ticket_manager_cli.py +494 -0
  68. codex_autorunner/core/time_utils.py +11 -0
  69. codex_autorunner/core/types.py +18 -0
  70. codex_autorunner/core/update.py +4 -5
  71. codex_autorunner/core/update_paths.py +28 -0
  72. codex_autorunner/core/usage.py +164 -12
  73. codex_autorunner/core/utils.py +125 -15
  74. codex_autorunner/flows/review/__init__.py +17 -0
  75. codex_autorunner/{core/review.py → flows/review/service.py} +37 -34
  76. codex_autorunner/flows/ticket_flow/definition.py +52 -3
  77. codex_autorunner/integrations/agents/__init__.py +11 -19
  78. codex_autorunner/integrations/agents/backend_orchestrator.py +302 -0
  79. codex_autorunner/integrations/agents/codex_adapter.py +90 -0
  80. codex_autorunner/integrations/agents/codex_backend.py +177 -25
  81. codex_autorunner/integrations/agents/opencode_adapter.py +108 -0
  82. codex_autorunner/integrations/agents/opencode_backend.py +305 -32
  83. codex_autorunner/integrations/agents/runner.py +86 -0
  84. codex_autorunner/integrations/agents/wiring.py +279 -0
  85. codex_autorunner/integrations/app_server/client.py +7 -60
  86. codex_autorunner/integrations/app_server/env.py +2 -107
  87. codex_autorunner/{core/app_server_events.py → integrations/app_server/event_buffer.py} +15 -8
  88. codex_autorunner/integrations/telegram/adapter.py +65 -0
  89. codex_autorunner/integrations/telegram/config.py +46 -0
  90. codex_autorunner/integrations/telegram/constants.py +1 -1
  91. codex_autorunner/integrations/telegram/doctor.py +228 -6
  92. codex_autorunner/integrations/telegram/handlers/callbacks.py +7 -0
  93. codex_autorunner/integrations/telegram/handlers/commands/execution.py +236 -74
  94. codex_autorunner/integrations/telegram/handlers/commands/files.py +314 -75
  95. codex_autorunner/integrations/telegram/handlers/commands/flows.py +1496 -71
  96. codex_autorunner/integrations/telegram/handlers/commands/workspace.py +498 -37
  97. codex_autorunner/integrations/telegram/handlers/commands_runtime.py +206 -48
  98. codex_autorunner/integrations/telegram/handlers/commands_spec.py +20 -3
  99. codex_autorunner/integrations/telegram/handlers/messages.py +27 -1
  100. codex_autorunner/integrations/telegram/handlers/selections.py +61 -1
  101. codex_autorunner/integrations/telegram/helpers.py +22 -1
  102. codex_autorunner/integrations/telegram/runtime.py +9 -4
  103. codex_autorunner/integrations/telegram/service.py +45 -10
  104. codex_autorunner/integrations/telegram/state.py +38 -0
  105. codex_autorunner/integrations/telegram/ticket_flow_bridge.py +338 -43
  106. codex_autorunner/integrations/telegram/transport.py +13 -4
  107. codex_autorunner/integrations/templates/__init__.py +27 -0
  108. codex_autorunner/integrations/templates/scan_agent.py +312 -0
  109. codex_autorunner/routes/__init__.py +37 -76
  110. codex_autorunner/routes/agents.py +2 -137
  111. codex_autorunner/routes/analytics.py +2 -238
  112. codex_autorunner/routes/app_server.py +2 -131
  113. codex_autorunner/routes/base.py +2 -596
  114. codex_autorunner/routes/file_chat.py +4 -833
  115. codex_autorunner/routes/flows.py +4 -977
  116. codex_autorunner/routes/messages.py +4 -456
  117. codex_autorunner/routes/repos.py +2 -196
  118. codex_autorunner/routes/review.py +2 -147
  119. codex_autorunner/routes/sessions.py +2 -175
  120. codex_autorunner/routes/settings.py +2 -168
  121. codex_autorunner/routes/shared.py +2 -275
  122. codex_autorunner/routes/system.py +4 -193
  123. codex_autorunner/routes/usage.py +2 -86
  124. codex_autorunner/routes/voice.py +2 -119
  125. codex_autorunner/routes/workspace.py +2 -270
  126. codex_autorunner/server.py +4 -4
  127. codex_autorunner/static/agentControls.js +61 -16
  128. codex_autorunner/static/app.js +126 -14
  129. codex_autorunner/static/archive.js +826 -0
  130. codex_autorunner/static/archiveApi.js +37 -0
  131. codex_autorunner/static/autoRefresh.js +7 -7
  132. codex_autorunner/static/chatUploads.js +137 -0
  133. codex_autorunner/static/dashboard.js +224 -171
  134. codex_autorunner/static/docChatCore.js +185 -13
  135. codex_autorunner/static/fileChat.js +68 -40
  136. codex_autorunner/static/fileboxUi.js +159 -0
  137. codex_autorunner/static/hub.js +114 -131
  138. codex_autorunner/static/index.html +375 -49
  139. codex_autorunner/static/messages.js +568 -87
  140. codex_autorunner/static/notifications.js +255 -0
  141. codex_autorunner/static/pma.js +1167 -0
  142. codex_autorunner/static/preserve.js +17 -0
  143. codex_autorunner/static/settings.js +128 -6
  144. codex_autorunner/static/smartRefresh.js +52 -0
  145. codex_autorunner/static/streamUtils.js +57 -0
  146. codex_autorunner/static/styles.css +9798 -6143
  147. codex_autorunner/static/tabs.js +152 -11
  148. codex_autorunner/static/templateReposSettings.js +225 -0
  149. codex_autorunner/static/terminal.js +18 -0
  150. codex_autorunner/static/ticketChatActions.js +165 -3
  151. codex_autorunner/static/ticketChatStream.js +17 -119
  152. codex_autorunner/static/ticketEditor.js +137 -15
  153. codex_autorunner/static/ticketTemplates.js +798 -0
  154. codex_autorunner/static/tickets.js +821 -98
  155. codex_autorunner/static/turnEvents.js +27 -0
  156. codex_autorunner/static/turnResume.js +33 -0
  157. codex_autorunner/static/utils.js +39 -0
  158. codex_autorunner/static/workspace.js +389 -82
  159. codex_autorunner/static/workspaceFileBrowser.js +15 -13
  160. codex_autorunner/surfaces/__init__.py +5 -0
  161. codex_autorunner/surfaces/cli/__init__.py +6 -0
  162. codex_autorunner/surfaces/cli/cli.py +2534 -0
  163. codex_autorunner/surfaces/cli/codex_cli.py +20 -0
  164. codex_autorunner/surfaces/cli/pma_cli.py +817 -0
  165. codex_autorunner/surfaces/telegram/__init__.py +3 -0
  166. codex_autorunner/surfaces/web/__init__.py +1 -0
  167. codex_autorunner/surfaces/web/app.py +2223 -0
  168. codex_autorunner/surfaces/web/hub_jobs.py +192 -0
  169. codex_autorunner/surfaces/web/middleware.py +587 -0
  170. codex_autorunner/surfaces/web/pty_session.py +370 -0
  171. codex_autorunner/surfaces/web/review.py +6 -0
  172. codex_autorunner/surfaces/web/routes/__init__.py +82 -0
  173. codex_autorunner/surfaces/web/routes/agents.py +138 -0
  174. codex_autorunner/surfaces/web/routes/analytics.py +284 -0
  175. codex_autorunner/surfaces/web/routes/app_server.py +132 -0
  176. codex_autorunner/surfaces/web/routes/archive.py +357 -0
  177. codex_autorunner/surfaces/web/routes/base.py +615 -0
  178. codex_autorunner/surfaces/web/routes/file_chat.py +1117 -0
  179. codex_autorunner/surfaces/web/routes/filebox.py +227 -0
  180. codex_autorunner/surfaces/web/routes/flows.py +1354 -0
  181. codex_autorunner/surfaces/web/routes/messages.py +490 -0
  182. codex_autorunner/surfaces/web/routes/pma.py +1652 -0
  183. codex_autorunner/surfaces/web/routes/repos.py +197 -0
  184. codex_autorunner/surfaces/web/routes/review.py +148 -0
  185. codex_autorunner/surfaces/web/routes/sessions.py +176 -0
  186. codex_autorunner/surfaces/web/routes/settings.py +169 -0
  187. codex_autorunner/surfaces/web/routes/shared.py +277 -0
  188. codex_autorunner/surfaces/web/routes/system.py +196 -0
  189. codex_autorunner/surfaces/web/routes/templates.py +634 -0
  190. codex_autorunner/surfaces/web/routes/usage.py +89 -0
  191. codex_autorunner/surfaces/web/routes/voice.py +120 -0
  192. codex_autorunner/surfaces/web/routes/workspace.py +271 -0
  193. codex_autorunner/surfaces/web/runner_manager.py +25 -0
  194. codex_autorunner/surfaces/web/schemas.py +469 -0
  195. codex_autorunner/surfaces/web/static_assets.py +490 -0
  196. codex_autorunner/surfaces/web/static_refresh.py +86 -0
  197. codex_autorunner/surfaces/web/terminal_sessions.py +78 -0
  198. codex_autorunner/tickets/__init__.py +8 -1
  199. codex_autorunner/tickets/agent_pool.py +53 -4
  200. codex_autorunner/tickets/files.py +37 -16
  201. codex_autorunner/tickets/lint.py +50 -0
  202. codex_autorunner/tickets/models.py +6 -1
  203. codex_autorunner/tickets/outbox.py +50 -2
  204. codex_autorunner/tickets/runner.py +396 -57
  205. codex_autorunner/web/__init__.py +5 -1
  206. codex_autorunner/web/app.py +2 -1949
  207. codex_autorunner/web/hub_jobs.py +2 -191
  208. codex_autorunner/web/middleware.py +2 -586
  209. codex_autorunner/web/pty_session.py +2 -369
  210. codex_autorunner/web/runner_manager.py +2 -24
  211. codex_autorunner/web/schemas.py +2 -376
  212. codex_autorunner/web/static_assets.py +4 -441
  213. codex_autorunner/web/static_refresh.py +2 -85
  214. codex_autorunner/web/terminal_sessions.py +2 -77
  215. codex_autorunner/workspace/paths.py +49 -33
  216. codex_autorunner-1.2.0.dist-info/METADATA +150 -0
  217. codex_autorunner-1.2.0.dist-info/RECORD +339 -0
  218. codex_autorunner/core/adapter_utils.py +0 -21
  219. codex_autorunner/core/engine.py +0 -2653
  220. codex_autorunner/core/static_assets.py +0 -55
  221. codex_autorunner-1.0.0.dist-info/METADATA +0 -246
  222. codex_autorunner-1.0.0.dist-info/RECORD +0 -251
  223. /codex_autorunner/{routes → surfaces/web/routes}/terminal_images.py +0 -0
  224. {codex_autorunner-1.0.0.dist-info → codex_autorunner-1.2.0.dist-info}/WHEEL +0 -0
  225. {codex_autorunner-1.0.0.dist-info → codex_autorunner-1.2.0.dist-info}/entry_points.txt +0 -0
  226. {codex_autorunner-1.0.0.dist-info → codex_autorunner-1.2.0.dist-info}/licenses/LICENSE +0 -0
  227. {codex_autorunner-1.0.0.dist-info → codex_autorunner-1.2.0.dist-info}/top_level.txt +0 -0
@@ -1,2653 +0,0 @@
1
- import asyncio
2
- import contextlib
3
- import dataclasses
4
- import json
5
- import logging
6
- import os
7
- import signal
8
- import threading
9
- import time
10
- import traceback
11
- from collections import Counter
12
- from datetime import datetime, timezone
13
- from logging.handlers import RotatingFileHandler
14
- from pathlib import Path
15
- from typing import IO, Any, Iterator, Optional
16
-
17
- import yaml
18
-
19
- from ..agents.opencode.logging import OpenCodeEventFormatter
20
- from ..agents.opencode.runtime import (
21
- OpenCodeTurnOutput,
22
- build_turn_id,
23
- collect_opencode_output,
24
- extract_session_id,
25
- map_approval_policy_to_permission,
26
- opencode_missing_env,
27
- parse_message_response,
28
- split_model_id,
29
- )
30
- from ..agents.opencode.supervisor import OpenCodeSupervisor, OpenCodeSupervisorError
31
- from ..agents.registry import validate_agent_id
32
- from ..integrations.app_server.client import (
33
- CodexAppServerError,
34
- _extract_thread_id,
35
- _extract_thread_id_for_turn,
36
- _extract_turn_id,
37
- )
38
- from ..integrations.app_server.env import build_app_server_env
39
- from ..integrations.app_server.supervisor import WorkspaceAppServerSupervisor
40
- from ..manifest import MANIFEST_VERSION
41
- from ..tickets.files import list_ticket_paths, ticket_is_done
42
- from .about_car import ensure_about_car_file
43
- from .adapter_utils import handle_agent_output
44
- from .app_server_events import AppServerEventBuffer
45
- from .app_server_logging import AppServerEventFormatter
46
- from .app_server_prompts import build_autorunner_prompt
47
- from .app_server_threads import AppServerThreadRegistry, default_app_server_threads_path
48
- from .config import (
49
- ConfigError,
50
- RepoConfig,
51
- _is_loopback_host,
52
- derive_repo_config,
53
- load_hub_config,
54
- load_repo_config,
55
- )
56
- from .docs import DocsManager, parse_todos
57
- from .git_utils import GitError, run_git
58
- from .locks import (
59
- DEFAULT_RUNNER_CMD_HINTS,
60
- FileLock,
61
- FileLockBusy,
62
- assess_lock,
63
- process_alive,
64
- read_lock_info,
65
- write_lock_info,
66
- )
67
- from .notifications import NotificationManager
68
- from .optional_dependencies import missing_optional_dependencies
69
- from .prompt import build_final_summary_prompt
70
- from .redaction import redact_text
71
- from .review_context import build_spec_progress_review_context
72
- from .run_index import RunIndexStore
73
- from .state import RunnerState, load_state, now_iso, save_state, state_lock
74
- from .static_assets import missing_static_assets, resolve_static_dir
75
- from .utils import (
76
- RepoNotFoundError,
77
- atomic_write,
78
- build_opencode_supervisor,
79
- ensure_executable,
80
- find_repo_root,
81
- )
82
-
83
-
84
- class LockError(Exception):
85
- pass
86
-
87
-
88
- def timestamp() -> str:
89
- return datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
90
-
91
-
92
- SUMMARY_FINALIZED_MARKER = "CAR:SUMMARY_FINALIZED"
93
- SUMMARY_FINALIZED_MARKER_PREFIX = f"<!-- {SUMMARY_FINALIZED_MARKER}"
94
- AUTORUNNER_APP_SERVER_MESSAGE = (
95
- "Continue working through TODO items from top to bottom."
96
- )
97
- AUTORUNNER_STOP_POLL_SECONDS = 1.0
98
- AUTORUNNER_INTERRUPT_GRACE_SECONDS = 30.0
99
-
100
-
101
- @dataclasses.dataclass
102
- class RunTelemetry:
103
- run_id: int
104
- thread_id: Optional[str] = None
105
- turn_id: Optional[str] = None
106
- token_total: Optional[dict[str, Any]] = None
107
- plan: Optional[Any] = None
108
- diff: Optional[Any] = None
109
-
110
-
111
- @dataclasses.dataclass
112
- class ActiveOpencodeRun:
113
- session_id: str
114
- turn_id: str
115
- client: Any
116
- interrupted: bool
117
- interrupt_event: asyncio.Event
118
-
119
-
120
- class Engine:
121
- def __init__(
122
- self,
123
- repo_root: Path,
124
- *,
125
- config: Optional[RepoConfig] = None,
126
- hub_path: Optional[Path] = None,
127
- ):
128
- if config is None:
129
- config = load_repo_config(repo_root, hub_path=hub_path)
130
- self.config = config
131
- self.repo_root = self.config.root
132
- self.docs = DocsManager(self.config)
133
- self.notifier = NotificationManager(self.config)
134
- self.state_path = self.repo_root / ".codex-autorunner" / "state.sqlite3"
135
- self.log_path = self.config.log.path
136
- self._run_index_store = RunIndexStore(self.state_path)
137
- self.lock_path = self.repo_root / ".codex-autorunner" / "lock"
138
- self.stop_path = self.repo_root / ".codex-autorunner" / "stop"
139
- self._hub_path = hub_path
140
- self._active_global_handler: Optional[RotatingFileHandler] = None
141
- self._active_run_log: Optional[IO[str]] = None
142
- self._app_server_threads = AppServerThreadRegistry(
143
- default_app_server_threads_path(self.repo_root)
144
- )
145
- self._app_server_threads_lock = threading.Lock()
146
- self._app_server_supervisor: Optional[WorkspaceAppServerSupervisor] = None
147
- self._app_server_logger = logging.getLogger("codex_autorunner.app_server")
148
- redact_enabled = self.config.security.get("redact_run_logs", True)
149
- self._app_server_event_formatter = AppServerEventFormatter(
150
- redact_enabled=redact_enabled
151
- )
152
- self._app_server_events = AppServerEventBuffer()
153
- self._opencode_event_formatter = OpenCodeEventFormatter()
154
- self._opencode_supervisor: Optional[OpenCodeSupervisor] = None
155
- self._run_telemetry_lock = threading.Lock()
156
- self._run_telemetry: Optional[RunTelemetry] = None
157
- self._last_telemetry_update_time: float = 0.0
158
- self._last_run_interrupted = False
159
- self._lock_handle: Optional[FileLock] = None
160
- # Ensure the interactive TUI briefing doc exists (for web Terminal "New").
161
- try:
162
- ensure_about_car_file(self.config)
163
- except (OSError, IOError) as exc:
164
- # Never fail Engine creation due to a best-effort helper doc.
165
- self._app_server_logger.debug(
166
- "Best-effort ABOUT_CAR.md creation failed: %s", exc
167
- )
168
-
169
- @staticmethod
170
- def from_cwd(repo: Optional[Path] = None) -> "Engine":
171
- root = find_repo_root(repo or Path.cwd())
172
- return Engine(root)
173
-
174
- def acquire_lock(self, force: bool = False) -> None:
175
- self._lock_handle = FileLock(self.lock_path)
176
- try:
177
- self._lock_handle.acquire(blocking=False)
178
- except FileLockBusy as exc:
179
- info = read_lock_info(self.lock_path)
180
- pid = info.pid
181
- if pid and process_alive(pid):
182
- raise LockError(
183
- f"Another autorunner is active (pid={pid}); stop it before continuing"
184
- ) from exc
185
- raise LockError(
186
- "Another autorunner is active; stop it before continuing"
187
- ) from exc
188
- info = read_lock_info(self.lock_path)
189
- pid = info.pid
190
- if pid and process_alive(pid) and not force:
191
- self._lock_handle.release()
192
- self._lock_handle = None
193
- raise LockError(
194
- f"Another autorunner is active (pid={pid}); use --force to override"
195
- )
196
- write_lock_info(
197
- self.lock_path,
198
- os.getpid(),
199
- started_at=now_iso(),
200
- lock_file=self._lock_handle.file,
201
- )
202
-
203
- def release_lock(self) -> None:
204
- if self._lock_handle is not None:
205
- self._lock_handle.release()
206
- self._lock_handle = None
207
- if self.lock_path.exists():
208
- self.lock_path.unlink()
209
-
210
- def repo_busy_reason(self) -> Optional[str]:
211
- if self.lock_path.exists():
212
- assessment = assess_lock(
213
- self.lock_path,
214
- expected_cmd_substrings=DEFAULT_RUNNER_CMD_HINTS,
215
- )
216
- if assessment.freeable:
217
- return "Autorunner lock is stale; clear it before continuing."
218
- pid = assessment.pid
219
- if pid and process_alive(pid):
220
- host = f" on {assessment.host}" if assessment.host else ""
221
- return f"Autorunner is running (pid={pid}{host}); try again later."
222
- return "Autorunner lock present; clear or resume before continuing."
223
-
224
- state = load_state(self.state_path)
225
- if state.status == "running":
226
- return "Autorunner is currently running; try again later."
227
- return None
228
-
229
- def request_stop(self) -> None:
230
- self.stop_path.parent.mkdir(parents=True, exist_ok=True)
231
- atomic_write(self.stop_path, f"{now_iso()}\n")
232
-
233
- def clear_stop_request(self) -> None:
234
- self.stop_path.unlink(missing_ok=True)
235
-
236
- def stop_requested(self) -> bool:
237
- return self.stop_path.exists()
238
-
239
- def _should_stop(self, external_stop_flag: Optional[threading.Event]) -> bool:
240
- if external_stop_flag and external_stop_flag.is_set():
241
- return True
242
- return self.stop_requested()
243
-
244
- def kill_running_process(self) -> Optional[int]:
245
- """Force-kill the process holding the lock, if any. Returns pid if killed."""
246
- if not self.lock_path.exists():
247
- return None
248
- info = read_lock_info(self.lock_path)
249
- pid = info.pid
250
- if pid and process_alive(pid):
251
- try:
252
- os.kill(pid, signal.SIGTERM)
253
- return pid
254
- except OSError:
255
- return None
256
- # stale lock
257
- self.lock_path.unlink(missing_ok=True)
258
- return None
259
-
260
- def runner_pid(self) -> Optional[int]:
261
- state = load_state(self.state_path)
262
- pid = state.runner_pid
263
- if pid and process_alive(pid):
264
- return pid
265
- info = read_lock_info(self.lock_path)
266
- if info.pid and process_alive(info.pid):
267
- return info.pid
268
- return None
269
-
270
- def todos_done(self) -> bool:
271
- # Ticket-first mode: completion is determined by ticket files, not TODO.md.
272
- ticket_dir = self.repo_root / ".codex-autorunner" / "tickets"
273
- ticket_paths = list_ticket_paths(ticket_dir)
274
- if not ticket_paths:
275
- return False
276
- return all(ticket_is_done(path) for path in ticket_paths)
277
-
278
- def summary_finalized(self) -> bool:
279
- # Legacy docs finalization no longer applies (no SUMMARY doc).
280
- return True
281
-
282
- def _stamp_summary_finalized(self, run_id: int) -> None:
283
- # No-op: summary file no longer exists.
284
- _ = run_id
285
- return
286
-
287
- async def _execute_run_step(
288
- self,
289
- prompt: str,
290
- run_id: int,
291
- *,
292
- external_stop_flag: Optional[threading.Event] = None,
293
- ) -> int:
294
- """
295
- Execute a single run step:
296
- 1. Update state to 'running'
297
- 2. Log start
298
- 3. Run Codex CLI
299
- 4. Log end
300
- 5. Update state to 'idle' or 'error'
301
- 6. Commit if successful and auto-commit is enabled
302
- """
303
- try:
304
- todo_before = self.docs.read_doc("todo")
305
- except (FileNotFoundError, OSError) as exc:
306
- self._app_server_logger.debug(
307
- "Failed to read TODO.md before run %s: %s", run_id, exc
308
- )
309
- todo_before = ""
310
- state = load_state(self.state_path)
311
- selected_agent = (state.autorunner_agent_override or "codex").strip().lower()
312
- try:
313
- validated_agent = validate_agent_id(selected_agent)
314
- except ValueError:
315
- validated_agent = "codex"
316
- self.log_line(
317
- run_id,
318
- f"info: unknown agent '{selected_agent}', defaulting to codex",
319
- )
320
- self._update_state("running", run_id, None, started=True)
321
- self._last_run_interrupted = False
322
- self._start_run_telemetry(run_id)
323
-
324
- actor: dict[str, Any] = {
325
- "backend": "codex_app_server",
326
- "agent_id": validated_agent,
327
- "surface": "hub" if self._hub_path else "cli",
328
- }
329
- mode: dict[str, Any] = {
330
- "approval_policy": state.autorunner_approval_policy or "never",
331
- "sandbox": state.autorunner_sandbox_mode or "dangerFullAccess",
332
- }
333
- runner_cfg = self.config.raw.get("runner") or {}
334
- review_cfg = runner_cfg.get("review")
335
- if isinstance(review_cfg, dict):
336
- mode["review_enabled"] = bool(review_cfg.get("enabled"))
337
-
338
- with self._run_log_context(run_id):
339
- self._write_run_marker(run_id, "start", actor=actor, mode=mode)
340
- if validated_agent == "opencode":
341
- exit_code = await self._run_opencode_app_server_async(
342
- prompt,
343
- run_id,
344
- model=state.autorunner_model_override,
345
- reasoning=state.autorunner_effort_override,
346
- external_stop_flag=external_stop_flag,
347
- )
348
- else:
349
- exit_code = await self._run_codex_app_server_async(
350
- prompt,
351
- run_id,
352
- external_stop_flag=external_stop_flag,
353
- )
354
- self._write_run_marker(run_id, "end", exit_code=exit_code)
355
-
356
- try:
357
- todo_after = self.docs.read_doc("todo")
358
- except (FileNotFoundError, OSError) as exc:
359
- self._app_server_logger.debug(
360
- "Failed to read TODO.md after run %s: %s", run_id, exc
361
- )
362
- todo_after = ""
363
- todo_delta = self._compute_todo_attribution(todo_before, todo_after)
364
- todo_snapshot = self._build_todo_snapshot(todo_before, todo_after)
365
- run_updates: dict[str, Any] = {
366
- "todo": todo_delta,
367
- "todo_snapshot": todo_snapshot,
368
- }
369
- telemetry = self._snapshot_run_telemetry(run_id)
370
- if (
371
- telemetry
372
- and telemetry.thread_id
373
- and isinstance(telemetry.token_total, dict)
374
- ):
375
- baseline = None
376
- # OpenCode reports per-turn totals, so skip cross-run deltas.
377
- if validated_agent != "opencode":
378
- baseline = self._find_thread_token_baseline(
379
- thread_id=telemetry.thread_id, run_id=run_id
380
- )
381
- delta = self._compute_token_delta(baseline, telemetry.token_total)
382
- run_updates["token_usage"] = {
383
- "delta": delta,
384
- "thread_total_before": baseline,
385
- "thread_total_after": telemetry.token_total,
386
- }
387
- artifacts: dict[str, str] = {}
388
- redact_enabled = self.config.security.get("redact_run_logs", True)
389
- if telemetry and telemetry.plan is not None:
390
- try:
391
- plan_content = (
392
- telemetry.plan
393
- if isinstance(telemetry.plan, str)
394
- else json.dumps(
395
- telemetry.plan, ensure_ascii=True, indent=2, default=str
396
- )
397
- )
398
- except (TypeError, ValueError) as exc:
399
- self._app_server_logger.debug(
400
- "Failed to serialize plan to JSON for run %s: %s", run_id, exc
401
- )
402
- plan_content = json.dumps(
403
- {"plan": str(telemetry.plan)}, ensure_ascii=True, indent=2
404
- )
405
- if redact_enabled:
406
- plan_content = redact_text(plan_content)
407
- plan_path = self._write_run_artifact(run_id, "plan.json", plan_content)
408
- artifacts["plan_path"] = str(plan_path)
409
- if telemetry and telemetry.diff is not None:
410
- normalized_diff = self._normalize_diff_payload(telemetry.diff)
411
- if normalized_diff is not None:
412
- diff_content = (
413
- normalized_diff
414
- if isinstance(normalized_diff, str)
415
- else json.dumps(
416
- normalized_diff, ensure_ascii=True, indent=2, default=str
417
- )
418
- )
419
- if redact_enabled:
420
- diff_content = redact_text(diff_content)
421
- diff_path = self._write_run_artifact(run_id, "diff.patch", diff_content)
422
- artifacts["diff_path"] = str(diff_path)
423
- if artifacts:
424
- run_updates["artifacts"] = artifacts
425
- if redact_enabled:
426
- from .redaction import get_redaction_patterns
427
-
428
- run_updates["security"] = {
429
- "redaction_enabled": True,
430
- "redaction_version": "1.0",
431
- "redaction_patterns": get_redaction_patterns(),
432
- }
433
- if run_updates:
434
- self._merge_run_index_entry(run_id, run_updates)
435
- self._clear_run_telemetry(run_id)
436
- self._update_state(
437
- "error" if exit_code != 0 else "idle",
438
- run_id,
439
- exit_code,
440
- finished=True,
441
- )
442
- if exit_code != 0:
443
- self.notifier.notify_run_finished(run_id=run_id, exit_code=exit_code)
444
-
445
- if exit_code == 0 and self.config.git_auto_commit:
446
- if self._last_run_interrupted:
447
- return exit_code
448
- self.maybe_git_commit(run_id)
449
-
450
- return exit_code
451
-
452
- async def _run_final_summary_job(
453
- self, run_id: int, *, external_stop_flag: Optional[threading.Event] = None
454
- ) -> int:
455
- """
456
- Run a dedicated Codex invocation that produces/updates SUMMARY.md as the final user report.
457
- """
458
- prev_output = self.extract_prev_output(run_id - 1)
459
- prompt = build_final_summary_prompt(self.config, self.docs, prev_output)
460
-
461
- exit_code = await self._execute_run_step(
462
- prompt, run_id, external_stop_flag=external_stop_flag
463
- )
464
-
465
- if exit_code == 0:
466
- self._stamp_summary_finalized(run_id)
467
- self.notifier.notify_run_finished(run_id=run_id, exit_code=exit_code)
468
- # Commit is already handled by _execute_run_step if auto-commit is enabled.
469
- return exit_code
470
-
471
- def extract_prev_output(self, run_id: int) -> Optional[str]:
472
- if run_id <= 0:
473
- return None
474
- run_log = self._run_log_path(run_id)
475
- if run_log.exists():
476
- try:
477
- text = run_log.read_text(encoding="utf-8")
478
- except (FileNotFoundError, OSError) as exc:
479
- self._app_server_logger.debug(
480
- "Failed to read previous run log for run %s: %s", run_id, exc
481
- )
482
- text = ""
483
- if text:
484
- lines = [
485
- line
486
- for line in text.splitlines()
487
- if not line.startswith("=== run ")
488
- ]
489
- text = _strip_log_prefixes("\n".join(lines))
490
- max_chars = self.config.prompt_prev_run_max_chars
491
- return text[-max_chars:] if text else None
492
- if not self.log_path.exists():
493
- return None
494
- start = f"=== run {run_id} start ==="
495
- end = f"=== run {run_id} end"
496
- # NOTE: do NOT read the full log file into memory. Logs can be very large
497
- # (especially with verbose Codex output) and this can OOM the server/runner.
498
- text = _read_tail_text(self.log_path, max_bytes=250_000)
499
- lines = text.splitlines()
500
- collecting = False
501
- collected = []
502
- for line in lines:
503
- if line.strip() == start:
504
- collecting = True
505
- continue
506
- if collecting and line.startswith(end):
507
- break
508
- if collecting:
509
- collected.append(line)
510
- if not collected:
511
- return None
512
- text = "\n".join(collected)
513
- text = _strip_log_prefixes(text)
514
- max_chars = self.config.prompt_prev_run_max_chars
515
- return text[-max_chars:]
516
-
517
- def read_run_block(self, run_id: int) -> Optional[str]:
518
- """Return a single run block from the log."""
519
- index_entry = self._load_run_index().get(str(run_id))
520
- run_log = None
521
- if index_entry:
522
- run_log_raw = index_entry.get("run_log_path")
523
- if isinstance(run_log_raw, str) and run_log_raw:
524
- run_log = Path(run_log_raw)
525
- if run_log is None:
526
- run_log = self._run_log_path(run_id)
527
- if run_log.exists():
528
- try:
529
- return run_log.read_text(encoding="utf-8")
530
- except (FileNotFoundError, OSError) as exc:
531
- self._app_server_logger.debug(
532
- "Failed to read run log block for run %s: %s", run_id, exc
533
- )
534
- return None
535
- if index_entry:
536
- block = self._read_log_range(run_id, index_entry)
537
- if block is not None:
538
- return block
539
- if not self.log_path.exists():
540
- return None
541
- start = f"=== run {run_id} start"
542
- end = f"=== run {run_id} end"
543
- # Avoid reading entire log into memory; prefer tail scan.
544
- max_bytes = 1_000_000
545
- text = _read_tail_text(self.log_path, max_bytes=max_bytes)
546
- lines = text.splitlines()
547
- buf = []
548
- printing = False
549
- for line in lines:
550
- if line.startswith(start):
551
- printing = True
552
- buf.append(line)
553
- continue
554
- if printing and line.startswith(end):
555
- buf.append(line)
556
- break
557
- if printing:
558
- buf.append(line)
559
- if buf:
560
- return "\n".join(buf)
561
- # If file is small, fall back to full read (safe).
562
- try:
563
- if self.log_path.stat().st_size <= max_bytes:
564
- lines = self.log_path.read_text(encoding="utf-8").splitlines()
565
- buf = []
566
- printing = False
567
- for line in lines:
568
- if line.startswith(start):
569
- printing = True
570
- buf.append(line)
571
- continue
572
- if printing and line.startswith(end):
573
- buf.append(line)
574
- break
575
- if printing:
576
- buf.append(line)
577
- return "\n".join(buf) if buf else None
578
- except (FileNotFoundError, OSError, ValueError) as exc:
579
- self._app_server_logger.debug(
580
- "Failed to read full log for run %s block: %s", run_id, exc
581
- )
582
- return None
583
- return None
584
-
585
- def tail_log(self, tail: int) -> str:
586
- if not self.log_path.exists():
587
- return ""
588
- # Bound memory usage: only read a chunk from the end.
589
- text = _read_tail_text(self.log_path, max_bytes=400_000)
590
- lines = text.splitlines()
591
- return "\n".join(lines[-tail:])
592
-
593
- def log_line(self, run_id: int, message: str) -> None:
594
- line = f"[{timestamp()}] run={run_id} {message}\n"
595
- if self._active_global_handler is not None:
596
- self._emit_global_line(line.rstrip("\n"))
597
- else:
598
- self._ensure_log_path()
599
- with self.log_path.open("a", encoding="utf-8") as f:
600
- f.write(line)
601
- if self._active_run_log is not None:
602
- try:
603
- self._active_run_log.write(line)
604
- self._active_run_log.flush()
605
- except (OSError, IOError) as exc:
606
- self._app_server_logger.warning(
607
- "Failed to write to active run log for run %s: %s", run_id, exc
608
- )
609
- else:
610
- run_log = self._run_log_path(run_id)
611
- self._ensure_run_log_dir()
612
- with run_log.open("a", encoding="utf-8") as f:
613
- f.write(line)
614
-
615
- def _emit_event(self, run_id: int, event: str, **payload: Any) -> None:
616
- import json as _json
617
-
618
- event_data = {
619
- "ts": timestamp(),
620
- "event": event,
621
- "run_id": run_id,
622
- }
623
- if payload:
624
- event_data.update(payload)
625
- events_path = self._events_log_path(run_id)
626
- self._ensure_run_log_dir()
627
- try:
628
- with events_path.open("a", encoding="utf-8") as f:
629
- f.write(_json.dumps(event_data) + "\n")
630
- except (OSError, IOError) as exc:
631
- self._app_server_logger.warning(
632
- "Failed to write event to events log for run %s: %s", run_id, exc
633
- )
634
-
635
- def _ensure_log_path(self) -> None:
636
- self.log_path.parent.mkdir(parents=True, exist_ok=True)
637
-
638
- def _run_log_path(self, run_id: int) -> Path:
639
- return self.log_path.parent / "runs" / f"run-{run_id}.log"
640
-
641
- def _events_log_path(self, run_id: int) -> Path:
642
- return self.log_path.parent / "runs" / f"run-{run_id}.events.jsonl"
643
-
644
- def _ensure_run_log_dir(self) -> None:
645
- (self.log_path.parent / "runs").mkdir(parents=True, exist_ok=True)
646
-
647
- def _write_run_marker(
648
- self,
649
- run_id: int,
650
- marker: str,
651
- exit_code: Optional[int] = None,
652
- *,
653
- actor: Optional[dict[str, Any]] = None,
654
- mode: Optional[dict[str, Any]] = None,
655
- ) -> None:
656
- suffix = ""
657
- if marker == "end":
658
- suffix = f" (code {exit_code})"
659
- self._emit_event(run_id, "run.finished", exit_code=exit_code)
660
- elif marker == "start":
661
- payload: dict[str, Any] = {}
662
- if actor is not None:
663
- payload["actor"] = actor
664
- if mode is not None:
665
- payload["mode"] = mode
666
- self._emit_event(run_id, "run.started", **payload)
667
- text = f"=== run {run_id} {marker}{suffix} ==="
668
- offset = self._emit_global_line(text)
669
- if self._active_run_log is not None:
670
- try:
671
- self._active_run_log.write(f"{text}\n")
672
- self._active_run_log.flush()
673
- except (OSError, IOError) as exc:
674
- self._app_server_logger.warning(
675
- "Failed to write marker to active run log for run %s: %s",
676
- run_id,
677
- exc,
678
- )
679
- else:
680
- self._ensure_run_log_dir()
681
- run_log = self._run_log_path(run_id)
682
- with run_log.open("a", encoding="utf-8") as f:
683
- f.write(f"{text}\n")
684
- self._update_run_index(
685
- run_id, marker, offset, exit_code, actor=actor, mode=mode
686
- )
687
-
688
- def _emit_global_line(self, text: str) -> Optional[tuple[int, int]]:
689
- if self._active_global_handler is None:
690
- self._ensure_log_path()
691
- try:
692
- with self.log_path.open("a", encoding="utf-8") as f:
693
- start = f.tell()
694
- f.write(f"{text}\n")
695
- f.flush()
696
- return (start, f.tell())
697
- except (OSError, IOError) as exc:
698
- self._app_server_logger.warning(
699
- "Failed to write global log line: %s", exc
700
- )
701
- return None
702
- handler = self._active_global_handler
703
- record = logging.LogRecord(
704
- name="codex_autorunner.engine",
705
- level=logging.INFO,
706
- pathname="",
707
- lineno=0,
708
- msg=text,
709
- args=(),
710
- exc_info=None,
711
- )
712
- handler.acquire()
713
- try:
714
- if handler.shouldRollover(record):
715
- handler.doRollover()
716
- if handler.stream is None:
717
- handler.stream = handler._open()
718
- start_offset = handler.stream.tell()
719
- logging.FileHandler.emit(handler, record)
720
- handler.flush()
721
- end_offset = handler.stream.tell()
722
- return (start_offset, end_offset)
723
- except (OSError, IOError, RuntimeError) as exc:
724
- self._app_server_logger.warning("Failed to emit log via handler: %s", exc)
725
- return None
726
- finally:
727
- handler.release()
728
-
729
- @contextlib.contextmanager
730
- def _run_log_context(self, run_id: int) -> Iterator[None]:
731
- self._ensure_log_path()
732
- self._ensure_run_log_dir()
733
- max_bytes = getattr(self.config.log, "max_bytes", None) or 0
734
- backup_count = getattr(self.config.log, "backup_count", 0) or 0
735
- handler = RotatingFileHandler(
736
- self.log_path,
737
- maxBytes=max_bytes,
738
- backupCount=backup_count,
739
- encoding="utf-8",
740
- )
741
- handler.setFormatter(logging.Formatter("%(message)s"))
742
- run_log = self._run_log_path(run_id)
743
- with run_log.open("a", encoding="utf-8") as run_handle:
744
- self._active_global_handler = handler
745
- self._active_run_log = run_handle
746
- try:
747
- yield
748
- finally:
749
- self._active_global_handler = None
750
- self._active_run_log = None
751
- try:
752
- handler.close()
753
- except (OSError, IOError) as exc:
754
- self._app_server_logger.debug(
755
- "Failed to close run log handler for run %s: %s", run_id, exc
756
- )
757
-
758
- def _start_run_telemetry(self, run_id: int) -> None:
759
- with self._run_telemetry_lock:
760
- self._run_telemetry = RunTelemetry(run_id=run_id)
761
- self._app_server_event_formatter.reset()
762
- self._opencode_event_formatter.reset()
763
-
764
- def _update_run_telemetry(self, run_id: int, **updates: Any) -> None:
765
- with self._run_telemetry_lock:
766
- telemetry = self._run_telemetry
767
- if telemetry is None or telemetry.run_id != run_id:
768
- return
769
- for key, value in updates.items():
770
- if hasattr(telemetry, key):
771
- setattr(telemetry, key, value)
772
-
773
- def _snapshot_run_telemetry(self, run_id: int) -> Optional[RunTelemetry]:
774
- with self._run_telemetry_lock:
775
- telemetry = self._run_telemetry
776
- if telemetry is None or telemetry.run_id != run_id:
777
- return None
778
- return dataclasses.replace(telemetry)
779
-
780
- def _clear_run_telemetry(self, run_id: int) -> None:
781
- with self._run_telemetry_lock:
782
- telemetry = self._run_telemetry
783
- if telemetry is None or telemetry.run_id != run_id:
784
- return
785
- self._run_telemetry = None
786
-
787
- @staticmethod
788
- def _normalize_diff_payload(diff: Any) -> Optional[Any]:
789
- if diff is None:
790
- return None
791
- if isinstance(diff, str):
792
- return diff if diff.strip() else None
793
- if isinstance(diff, dict):
794
- # Prefer meaningful fields if present.
795
- for key in ("diff", "patch", "content", "value"):
796
- if key in diff:
797
- val = diff.get(key)
798
- if isinstance(val, str) and val.strip():
799
- return val
800
- if val not in (None, "", [], {}, ()):
801
- return diff
802
- for val in diff.values():
803
- if isinstance(val, str) and val.strip():
804
- return diff
805
- if val not in (None, "", [], {}, ()):
806
- return diff
807
- return None
808
- return diff
809
-
810
- def _maybe_update_run_index_telemetry(
811
- self, run_id: int, min_interval_seconds: float = 3.0
812
- ) -> None:
813
- import time as _time
814
-
815
- now = _time.time()
816
- if now - self._last_telemetry_update_time < min_interval_seconds:
817
- return
818
- telemetry = self._snapshot_run_telemetry(run_id)
819
- if telemetry is None:
820
- return
821
- if telemetry.thread_id and isinstance(telemetry.token_total, dict):
822
- with state_lock(self.state_path):
823
- state = load_state(self.state_path)
824
- selected_agent = (
825
- (state.autorunner_agent_override or "codex").strip().lower()
826
- )
827
- baseline = None
828
- if selected_agent != "opencode":
829
- baseline = self._find_thread_token_baseline(
830
- thread_id=telemetry.thread_id, run_id=run_id
831
- )
832
- delta = self._compute_token_delta(baseline, telemetry.token_total)
833
- self._merge_run_index_entry(
834
- run_id,
835
- {
836
- "token_usage": {
837
- "delta": delta,
838
- "thread_total_before": baseline,
839
- "thread_total_after": telemetry.token_total,
840
- }
841
- },
842
- )
843
- self._last_telemetry_update_time = now
844
-
845
- async def _handle_app_server_notification(self, message: dict[str, Any]) -> None:
846
- if not isinstance(message, dict):
847
- return
848
- method = message.get("method")
849
- params_raw = message.get("params")
850
- params = params_raw if isinstance(params_raw, dict) else {}
851
- thread_id = (
852
- _extract_thread_id_for_turn(params)
853
- or _extract_thread_id(params)
854
- or _extract_thread_id(message)
855
- )
856
- turn_id = _extract_turn_id(params) or _extract_turn_id(message)
857
- run_id: Optional[int] = None
858
- with self._run_telemetry_lock:
859
- telemetry = self._run_telemetry
860
- if telemetry is None:
861
- return
862
- if telemetry.thread_id and thread_id and telemetry.thread_id != thread_id:
863
- return
864
- if telemetry.turn_id and turn_id and telemetry.turn_id != turn_id:
865
- return
866
- if telemetry.thread_id is None and thread_id:
867
- telemetry.thread_id = thread_id
868
- if telemetry.turn_id is None and turn_id:
869
- telemetry.turn_id = turn_id
870
- run_id = telemetry.run_id
871
- if method == "thread/tokenUsage/updated":
872
- token_usage = (
873
- params.get("token_usage") or params.get("tokenUsage") or {}
874
- )
875
- if isinstance(token_usage, dict):
876
- total = token_usage.get("total") or token_usage.get("totals")
877
- if isinstance(total, dict):
878
- telemetry.token_total = total
879
- self._maybe_update_run_index_telemetry(run_id)
880
- self._emit_event(run_id, "token.updated", token_total=total)
881
- if method == "turn/plan/updated":
882
- telemetry.plan = params.get("plan") if "plan" in params else params
883
- if method == "turn/diff/updated":
884
- diff: Any = None
885
- for key in ("diff", "patch", "content", "value"):
886
- if key in params:
887
- diff = params.get(key)
888
- break
889
- telemetry.diff = diff if diff is not None else params or None
890
- if run_id is None:
891
- return
892
- for line in self._app_server_event_formatter.format_event(message):
893
- self.log_line(run_id, f"stdout: {line}" if line else "stdout: ")
894
-
895
- def _load_run_index(self) -> dict[str, dict]:
896
- return self._run_index_store.load_all()
897
-
898
- def reconcile_run_index(self) -> None:
899
- """Best-effort: mark stale runs that still look 'running' in the run index.
900
-
901
- The Runs UI considers a run "running" when both `finished_at` and `exit_code`
902
- are missing. If the runner process was killed or crashed, the `end` marker is
903
- never written, so the entry stays "running" forever. This method uses the
904
- runner state + lock pid as the authoritative signal for whether a run can
905
- still be active, then forces stale entries to a finished/error state.
906
- """
907
- try:
908
- state = load_state(self.state_path)
909
- except Exception as exc:
910
- self._app_server_logger.warning(
911
- "Failed to load state during run index reconciliation: %s", exc
912
- )
913
- return
914
-
915
- active_pid: Optional[int] = None
916
- pid = state.runner_pid
917
- if pid and process_alive(pid):
918
- active_pid = pid
919
- else:
920
- info = read_lock_info(self.lock_path)
921
- if info.pid and process_alive(info.pid):
922
- active_pid = info.pid
923
-
924
- active_run_id: Optional[int] = None
925
- if (
926
- active_pid is not None
927
- and state.status == "running"
928
- and state.last_run_id is not None
929
- ):
930
- active_run_id = int(state.last_run_id)
931
-
932
- now = now_iso()
933
- try:
934
- index = self._run_index_store.load_all()
935
- except Exception as exc:
936
- self._app_server_logger.warning(
937
- "Failed to load run index during reconciliation: %s", exc
938
- )
939
- return
940
-
941
- for key, entry in index.items():
942
- try:
943
- run_id = int(key)
944
- except (TypeError, ValueError):
945
- continue
946
- if not isinstance(entry, dict):
947
- continue
948
- if run_id <= 0:
949
- continue
950
-
951
- if active_run_id is not None and run_id == active_run_id:
952
- continue
953
-
954
- if entry.get("reconciled_at") is not None:
955
- continue
956
-
957
- finished_at = entry.get("finished_at")
958
- exit_code = entry.get("exit_code")
959
-
960
- if isinstance(finished_at, str) and finished_at:
961
- continue
962
- if exit_code is not None:
963
- continue
964
-
965
- inferred_exit: int
966
- if state.last_run_id == run_id and state.last_exit_code is not None:
967
- inferred_exit = int(state.last_exit_code)
968
- else:
969
- inferred_exit = 1
970
-
971
- try:
972
- self._run_index_store.merge_entry(
973
- run_id,
974
- {
975
- "finished_at": now,
976
- "exit_code": inferred_exit,
977
- "reconciled_at": now,
978
- "reconciled_reason": (
979
- "runner_active"
980
- if active_pid is not None
981
- else "runner_inactive"
982
- ),
983
- },
984
- )
985
- except Exception as exc:
986
- self._app_server_logger.warning(
987
- "Failed to reconcile run index entry for run %d: %s", run_id, exc
988
- )
989
- continue
990
-
991
- def _merge_run_index_entry(self, run_id: int, updates: dict[str, Any]) -> None:
992
- self._run_index_store.merge_entry(run_id, updates)
993
-
994
- def _update_run_index(
995
- self,
996
- run_id: int,
997
- marker: str,
998
- offset: Optional[tuple[int, int]],
999
- exit_code: Optional[int],
1000
- *,
1001
- actor: Optional[dict[str, Any]] = None,
1002
- mode: Optional[dict[str, Any]] = None,
1003
- ) -> None:
1004
- self._run_index_store.update_marker(
1005
- run_id,
1006
- marker,
1007
- offset,
1008
- exit_code,
1009
- log_path=str(self.log_path),
1010
- run_log_path=str(self._run_log_path(run_id)),
1011
- actor=actor,
1012
- mode=mode,
1013
- )
1014
-
1015
- def _list_from_counts(self, source: list[str], counts: Counter[str]) -> list[str]:
1016
- if not source or not counts:
1017
- return []
1018
- remaining = Counter(counts)
1019
- items: list[str] = []
1020
- for entry in source:
1021
- if remaining[entry] > 0:
1022
- items.append(entry)
1023
- remaining[entry] -= 1
1024
- return items
1025
-
1026
- def _compute_todo_attribution(
1027
- self, before_text: str, after_text: str
1028
- ) -> dict[str, Any]:
1029
- before_out, before_done = parse_todos(before_text or "")
1030
- after_out, after_done = parse_todos(after_text or "")
1031
- before_out_counter = Counter(before_out)
1032
- before_done_counter = Counter(before_done)
1033
- after_out_counter = Counter(after_out)
1034
- after_done_counter = Counter(after_done)
1035
-
1036
- completed_counts: Counter[str] = Counter()
1037
- for item, count in after_done_counter.items():
1038
- if before_out_counter[item] > 0:
1039
- completed_counts[item] = min(before_out_counter[item], count)
1040
-
1041
- reopened_counts: Counter[str] = Counter()
1042
- for item, count in after_out_counter.items():
1043
- if before_done_counter[item] > 0:
1044
- reopened_counts[item] = min(before_done_counter[item], count)
1045
-
1046
- new_outstanding_counts = after_out_counter - before_out_counter
1047
- added_counts = new_outstanding_counts - reopened_counts
1048
-
1049
- completed = self._list_from_counts(after_done, completed_counts)
1050
- reopened = self._list_from_counts(after_out, reopened_counts)
1051
- added = self._list_from_counts(after_out, added_counts)
1052
-
1053
- return {
1054
- "completed": completed,
1055
- "added": added,
1056
- "reopened": reopened,
1057
- "counts": {
1058
- "completed": len(completed),
1059
- "added": len(added),
1060
- "reopened": len(reopened),
1061
- },
1062
- }
1063
-
1064
- def _build_todo_snapshot(self, before_text: str, after_text: str) -> dict[str, Any]:
1065
- before_out, before_done = parse_todos(before_text or "")
1066
- after_out, after_done = parse_todos(after_text or "")
1067
- return {
1068
- "before": {
1069
- "outstanding": before_out,
1070
- "done": before_done,
1071
- "counts": {
1072
- "outstanding": len(before_out),
1073
- "done": len(before_done),
1074
- },
1075
- },
1076
- "after": {
1077
- "outstanding": after_out,
1078
- "done": after_done,
1079
- "counts": {
1080
- "outstanding": len(after_out),
1081
- "done": len(after_done),
1082
- },
1083
- },
1084
- }
1085
-
1086
- def _find_thread_token_baseline(
1087
- self, *, thread_id: str, run_id: int
1088
- ) -> Optional[dict[str, Any]]:
1089
- index = self._load_run_index()
1090
- best_run = -1
1091
- baseline: Optional[dict[str, Any]] = None
1092
- for key, entry in index.items():
1093
- try:
1094
- entry_id = int(key)
1095
- except (TypeError, ValueError) as exc:
1096
- self._app_server_logger.debug(
1097
- "Failed to parse run index key '%s' while resolving run %s: %s",
1098
- key,
1099
- run_id,
1100
- exc,
1101
- )
1102
- continue
1103
- if entry_id >= run_id:
1104
- continue
1105
- app_server = entry.get("app_server")
1106
- if not isinstance(app_server, dict):
1107
- continue
1108
- if app_server.get("thread_id") != thread_id:
1109
- continue
1110
- token_usage = entry.get("token_usage")
1111
- if not isinstance(token_usage, dict):
1112
- continue
1113
- total = token_usage.get("thread_total_after")
1114
- if isinstance(total, dict) and entry_id > best_run:
1115
- best_run = entry_id
1116
- baseline = total
1117
- return baseline
1118
-
1119
- def _compute_token_delta(
1120
- self,
1121
- baseline: Optional[dict[str, Any]],
1122
- final_total: Optional[dict[str, Any]],
1123
- ) -> Optional[dict[str, Any]]:
1124
- if not isinstance(final_total, dict):
1125
- return None
1126
- base = baseline if isinstance(baseline, dict) else {}
1127
- delta: dict[str, Any] = {}
1128
- for key, value in final_total.items():
1129
- if not isinstance(value, (int, float)):
1130
- continue
1131
- prior = base.get(key, 0)
1132
- if isinstance(prior, (int, float)):
1133
- delta[key] = value - prior
1134
- else:
1135
- delta[key] = value
1136
- return delta
1137
-
1138
- def _build_app_server_meta(
1139
- self,
1140
- *,
1141
- thread_id: str,
1142
- turn_id: str,
1143
- thread_info: Optional[dict[str, Any]],
1144
- model: Optional[str],
1145
- reasoning_effort: Optional[str],
1146
- ) -> dict[str, Any]:
1147
- meta: dict[str, Any] = {"thread_id": thread_id, "turn_id": turn_id}
1148
- if model:
1149
- meta["model"] = model
1150
- if reasoning_effort:
1151
- meta["reasoning_effort"] = reasoning_effort
1152
- if not isinstance(thread_info, dict):
1153
- return meta
1154
-
1155
- def _first_string(keys: tuple[str, ...]) -> Optional[str]:
1156
- for key in keys:
1157
- value = thread_info.get(key)
1158
- if isinstance(value, str) and value:
1159
- return value
1160
- return None
1161
-
1162
- if "model" not in meta:
1163
- thread_model = _first_string(("model", "model_id", "modelId", "model_name"))
1164
- if thread_model:
1165
- meta["model"] = thread_model
1166
- provider = _first_string(
1167
- ("model_provider", "modelProvider", "provider", "model_provider_name")
1168
- )
1169
- if provider:
1170
- meta["model_provider"] = provider
1171
- if "reasoning_effort" not in meta:
1172
- thread_effort = _first_string(
1173
- ("reasoning_effort", "reasoningEffort", "effort")
1174
- )
1175
- if thread_effort:
1176
- meta["reasoning_effort"] = thread_effort
1177
- return meta
1178
-
1179
- def _write_run_artifact(self, run_id: int, name: str, content: str) -> Path:
1180
- self._ensure_run_log_dir()
1181
- path = self.log_path.parent / "runs" / f"run-{run_id}.{name}"
1182
- atomic_write(path, content)
1183
- return path
1184
-
1185
- def _read_log_range(self, run_id: int, entry: dict) -> Optional[str]:
1186
- start = entry.get("start_offset")
1187
- end = entry.get("end_offset")
1188
- if start is None or end is None:
1189
- return None
1190
- try:
1191
- start_offset = int(start)
1192
- end_offset = int(end)
1193
- except (TypeError, ValueError) as exc:
1194
- self._app_server_logger.debug(
1195
- "Failed to parse log range offsets for run %s: %s", run_id, exc
1196
- )
1197
- return None
1198
- if end_offset < start_offset:
1199
- return None
1200
- log_path = Path(entry.get("log_path", self.log_path))
1201
- if not log_path.exists():
1202
- return None
1203
- try:
1204
- size = log_path.stat().st_size
1205
- if size < end_offset:
1206
- return None
1207
- with log_path.open("rb") as f:
1208
- f.seek(start_offset)
1209
- data = f.read(end_offset - start_offset)
1210
- return data.decode("utf-8", errors="replace")
1211
- except (FileNotFoundError, OSError) as exc:
1212
- self._app_server_logger.debug(
1213
- "Failed to read log range for run %s: %s", run_id, exc
1214
- )
1215
- return None
1216
-
1217
- def _build_app_server_prompt(self, prev_output: Optional[str]) -> str:
1218
- return build_autorunner_prompt(
1219
- self.config,
1220
- message=AUTORUNNER_APP_SERVER_MESSAGE,
1221
- prev_run_summary=prev_output,
1222
- )
1223
-
1224
- def run_codex_app_server(
1225
- self,
1226
- prompt: str,
1227
- run_id: int,
1228
- *,
1229
- external_stop_flag: Optional[threading.Event] = None,
1230
- ) -> int:
1231
- try:
1232
- return asyncio.run(
1233
- self._run_codex_app_server_async(
1234
- prompt,
1235
- run_id,
1236
- external_stop_flag=external_stop_flag,
1237
- reuse_supervisor=False,
1238
- )
1239
- )
1240
- except RuntimeError as exc:
1241
- if "asyncio.run" in str(exc):
1242
- self.log_line(
1243
- run_id,
1244
- "error: app-server backend cannot run inside an active event loop",
1245
- )
1246
- return 1
1247
- raise
1248
-
1249
- async def _run_codex_app_server_async(
1250
- self,
1251
- prompt: str,
1252
- run_id: int,
1253
- *,
1254
- external_stop_flag: Optional[threading.Event] = None,
1255
- reuse_supervisor: bool = True,
1256
- ) -> int:
1257
- config = self.config
1258
- if not config.app_server.command:
1259
- self.log_line(
1260
- run_id,
1261
- "error: app-server backend requires app_server.command to be configured",
1262
- )
1263
- return 1
1264
-
1265
- def _env_builder(
1266
- workspace_root: Path, _workspace_id: str, state_dir: Path
1267
- ) -> dict[str, str]:
1268
- state_dir.mkdir(parents=True, exist_ok=True)
1269
- return build_app_server_env(
1270
- config.app_server.command,
1271
- workspace_root,
1272
- state_dir,
1273
- logger=self._app_server_logger,
1274
- event_prefix="autorunner",
1275
- )
1276
-
1277
- supervisor = (
1278
- self._ensure_app_server_supervisor(_env_builder)
1279
- if reuse_supervisor
1280
- else self._build_app_server_supervisor(_env_builder)
1281
- )
1282
- with state_lock(self.state_path):
1283
- state = load_state(self.state_path)
1284
- effective_model = state.autorunner_model_override or config.codex_model
1285
- effective_effort = state.autorunner_effort_override or config.codex_reasoning
1286
- approval_policy = state.autorunner_approval_policy or "never"
1287
- sandbox_mode = state.autorunner_sandbox_mode or "dangerFullAccess"
1288
- if sandbox_mode == "workspaceWrite":
1289
- sandbox_policy: Any = {
1290
- "type": "workspaceWrite",
1291
- "writableRoots": [str(self.repo_root)],
1292
- "networkAccess": bool(state.autorunner_workspace_write_network),
1293
- }
1294
- else:
1295
- sandbox_policy = sandbox_mode
1296
- try:
1297
- client = await supervisor.get_client(self.repo_root)
1298
- with self._app_server_threads_lock:
1299
- thread_id = self._app_server_threads.get_thread_id("autorunner")
1300
- thread_info: Optional[dict[str, Any]] = None
1301
- if thread_id:
1302
- try:
1303
- resume_result = await client.thread_resume(thread_id)
1304
- resumed = resume_result.get("id")
1305
- if isinstance(resumed, str) and resumed:
1306
- thread_id = resumed
1307
- self._app_server_threads.set_thread_id(
1308
- "autorunner", thread_id
1309
- )
1310
- if isinstance(resume_result, dict):
1311
- thread_info = resume_result
1312
- except CodexAppServerError:
1313
- self._app_server_threads.reset_thread("autorunner")
1314
- thread_id = None
1315
- if not thread_id:
1316
- thread = await client.thread_start(str(self.repo_root))
1317
- thread_id = thread.get("id")
1318
- if not isinstance(thread_id, str) or not thread_id:
1319
- self.log_line(
1320
- run_id, "error: app-server did not return a thread id"
1321
- )
1322
- return 1
1323
- self._app_server_threads.set_thread_id("autorunner", thread_id)
1324
- if isinstance(thread, dict):
1325
- thread_info = thread
1326
- if thread_id:
1327
- self._update_run_telemetry(run_id, thread_id=thread_id)
1328
- turn_kwargs: dict[str, Any] = {}
1329
- if effective_model:
1330
- turn_kwargs["model"] = str(effective_model)
1331
- if effective_effort:
1332
- turn_kwargs["effort"] = str(effective_effort)
1333
- handle = await client.turn_start(
1334
- thread_id,
1335
- prompt,
1336
- approval_policy=approval_policy,
1337
- sandbox_policy=sandbox_policy,
1338
- **turn_kwargs,
1339
- )
1340
- app_server_meta = self._build_app_server_meta(
1341
- thread_id=thread_id,
1342
- turn_id=handle.turn_id,
1343
- thread_info=thread_info,
1344
- model=turn_kwargs.get("model"),
1345
- reasoning_effort=turn_kwargs.get("effort"),
1346
- )
1347
- self._merge_run_index_entry(run_id, {"app_server": app_server_meta})
1348
- self._update_run_telemetry(
1349
- run_id, thread_id=thread_id, turn_id=handle.turn_id
1350
- )
1351
- turn_timeout = config.app_server.turn_timeout_seconds
1352
- turn_result, interrupted = await self._wait_for_turn_with_stop(
1353
- client,
1354
- handle,
1355
- run_id,
1356
- timeout=turn_timeout,
1357
- external_stop_flag=external_stop_flag,
1358
- supervisor=supervisor,
1359
- )
1360
- self._last_run_interrupted = interrupted
1361
- handle_agent_output(
1362
- self._log_app_server_output,
1363
- self._write_run_artifact,
1364
- self._merge_run_index_entry,
1365
- run_id,
1366
- turn_result.agent_messages,
1367
- )
1368
- if turn_result.errors:
1369
- for error in turn_result.errors:
1370
- self.log_line(run_id, f"error: {error}")
1371
- return 1
1372
- return 0
1373
- except asyncio.TimeoutError:
1374
- self.log_line(run_id, "error: app-server turn timed out")
1375
- return 1
1376
- except CodexAppServerError as exc:
1377
- self.log_line(run_id, f"error: {exc}")
1378
- return 1
1379
- except Exception as exc: # pragma: no cover - defensive
1380
- self.log_line(run_id, f"error: app-server failed: {exc}")
1381
- return 1
1382
- finally:
1383
- if not reuse_supervisor:
1384
- await supervisor.close_all()
1385
-
1386
- def _log_app_server_output(self, run_id: int, messages: list[str]) -> None:
1387
- if not messages:
1388
- return
1389
- for message in messages:
1390
- text = str(message)
1391
- lines = text.splitlines() or [""]
1392
- for line in lines:
1393
- self.log_line(run_id, f"stdout: {line}" if line else "stdout: ")
1394
-
1395
- def maybe_git_commit(self, run_id: int) -> None:
1396
- msg = self.config.git_commit_message_template.replace(
1397
- "{run_id}", str(run_id)
1398
- ).replace("#{run_id}", str(run_id))
1399
- paths = []
1400
- for key in ("active_context", "decisions", "spec"):
1401
- try:
1402
- paths.append(self.config.doc_path(key))
1403
- except KeyError:
1404
- pass
1405
- add_paths = [str(p.relative_to(self.repo_root)) for p in paths if p.exists()]
1406
- if not add_paths:
1407
- return
1408
- try:
1409
- add_proc = run_git(["add", *add_paths], self.repo_root, check=False)
1410
- if add_proc.returncode != 0:
1411
- detail = (
1412
- add_proc.stderr or add_proc.stdout or ""
1413
- ).strip() or f"exit {add_proc.returncode}"
1414
- self.log_line(run_id, f"git add failed: {detail}")
1415
- return
1416
- except GitError as exc:
1417
- self.log_line(run_id, f"git add failed: {exc}")
1418
- return
1419
- try:
1420
- commit_proc = run_git(
1421
- ["commit", "-m", msg],
1422
- self.repo_root,
1423
- check=False,
1424
- timeout_seconds=120,
1425
- )
1426
- if commit_proc.returncode != 0:
1427
- detail = (
1428
- commit_proc.stderr or commit_proc.stdout or ""
1429
- ).strip() or f"exit {commit_proc.returncode}"
1430
- self.log_line(run_id, f"git commit failed: {detail}")
1431
- except GitError as exc:
1432
- self.log_line(run_id, f"git commit failed: {exc}")
1433
-
1434
- def _build_app_server_supervisor(
1435
- self, env_builder: Any
1436
- ) -> WorkspaceAppServerSupervisor:
1437
- config = self.config.app_server
1438
- return WorkspaceAppServerSupervisor(
1439
- config.command,
1440
- state_root=config.state_root,
1441
- env_builder=env_builder,
1442
- logger=self._app_server_logger,
1443
- notification_handler=self._handle_app_server_notification,
1444
- auto_restart=config.auto_restart,
1445
- max_handles=config.max_handles,
1446
- idle_ttl_seconds=config.idle_ttl_seconds,
1447
- request_timeout=config.request_timeout,
1448
- turn_stall_timeout_seconds=config.turn_stall_timeout_seconds,
1449
- turn_stall_poll_interval_seconds=config.turn_stall_poll_interval_seconds,
1450
- turn_stall_recovery_min_interval_seconds=config.turn_stall_recovery_min_interval_seconds,
1451
- max_message_bytes=config.client.max_message_bytes,
1452
- oversize_preview_bytes=config.client.oversize_preview_bytes,
1453
- max_oversize_drain_bytes=config.client.max_oversize_drain_bytes,
1454
- restart_backoff_initial_seconds=config.client.restart_backoff_initial_seconds,
1455
- restart_backoff_max_seconds=config.client.restart_backoff_max_seconds,
1456
- restart_backoff_jitter_ratio=config.client.restart_backoff_jitter_ratio,
1457
- )
1458
-
1459
- def _ensure_app_server_supervisor(
1460
- self, env_builder: Any
1461
- ) -> WorkspaceAppServerSupervisor:
1462
- if self._app_server_supervisor is None:
1463
- self._app_server_supervisor = self._build_app_server_supervisor(env_builder)
1464
- return self._app_server_supervisor
1465
-
1466
- async def _close_app_server_supervisor(self) -> None:
1467
- if self._app_server_supervisor is None:
1468
- return
1469
- supervisor = self._app_server_supervisor
1470
- self._app_server_supervisor = None
1471
- try:
1472
- await supervisor.close_all()
1473
- except Exception as exc:
1474
- self._app_server_logger.warning(
1475
- "app-server supervisor close failed: %s", exc
1476
- )
1477
-
1478
- def _build_opencode_supervisor(self) -> Optional[OpenCodeSupervisor]:
1479
- config = self.config.app_server
1480
- opencode_command = self.config.agent_serve_command("opencode")
1481
- opencode_binary = None
1482
- try:
1483
- opencode_binary = self.config.agent_binary("opencode")
1484
- except ConfigError:
1485
- opencode_binary = None
1486
-
1487
- agent_config = self.config.agents.get("opencode")
1488
- subagent_models = agent_config.subagent_models if agent_config else None
1489
-
1490
- supervisor = build_opencode_supervisor(
1491
- opencode_command=opencode_command,
1492
- opencode_binary=opencode_binary,
1493
- workspace_root=self.repo_root,
1494
- logger=self._app_server_logger,
1495
- request_timeout=config.request_timeout,
1496
- max_handles=config.max_handles,
1497
- idle_ttl_seconds=config.idle_ttl_seconds,
1498
- session_stall_timeout_seconds=self.config.opencode.session_stall_timeout_seconds,
1499
- base_env=None,
1500
- subagent_models=subagent_models,
1501
- )
1502
-
1503
- if supervisor is None:
1504
- self._app_server_logger.info(
1505
- "OpenCode command unavailable; skipping opencode supervisor."
1506
- )
1507
- return None
1508
-
1509
- return supervisor
1510
-
1511
- def _ensure_opencode_supervisor(self) -> Optional[OpenCodeSupervisor]:
1512
- if self._opencode_supervisor is None:
1513
- self._opencode_supervisor = self._build_opencode_supervisor()
1514
- return self._opencode_supervisor
1515
-
1516
- async def _close_opencode_supervisor(self) -> None:
1517
- if self._opencode_supervisor is None:
1518
- return
1519
- supervisor = self._opencode_supervisor
1520
- self._opencode_supervisor = None
1521
- try:
1522
- await supervisor.close_all()
1523
- except Exception as exc:
1524
- self._app_server_logger.warning("opencode supervisor close failed: %s", exc)
1525
-
1526
- async def _wait_for_stop(
1527
- self,
1528
- external_stop_flag: Optional[threading.Event],
1529
- stop_event: Optional[asyncio.Event] = None,
1530
- ) -> None:
1531
- while not self._should_stop(external_stop_flag):
1532
- await asyncio.sleep(AUTORUNNER_STOP_POLL_SECONDS)
1533
- if stop_event is not None:
1534
- stop_event.set()
1535
-
1536
- async def _wait_for_turn_with_stop(
1537
- self,
1538
- client: Any,
1539
- handle: Any,
1540
- run_id: int,
1541
- *,
1542
- timeout: Optional[float],
1543
- external_stop_flag: Optional[threading.Event],
1544
- supervisor: Optional[WorkspaceAppServerSupervisor] = None,
1545
- ) -> tuple[Any, bool]:
1546
- stop_task = asyncio.create_task(self._wait_for_stop(external_stop_flag))
1547
- turn_task = asyncio.create_task(handle.wait(timeout=None))
1548
- timeout_task: Optional[asyncio.Task] = (
1549
- asyncio.create_task(asyncio.sleep(timeout)) if timeout else None
1550
- )
1551
- interrupted = False
1552
- try:
1553
- tasks = {stop_task, turn_task}
1554
- if timeout_task is not None:
1555
- tasks.add(timeout_task)
1556
- done, _pending = await asyncio.wait(
1557
- tasks, return_when=asyncio.FIRST_COMPLETED
1558
- )
1559
- if turn_task in done:
1560
- result = await turn_task
1561
- return result, interrupted
1562
- timed_out = timeout_task is not None and timeout_task in done
1563
- stopped = stop_task in done
1564
- if timed_out:
1565
- self.log_line(
1566
- run_id, "error: app-server turn timed out; interrupting app-server"
1567
- )
1568
- if stopped and not turn_task.done():
1569
- interrupted = True
1570
- self.log_line(run_id, "info: stop requested; interrupting app-server")
1571
- if not turn_task.done():
1572
- try:
1573
- await client.turn_interrupt(
1574
- handle.turn_id, thread_id=handle.thread_id
1575
- )
1576
- except CodexAppServerError as exc:
1577
- self.log_line(run_id, f"error: app-server interrupt failed: {exc}")
1578
- if interrupted:
1579
- self.kill_running_process()
1580
- raise
1581
- done, _pending = await asyncio.wait(
1582
- {turn_task}, timeout=AUTORUNNER_INTERRUPT_GRACE_SECONDS
1583
- )
1584
- if not done:
1585
- self.log_line(
1586
- run_id,
1587
- "error: app-server interrupt timed out; cleaning up",
1588
- )
1589
- if interrupted:
1590
- self.kill_running_process()
1591
- raise CodexAppServerError("App-server interrupt timed out")
1592
- if supervisor is not None:
1593
- await supervisor.close_all()
1594
- raise asyncio.TimeoutError()
1595
- result = await turn_task
1596
- if timed_out:
1597
- raise asyncio.TimeoutError()
1598
- return result, interrupted
1599
- finally:
1600
- stop_task.cancel()
1601
- with contextlib.suppress(asyncio.CancelledError):
1602
- await stop_task
1603
- if timeout_task is not None:
1604
- timeout_task.cancel()
1605
- with contextlib.suppress(asyncio.CancelledError):
1606
- await timeout_task
1607
-
1608
- async def _abort_opencode(self, client: Any, session_id: str, run_id: int) -> None:
1609
- try:
1610
- await client.abort(session_id)
1611
- except Exception as exc:
1612
- self.log_line(run_id, f"error: opencode abort failed: {exc}")
1613
-
1614
- async def _run_opencode_app_server_async(
1615
- self,
1616
- prompt: str,
1617
- run_id: int,
1618
- *,
1619
- model: Optional[str],
1620
- reasoning: Optional[str],
1621
- external_stop_flag: Optional[threading.Event] = None,
1622
- ) -> int:
1623
- supervisor = self._ensure_opencode_supervisor()
1624
- if supervisor is None:
1625
- self.log_line(
1626
- run_id, "error: opencode backend is not configured in this repo"
1627
- )
1628
- return 1
1629
- try:
1630
- client = await supervisor.get_client(self.repo_root)
1631
- except OpenCodeSupervisorError as exc:
1632
- self.log_line(run_id, f"error: opencode backend unavailable: {exc}")
1633
- return 1
1634
-
1635
- with self._app_server_threads_lock:
1636
- key = "autorunner.opencode"
1637
- thread_id = self._app_server_threads.get_thread_id(key)
1638
- if thread_id:
1639
- try:
1640
- await client.get_session(thread_id)
1641
- except Exception as exc:
1642
- self._app_server_logger.debug(
1643
- "Failed to get existing opencode session '%s' for run %s: %s",
1644
- thread_id,
1645
- run_id,
1646
- exc,
1647
- )
1648
- self._app_server_threads.reset_thread(key)
1649
- thread_id = None
1650
- if not thread_id:
1651
- session = await client.create_session(directory=str(self.repo_root))
1652
- thread_id = extract_session_id(session, allow_fallback_id=True)
1653
- if not isinstance(thread_id, str) or not thread_id:
1654
- self.log_line(run_id, "error: opencode did not return a session id")
1655
- return 1
1656
- self._app_server_threads.set_thread_id(key, thread_id)
1657
-
1658
- model_payload = split_model_id(model)
1659
- missing_env = await opencode_missing_env(
1660
- client, str(self.repo_root), model_payload
1661
- )
1662
- if missing_env:
1663
- provider_id = model_payload.get("providerID") if model_payload else None
1664
- self.log_line(
1665
- run_id,
1666
- "error: opencode provider "
1667
- f"{provider_id or 'selected'} requires env vars: "
1668
- f"{', '.join(missing_env)}",
1669
- )
1670
- return 1
1671
- opencode_turn_started = False
1672
- await supervisor.mark_turn_started(self.repo_root)
1673
- opencode_turn_started = True
1674
- turn_id = build_turn_id(thread_id)
1675
- self._update_run_telemetry(run_id, thread_id=thread_id, turn_id=turn_id)
1676
- app_server_meta = self._build_app_server_meta(
1677
- thread_id=thread_id,
1678
- turn_id=turn_id,
1679
- thread_info=None,
1680
- model=model,
1681
- reasoning_effort=reasoning,
1682
- )
1683
- app_server_meta["agent"] = "opencode"
1684
- self._merge_run_index_entry(run_id, {"app_server": app_server_meta})
1685
-
1686
- active = ActiveOpencodeRun(
1687
- session_id=thread_id,
1688
- turn_id=turn_id,
1689
- client=client,
1690
- interrupted=False,
1691
- interrupt_event=asyncio.Event(),
1692
- )
1693
- with state_lock(self.state_path):
1694
- state = load_state(self.state_path)
1695
- permission_policy = map_approval_policy_to_permission(
1696
- state.autorunner_approval_policy, default="allow"
1697
- )
1698
-
1699
- async def _opencode_part_handler(
1700
- part_type: str, part: dict[str, Any], delta_text: Optional[str]
1701
- ) -> None:
1702
- if part_type == "usage" and isinstance(part, dict):
1703
- for line in self._opencode_event_formatter.format_usage(part):
1704
- self.log_line(run_id, f"stdout: {line}" if line else "stdout: ")
1705
- else:
1706
- for line in self._opencode_event_formatter.format_part(
1707
- part_type, part, delta_text
1708
- ):
1709
- self.log_line(run_id, f"stdout: {line}" if line else "stdout: ")
1710
-
1711
- ready_event = asyncio.Event()
1712
- output_task = asyncio.create_task(
1713
- collect_opencode_output(
1714
- client,
1715
- session_id=thread_id,
1716
- workspace_path=str(self.repo_root),
1717
- model_payload=model_payload,
1718
- permission_policy=permission_policy,
1719
- question_policy="auto_first_option",
1720
- should_stop=active.interrupt_event.is_set,
1721
- part_handler=_opencode_part_handler,
1722
- ready_event=ready_event,
1723
- stall_timeout_seconds=self.config.opencode.session_stall_timeout_seconds,
1724
- )
1725
- )
1726
- with contextlib.suppress(asyncio.TimeoutError):
1727
- await asyncio.wait_for(ready_event.wait(), timeout=2.0)
1728
- prompt_task = asyncio.create_task(
1729
- client.prompt_async(
1730
- thread_id,
1731
- message=prompt,
1732
- model=model_payload,
1733
- variant=reasoning,
1734
- )
1735
- )
1736
- stop_task = asyncio.create_task(self._wait_for_stop(external_stop_flag))
1737
- timeout_task = None
1738
- turn_timeout = self.config.app_server.turn_timeout_seconds
1739
- if turn_timeout:
1740
- timeout_task = asyncio.create_task(asyncio.sleep(turn_timeout))
1741
- timed_out = False
1742
- try:
1743
- try:
1744
- prompt_response = await prompt_task
1745
- prompt_info = (
1746
- prompt_response.get("info")
1747
- if isinstance(prompt_response, dict)
1748
- else {}
1749
- )
1750
- tokens = (
1751
- prompt_info.get("tokens") if isinstance(prompt_info, dict) else {}
1752
- )
1753
- if isinstance(tokens, dict):
1754
- input_tokens = int(tokens.get("input", 0) or 0)
1755
- cached_read = (
1756
- int(tokens.get("cache", {}).get("read", 0) or 0)
1757
- if isinstance(tokens.get("cache"), dict)
1758
- else 0
1759
- )
1760
- output_tokens = int(tokens.get("output", 0) or 0)
1761
- reasoning_tokens = int(tokens.get("reasoning", 0) or 0)
1762
- total_tokens = (
1763
- input_tokens + cached_read + output_tokens + reasoning_tokens
1764
- )
1765
- token_total = {
1766
- "total": total_tokens,
1767
- "input_tokens": input_tokens,
1768
- "prompt_tokens": input_tokens,
1769
- "cached_input_tokens": cached_read,
1770
- "output_tokens": output_tokens,
1771
- "completion_tokens": output_tokens,
1772
- "reasoning_tokens": reasoning_tokens,
1773
- "reasoning_output_tokens": reasoning_tokens,
1774
- }
1775
- self._update_run_telemetry(run_id, token_total=token_total)
1776
- except Exception as exc:
1777
- active.interrupt_event.set()
1778
- output_task.cancel()
1779
- with contextlib.suppress(asyncio.CancelledError):
1780
- await output_task
1781
- self.log_line(run_id, f"error: opencode prompt failed: {exc}")
1782
- return 1
1783
- tasks = {output_task, stop_task}
1784
- if timeout_task is not None:
1785
- tasks.add(timeout_task)
1786
- done, _pending = await asyncio.wait(
1787
- tasks, return_when=asyncio.FIRST_COMPLETED
1788
- )
1789
- timed_out = timeout_task is not None and timeout_task in done
1790
- stopped = stop_task in done
1791
- if timed_out:
1792
- self.log_line(
1793
- run_id, "error: opencode turn timed out; aborting session"
1794
- )
1795
- active.interrupt_event.set()
1796
- if stopped:
1797
- active.interrupted = True
1798
- active.interrupt_event.set()
1799
- self.log_line(run_id, "info: stop requested; aborting opencode")
1800
- if timed_out or stopped:
1801
- await self._abort_opencode(client, thread_id, run_id)
1802
- done, _pending = await asyncio.wait(
1803
- {output_task}, timeout=AUTORUNNER_INTERRUPT_GRACE_SECONDS
1804
- )
1805
- if not done:
1806
- output_task.cancel()
1807
- with contextlib.suppress(asyncio.CancelledError):
1808
- await output_task
1809
- if timed_out:
1810
- return 1
1811
- self._last_run_interrupted = active.interrupted
1812
- return 0
1813
- output_result = await output_task
1814
- if not output_result.text and not output_result.error:
1815
- fallback = parse_message_response(prompt_response)
1816
- if fallback.text:
1817
- output_result = OpenCodeTurnOutput(
1818
- text=fallback.text, error=fallback.error
1819
- )
1820
- self.log_line(run_id, "info: opencode fallback message used")
1821
- finally:
1822
- # Flush buffered reasoning deltas before cleanup, so partial reasoning is still logged
1823
- # even when the turn is aborted, times out, or is interrupted.
1824
- for line in self._opencode_event_formatter.flush_all_reasoning():
1825
- self.log_line(run_id, f"stdout: {line}" if line else "stdout: ")
1826
- stop_task.cancel()
1827
- with contextlib.suppress(asyncio.CancelledError):
1828
- await stop_task
1829
- if timeout_task is not None:
1830
- timeout_task.cancel()
1831
- with contextlib.suppress(asyncio.CancelledError):
1832
- await timeout_task
1833
- if opencode_turn_started:
1834
- await supervisor.mark_turn_finished(self.repo_root)
1835
-
1836
- if not output_result.text:
1837
- self.log_line(
1838
- run_id,
1839
- "info: opencode returned empty output (error=%s)"
1840
- % (output_result.error or "none"),
1841
- )
1842
- if output_result.text:
1843
- handle_agent_output(
1844
- self._log_app_server_output,
1845
- self._write_run_artifact,
1846
- self._merge_run_index_entry,
1847
- run_id,
1848
- output_result.text,
1849
- )
1850
- if output_result.error:
1851
- self.log_line(
1852
- run_id, f"error: opencode session error: {output_result.error}"
1853
- )
1854
- return 1
1855
- self._last_run_interrupted = active.interrupted
1856
- if timed_out:
1857
- return 1
1858
- return 0
1859
-
1860
- async def _run_loop_async(
1861
- self,
1862
- stop_after_runs: Optional[int] = None,
1863
- external_stop_flag: Optional[threading.Event] = None,
1864
- ) -> None:
1865
- state = load_state(self.state_path)
1866
- run_id = (state.last_run_id or 0) + 1
1867
- last_exit_code: Optional[int] = state.last_exit_code
1868
- start_wallclock = time.time()
1869
- target_runs = (
1870
- stop_after_runs
1871
- if stop_after_runs is not None
1872
- else (
1873
- state.runner_stop_after_runs
1874
- if state.runner_stop_after_runs is not None
1875
- else self.config.runner_stop_after_runs
1876
- )
1877
- )
1878
- no_progress_count = 0
1879
- ticket_dir = self.repo_root / ".codex-autorunner" / "tickets"
1880
- initial_tickets = list_ticket_paths(ticket_dir)
1881
- last_done_count = sum(1 for path in initial_tickets if ticket_is_done(path))
1882
- last_outstanding_count = len(initial_tickets) - last_done_count
1883
- exit_reason: Optional[str] = None
1884
-
1885
- try:
1886
- while True:
1887
- if self._should_stop(external_stop_flag):
1888
- self.clear_stop_request()
1889
- self._update_state(
1890
- "idle", run_id - 1, last_exit_code, finished=True
1891
- )
1892
- exit_reason = "stop_requested"
1893
- break
1894
- if self.config.runner_max_wallclock_seconds is not None:
1895
- if (
1896
- time.time() - start_wallclock
1897
- > self.config.runner_max_wallclock_seconds
1898
- ):
1899
- self._update_state(
1900
- "idle", run_id - 1, state.last_exit_code, finished=True
1901
- )
1902
- exit_reason = "max_wallclock_seconds"
1903
- break
1904
-
1905
- if self.todos_done():
1906
- if not self.summary_finalized():
1907
- exit_code = await self._run_final_summary_job(
1908
- run_id, external_stop_flag=external_stop_flag
1909
- )
1910
- last_exit_code = exit_code
1911
- exit_reason = (
1912
- "error_exit" if exit_code != 0 else "todos_complete"
1913
- )
1914
- else:
1915
- current = load_state(self.state_path)
1916
- last_exit_code = current.last_exit_code
1917
- self._update_state(
1918
- "idle", run_id - 1, last_exit_code, finished=True
1919
- )
1920
- exit_reason = "todos_complete"
1921
- break
1922
-
1923
- prev_output = self.extract_prev_output(run_id - 1)
1924
- prompt = self._build_app_server_prompt(prev_output)
1925
-
1926
- exit_code = await self._execute_run_step(
1927
- prompt, run_id, external_stop_flag=external_stop_flag
1928
- )
1929
- last_exit_code = exit_code
1930
-
1931
- if exit_code != 0:
1932
- exit_reason = "error_exit"
1933
- break
1934
-
1935
- # Check for no progress across runs
1936
- current_tickets = list_ticket_paths(ticket_dir)
1937
- current_done_count = sum(
1938
- 1 for path in current_tickets if ticket_is_done(path)
1939
- )
1940
- current_outstanding_count = len(current_tickets) - current_done_count
1941
-
1942
- # Check if there was any meaningful progress
1943
- has_progress = (
1944
- current_outstanding_count != last_outstanding_count
1945
- or current_done_count != last_done_count
1946
- )
1947
-
1948
- # Check if there was any meaningful output (diff, plan, etc.)
1949
- has_output = False
1950
- run_entry = self._run_index_store.get_entry(run_id)
1951
- if run_entry:
1952
- artifacts = run_entry.get("artifacts", {})
1953
- if isinstance(artifacts, dict):
1954
- diff_path = artifacts.get("diff_path")
1955
- if diff_path:
1956
- try:
1957
- diff_content = (
1958
- Path(diff_path).read_text(encoding="utf-8").strip()
1959
- )
1960
- has_output = len(diff_content) > 0
1961
- except (OSError, IOError):
1962
- pass
1963
- if not has_output:
1964
- plan_path = artifacts.get("plan_path")
1965
- if plan_path:
1966
- try:
1967
- plan_content = (
1968
- Path(plan_path)
1969
- .read_text(encoding="utf-8")
1970
- .strip()
1971
- )
1972
- has_output = len(plan_content) > 0
1973
- except (OSError, IOError):
1974
- pass
1975
-
1976
- if not has_progress and not has_output:
1977
- no_progress_count += 1
1978
-
1979
- evidence = {
1980
- "outstanding_count": current_outstanding_count,
1981
- "done_count": current_done_count,
1982
- "has_diff": bool(
1983
- run_entry
1984
- and isinstance(run_entry.get("artifacts"), dict)
1985
- and run_entry["artifacts"].get("diff_path")
1986
- ),
1987
- "has_plan": bool(
1988
- run_entry
1989
- and isinstance(run_entry.get("artifacts"), dict)
1990
- and run_entry["artifacts"].get("plan_path")
1991
- ),
1992
- "run_id": run_id,
1993
- }
1994
- self._emit_event(
1995
- run_id, "run.no_progress", count=no_progress_count, **evidence
1996
- )
1997
- self.log_line(
1998
- run_id,
1999
- f"info: no progress detected ({no_progress_count}/{self.config.runner_no_progress_threshold} runs without progress)",
2000
- )
2001
- if no_progress_count >= self.config.runner_no_progress_threshold:
2002
- self.log_line(
2003
- run_id,
2004
- f"info: stopping after {no_progress_count} consecutive runs with no progress (threshold: {self.config.runner_no_progress_threshold})",
2005
- )
2006
- self._update_state(
2007
- "idle",
2008
- run_id,
2009
- exit_code,
2010
- finished=True,
2011
- )
2012
- exit_reason = "no_progress_threshold"
2013
- break
2014
- else:
2015
- no_progress_count = 0
2016
-
2017
- last_outstanding_count = current_outstanding_count
2018
- last_done_count = current_done_count
2019
-
2020
- # If TODO is now complete, run the final report job once and stop.
2021
- if self.todos_done() and not self.summary_finalized():
2022
- exit_code = await self._run_final_summary_job(
2023
- run_id + 1, external_stop_flag=external_stop_flag
2024
- )
2025
- last_exit_code = exit_code
2026
- exit_reason = "error_exit" if exit_code != 0 else "todos_complete"
2027
- break
2028
-
2029
- if target_runs is not None and run_id >= target_runs:
2030
- exit_reason = "stop_after_runs"
2031
- break
2032
-
2033
- run_id += 1
2034
- if self._should_stop(external_stop_flag):
2035
- self.clear_stop_request()
2036
- self._update_state("idle", run_id - 1, exit_code, finished=True)
2037
- exit_reason = "stop_requested"
2038
- break
2039
- await asyncio.sleep(self.config.runner_sleep_seconds)
2040
- except Exception as exc:
2041
- # Never silently die: persist's reason to agent log and surface in state.
2042
- exit_reason = exit_reason or "error_exit"
2043
- try:
2044
- self.log_line(run_id, f"FATAL: run_loop crashed: {exc!r}")
2045
- tb = traceback.format_exc()
2046
- for line in tb.splitlines():
2047
- self.log_line(run_id, f"traceback: {line}")
2048
- except (OSError, IOError) as exc:
2049
- self._app_server_logger.error(
2050
- "Failed to log run_loop crash for run %s: %s", run_id, exc
2051
- )
2052
- try:
2053
- self._update_state("error", run_id, 1, finished=True)
2054
- except (OSError, IOError) as exc:
2055
- self._app_server_logger.error(
2056
- "Failed to update state after run_loop crash for run %s: %s",
2057
- run_id,
2058
- exc,
2059
- )
2060
- finally:
2061
- try:
2062
- await self._maybe_run_end_review(
2063
- exit_reason=exit_reason or "unknown",
2064
- last_exit_code=last_exit_code,
2065
- )
2066
- except Exception as exc:
2067
- self._app_server_logger.warning(
2068
- "End-of-run review failed for run %s: %s", run_id, exc
2069
- )
2070
- await self._close_app_server_supervisor()
2071
- await self._close_opencode_supervisor()
2072
- # IMPORTANT: lock ownership is managed by the caller (CLI/Hub/Server runner).
2073
- # Engine.run_loop must never unconditionally mutate the lock file.
2074
-
2075
- async def _maybe_run_end_review(
2076
- self, *, exit_reason: str, last_exit_code: Optional[int]
2077
- ) -> None:
2078
- runner_cfg = self.config.raw.get("runner") or {}
2079
- review_cfg = runner_cfg.get("review")
2080
- if not isinstance(review_cfg, dict) or not review_cfg.get("enabled"):
2081
- return
2082
-
2083
- trigger_cfg = review_cfg.get("trigger") or {}
2084
- reason_key_map = {
2085
- "todos_complete": "on_todos_complete",
2086
- "no_progress_threshold": "on_no_progress_stop",
2087
- "stop_after_runs": "on_max_runs_stop",
2088
- # Share the max-runs trigger for wallclock cutoffs to avoid extra config flags.
2089
- "max_wallclock_seconds": "on_max_runs_stop",
2090
- "stop_requested": "on_stop_requested",
2091
- "error_exit": "on_error_exit",
2092
- }
2093
- trigger_key = reason_key_map.get(exit_reason)
2094
- if not trigger_key or not trigger_cfg.get(trigger_key, False):
2095
- return
2096
-
2097
- state = load_state(self.state_path)
2098
- last_run_id = state.last_run_id
2099
- if last_run_id is None:
2100
- return
2101
-
2102
- top_review_cfg = self.config.raw.get("review") or {}
2103
- agent = review_cfg.get("agent") or top_review_cfg.get("agent") or "opencode"
2104
- model = review_cfg.get("model") or top_review_cfg.get("model")
2105
- reasoning = review_cfg.get("reasoning") or top_review_cfg.get("reasoning")
2106
- max_wallclock_seconds = review_cfg.get("max_wallclock_seconds")
2107
- if max_wallclock_seconds is None:
2108
- max_wallclock_seconds = top_review_cfg.get("max_wallclock_seconds")
2109
-
2110
- context_cfg = review_cfg.get("context") or {}
2111
- primary_docs = context_cfg.get("primary_docs") or ["spec", "progress"]
2112
- include_docs = context_cfg.get("include_docs") or []
2113
- include_last_run_artifacts = bool(
2114
- context_cfg.get("include_last_run_artifacts", True)
2115
- )
2116
- max_doc_chars = context_cfg.get("max_doc_chars", 20000)
2117
- try:
2118
- max_doc_chars = int(max_doc_chars)
2119
- except (TypeError, ValueError):
2120
- max_doc_chars = 20000
2121
-
2122
- context_md = build_spec_progress_review_context(
2123
- self,
2124
- exit_reason=exit_reason,
2125
- last_run_id=last_run_id,
2126
- last_exit_code=last_exit_code,
2127
- max_doc_chars=max_doc_chars,
2128
- primary_docs=primary_docs,
2129
- include_docs=include_docs,
2130
- include_last_run_artifacts=include_last_run_artifacts,
2131
- )
2132
-
2133
- payload: dict[str, Any] = {
2134
- "agent": agent,
2135
- "model": model,
2136
- "reasoning": reasoning,
2137
- "max_wallclock_seconds": max_wallclock_seconds,
2138
- }
2139
- payload = {k: v for k, v in payload.items() if v is not None}
2140
-
2141
- opencode_supervisor: Optional[OpenCodeSupervisor] = None
2142
- app_server_supervisor: Optional[WorkspaceAppServerSupervisor] = None
2143
-
2144
- if agent == "codex":
2145
- if not self.config.app_server.command:
2146
- self._app_server_logger.info(
2147
- "Skipping end-of-run review: codex backend not configured"
2148
- )
2149
- return
2150
-
2151
- def _env_builder(
2152
- workspace_root: Path, _workspace_id: str, state_dir: Path
2153
- ) -> dict[str, str]:
2154
- state_dir.mkdir(parents=True, exist_ok=True)
2155
- return build_app_server_env(
2156
- self.config.app_server.command,
2157
- workspace_root,
2158
- state_dir,
2159
- logger=self._app_server_logger,
2160
- event_prefix="review",
2161
- )
2162
-
2163
- app_server_supervisor = self._ensure_app_server_supervisor(_env_builder)
2164
- else:
2165
- opencode_supervisor = self._ensure_opencode_supervisor()
2166
- if opencode_supervisor is None:
2167
- self._app_server_logger.info(
2168
- "Skipping end-of-run review: opencode backend not configured"
2169
- )
2170
- return
2171
-
2172
- from .review import ReviewService
2173
-
2174
- review_service = ReviewService(
2175
- self,
2176
- opencode_supervisor=opencode_supervisor,
2177
- app_server_supervisor=app_server_supervisor,
2178
- logger=self._app_server_logger,
2179
- )
2180
- result_state = await review_service.run_blocking_async(
2181
- payload=payload,
2182
- prompt_kind="spec_progress",
2183
- seed_context_files={"AUTORUNNER_CONTEXT.md": context_md},
2184
- ignore_repo_busy=True,
2185
- )
2186
-
2187
- review_id = result_state.get("id")
2188
- artifacts_cfg = review_cfg.get("artifacts") or {}
2189
- attach = bool(artifacts_cfg.get("attach_to_last_run_index", True))
2190
- if attach:
2191
- artifacts_update: dict[str, str] = {}
2192
- final_report = result_state.get("final_output_path")
2193
- scratch_bundle = result_state.get("scratchpad_bundle_path")
2194
- if isinstance(final_report, str) and final_report:
2195
- artifacts_update["final_review_report_path"] = final_report
2196
- if isinstance(scratch_bundle, str) and scratch_bundle:
2197
- artifacts_update["final_review_scratchpad_bundle_path"] = scratch_bundle
2198
- if artifacts_update:
2199
- self._merge_run_index_entry(
2200
- last_run_id,
2201
- {"artifacts": artifacts_update},
2202
- )
2203
- if review_id:
2204
- self.log_line(
2205
- last_run_id,
2206
- f"info: end-of-run review completed (review_id={review_id})",
2207
- )
2208
-
2209
- def run_loop(
2210
- self,
2211
- stop_after_runs: Optional[int] = None,
2212
- external_stop_flag: Optional[threading.Event] = None,
2213
- ) -> None:
2214
- try:
2215
- asyncio.run(self._run_loop_async(stop_after_runs, external_stop_flag))
2216
- except RuntimeError as exc:
2217
- if "asyncio.run" in str(exc):
2218
- raise
2219
- raise
2220
-
2221
- def run_once(self) -> None:
2222
- self.run_loop(stop_after_runs=1)
2223
-
2224
- def _update_state(
2225
- self,
2226
- status: str,
2227
- run_id: int,
2228
- exit_code: Optional[int],
2229
- *,
2230
- started: bool = False,
2231
- finished: bool = False,
2232
- ) -> None:
2233
- prev_status: Optional[str] = None
2234
- last_run_started_at: Optional[str] = None
2235
- last_run_finished_at: Optional[str] = None
2236
- with state_lock(self.state_path):
2237
- current = load_state(self.state_path)
2238
- prev_status = current.status
2239
- last_run_started_at = current.last_run_started_at
2240
- last_run_finished_at = current.last_run_finished_at
2241
- runner_pid = current.runner_pid
2242
- if started:
2243
- last_run_started_at = now_iso()
2244
- last_run_finished_at = None
2245
- runner_pid = os.getpid()
2246
- if finished:
2247
- last_run_finished_at = now_iso()
2248
- runner_pid = None
2249
- new_state = RunnerState(
2250
- last_run_id=run_id,
2251
- status=status,
2252
- last_exit_code=exit_code,
2253
- last_run_started_at=last_run_started_at,
2254
- last_run_finished_at=last_run_finished_at,
2255
- autorunner_agent_override=current.autorunner_agent_override,
2256
- autorunner_model_override=current.autorunner_model_override,
2257
- autorunner_effort_override=current.autorunner_effort_override,
2258
- autorunner_approval_policy=current.autorunner_approval_policy,
2259
- autorunner_sandbox_mode=current.autorunner_sandbox_mode,
2260
- autorunner_workspace_write_network=current.autorunner_workspace_write_network,
2261
- runner_pid=runner_pid,
2262
- sessions=current.sessions,
2263
- repo_to_session=current.repo_to_session,
2264
- )
2265
- save_state(self.state_path, new_state)
2266
- if run_id > 0 and prev_status != status:
2267
- payload: dict[str, Any] = {
2268
- "from_status": prev_status,
2269
- "to_status": status,
2270
- }
2271
- if exit_code is not None:
2272
- payload["exit_code"] = exit_code
2273
- if started and last_run_started_at:
2274
- payload["started_at"] = last_run_started_at
2275
- if finished and last_run_finished_at:
2276
- payload["finished_at"] = last_run_finished_at
2277
- self._emit_event(run_id, "run.state_changed", **payload)
2278
-
2279
-
2280
- def clear_stale_lock(lock_path: Path) -> bool:
2281
- assessment = assess_lock(
2282
- lock_path,
2283
- expected_cmd_substrings=DEFAULT_RUNNER_CMD_HINTS,
2284
- )
2285
- if assessment.freeable:
2286
- lock_path.unlink(missing_ok=True)
2287
- return True
2288
- return False
2289
-
2290
-
2291
- def _strip_log_prefixes(text: str) -> str:
2292
- """Strip log prefixes and clip to content after token-usage marker if present."""
2293
- lines = text.splitlines()
2294
- cleaned_lines = []
2295
- token_marker_idx = None
2296
- for idx, line in enumerate(lines):
2297
- if "stdout: tokens used" in line:
2298
- token_marker_idx = idx
2299
- break
2300
- if token_marker_idx is not None:
2301
- lines = lines[token_marker_idx + 1 :]
2302
-
2303
- for line in lines:
2304
- if "] run=" in line and "stdout:" in line:
2305
- try:
2306
- _, remainder = line.split("stdout:", 1)
2307
- cleaned_lines.append(remainder.strip())
2308
- continue
2309
- except ValueError:
2310
- pass
2311
- cleaned_lines.append(line)
2312
- return "\n".join(cleaned_lines).strip()
2313
-
2314
-
2315
- def _read_tail_text(path: Path, *, max_bytes: int) -> str:
2316
- """
2317
- Read at most last `max_bytes` bytes from a UTF-8-ish text file.
2318
- Returns decoded text with errors replaced.
2319
- """
2320
- logger = logging.getLogger("codex_autorunner.engine")
2321
- try:
2322
- size = path.stat().st_size
2323
- except OSError as exc:
2324
- logger.debug("Failed to stat log file for tail read: %s", exc)
2325
- return ""
2326
- if size <= 0:
2327
- return ""
2328
- try:
2329
- with path.open("rb") as f:
2330
- if size > max_bytes:
2331
- f.seek(-max_bytes, os.SEEK_END)
2332
- data = f.read()
2333
- return data.decode("utf-8", errors="replace")
2334
- except (FileNotFoundError, OSError, IOError) as exc:
2335
- logger.debug("Failed to read tail of log file: %s", exc)
2336
- return ""
2337
- if size <= 0:
2338
- return ""
2339
- try:
2340
- with path.open("rb") as f:
2341
- if size > max_bytes:
2342
- f.seek(-max_bytes, os.SEEK_END)
2343
- data = f.read()
2344
- return data.decode("utf-8", errors="replace")
2345
- except Exception:
2346
- return ""
2347
-
2348
-
2349
- @dataclasses.dataclass(frozen=True)
2350
- class DoctorCheck:
2351
- check_id: str
2352
- status: str
2353
- message: str
2354
- fix: Optional[str] = None
2355
-
2356
- def to_dict(self) -> dict:
2357
- payload = {
2358
- "id": self.check_id,
2359
- "status": self.status,
2360
- "message": self.message,
2361
- }
2362
- if self.fix:
2363
- payload["fix"] = self.fix
2364
- return payload
2365
-
2366
-
2367
- @dataclasses.dataclass(frozen=True)
2368
- class DoctorReport:
2369
- checks: list[DoctorCheck]
2370
-
2371
- def has_errors(self) -> bool:
2372
- return any(check.status == "error" for check in self.checks)
2373
-
2374
- def to_dict(self) -> dict:
2375
- return {
2376
- "ok": sum(1 for check in self.checks if check.status == "ok"),
2377
- "warnings": sum(1 for check in self.checks if check.status == "warning"),
2378
- "errors": sum(1 for check in self.checks if check.status == "error"),
2379
- "checks": [check.to_dict() for check in self.checks],
2380
- }
2381
-
2382
-
2383
- def _append_check(
2384
- checks: list[DoctorCheck],
2385
- check_id: str,
2386
- status: str,
2387
- message: str,
2388
- fix: Optional[str] = None,
2389
- ) -> None:
2390
- checks.append(
2391
- DoctorCheck(check_id=check_id, status=status, message=message, fix=fix)
2392
- )
2393
-
2394
-
2395
- def _parse_manifest_version(manifest_path: Path) -> Optional[int]:
2396
- logger = logging.getLogger("codex_autorunner.engine")
2397
- try:
2398
- raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
2399
- except (FileNotFoundError, OSError, yaml.YAMLError) as exc:
2400
- logger.debug("Failed to parse manifest version: %s", exc)
2401
- return None
2402
- if not isinstance(raw, dict):
2403
- return None
2404
- version = raw.get("version")
2405
- return int(version) if isinstance(version, int) else None
2406
-
2407
-
2408
- def _manifest_has_worktrees(manifest_path: Path) -> bool:
2409
- logger = logging.getLogger("codex_autorunner.engine")
2410
- try:
2411
- raw = yaml.safe_load(manifest_path.read_text(encoding="utf-8")) or {}
2412
- except (FileNotFoundError, OSError, yaml.YAMLError) as exc:
2413
- logger.debug("Failed to parse manifest for worktrees: %s", exc)
2414
- return False
2415
- if not isinstance(raw, dict):
2416
- return False
2417
- repos = raw.get("repos")
2418
- if not isinstance(repos, list):
2419
- return False
2420
- for entry in repos:
2421
- if isinstance(entry, dict) and entry.get("kind") == "worktree":
2422
- return True
2423
- return False
2424
-
2425
-
2426
- def doctor(start_path: Path) -> DoctorReport:
2427
- hub_config = load_hub_config(start_path)
2428
- repo_config: Optional[RepoConfig] = None
2429
- try:
2430
- repo_root = find_repo_root(start_path)
2431
- repo_config = derive_repo_config(hub_config, repo_root)
2432
- except RepoNotFoundError:
2433
- repo_config = None
2434
- checks: list[DoctorCheck] = []
2435
- config = repo_config or hub_config
2436
- root = config.root
2437
-
2438
- if repo_config is not None:
2439
- missing = []
2440
- for key in ("todo", "progress", "opinions"):
2441
- path = repo_config.doc_path(key)
2442
- if not path.exists():
2443
- missing.append(path)
2444
- if missing:
2445
- names = ", ".join(str(p) for p in missing)
2446
- _append_check(
2447
- checks,
2448
- "docs.required",
2449
- "error",
2450
- f"Missing doc files: {names}",
2451
- "Run `car init` or create the missing files.",
2452
- )
2453
- else:
2454
- _append_check(
2455
- checks,
2456
- "docs.required",
2457
- "ok",
2458
- "Required doc files are present.",
2459
- )
2460
-
2461
- if ensure_executable(repo_config.codex_binary):
2462
- _append_check(
2463
- checks,
2464
- "codex.binary",
2465
- "ok",
2466
- f"Codex binary resolved: {repo_config.codex_binary}",
2467
- )
2468
- else:
2469
- _append_check(
2470
- checks,
2471
- "codex.binary",
2472
- "error",
2473
- f"Codex binary not found in PATH: {repo_config.codex_binary}",
2474
- "Install Codex or set codex.binary to a full path.",
2475
- )
2476
-
2477
- voice_enabled = bool(repo_config.voice.get("enabled", True))
2478
- if voice_enabled:
2479
- missing_voice = missing_optional_dependencies(
2480
- (
2481
- ("httpx", "httpx"),
2482
- (("multipart", "python_multipart"), "python-multipart"),
2483
- )
2484
- )
2485
- if missing_voice:
2486
- deps_list = ", ".join(missing_voice)
2487
- _append_check(
2488
- checks,
2489
- "voice.dependencies",
2490
- "error",
2491
- f"Voice is enabled but missing optional deps: {deps_list}",
2492
- "Install with `pip install codex-autorunner[voice]`.",
2493
- )
2494
- else:
2495
- _append_check(
2496
- checks,
2497
- "voice.dependencies",
2498
- "ok",
2499
- "Voice dependencies are installed.",
2500
- )
2501
-
2502
- env_candidates = [
2503
- root / ".env",
2504
- root / ".codex-autorunner" / ".env",
2505
- ]
2506
- env_found = [str(path) for path in env_candidates if path.exists()]
2507
- if env_found:
2508
- _append_check(
2509
- checks,
2510
- "dotenv.locations",
2511
- "ok",
2512
- f"Found .env files: {', '.join(env_found)}",
2513
- )
2514
- else:
2515
- _append_check(
2516
- checks,
2517
- "dotenv.locations",
2518
- "warning",
2519
- "No .env files found in repo root or .codex-autorunner/.env.",
2520
- "Create one of these files if you rely on env vars.",
2521
- )
2522
-
2523
- host = str(config.server_host or "")
2524
- if not _is_loopback_host(host):
2525
- if not config.server_auth_token_env:
2526
- _append_check(
2527
- checks,
2528
- "server.auth",
2529
- "error",
2530
- f"Non-loopback host {host} requires server.auth_token_env.",
2531
- "Set server.auth_token_env or bind to 127.0.0.1.",
2532
- )
2533
- else:
2534
- token_val = os.environ.get(config.server_auth_token_env)
2535
- if not token_val:
2536
- _append_check(
2537
- checks,
2538
- "server.auth",
2539
- "warning",
2540
- f"Auth token env var {config.server_auth_token_env} is not set.",
2541
- "Export the env var or add it to .env.",
2542
- )
2543
- else:
2544
- _append_check(
2545
- checks,
2546
- "server.auth",
2547
- "ok",
2548
- "Server auth token env var is set for non-loopback host.",
2549
- )
2550
-
2551
- static_dir, static_context = resolve_static_dir()
2552
- try:
2553
- missing_assets = missing_static_assets(static_dir)
2554
- if missing_assets:
2555
- _append_check(
2556
- checks,
2557
- "static.assets",
2558
- "error",
2559
- f"Static UI assets missing in {static_dir}: {', '.join(missing_assets)}",
2560
- "Reinstall the package or rebuild the UI assets.",
2561
- )
2562
- else:
2563
- _append_check(
2564
- checks,
2565
- "static.assets",
2566
- "ok",
2567
- f"Static UI assets present in {static_dir}",
2568
- )
2569
- finally:
2570
- if static_context is not None:
2571
- static_context.close()
2572
-
2573
- if hub_config.manifest_path.exists():
2574
- version = _parse_manifest_version(hub_config.manifest_path)
2575
- if version is None:
2576
- _append_check(
2577
- checks,
2578
- "hub.manifest.version",
2579
- "error",
2580
- f"Failed to read manifest version from {hub_config.manifest_path}.",
2581
- "Fix the manifest YAML or regenerate it with `car hub scan`.",
2582
- )
2583
- elif version != MANIFEST_VERSION:
2584
- _append_check(
2585
- checks,
2586
- "hub.manifest.version",
2587
- "error",
2588
- f"Hub manifest version {version} unsupported (expected {MANIFEST_VERSION}).",
2589
- "Regenerate the manifest (delete it and run `car hub scan`).",
2590
- )
2591
- else:
2592
- _append_check(
2593
- checks,
2594
- "hub.manifest.version",
2595
- "ok",
2596
- f"Hub manifest version {version} is supported.",
2597
- )
2598
- else:
2599
- _append_check(
2600
- checks,
2601
- "hub.manifest.exists",
2602
- "warning",
2603
- f"Hub manifest missing at {hub_config.manifest_path}.",
2604
- "Run `car hub scan` or `car hub create` to generate it.",
2605
- )
2606
-
2607
- if not hub_config.repos_root.exists():
2608
- _append_check(
2609
- checks,
2610
- "hub.repos_root",
2611
- "error",
2612
- f"Hub repos_root does not exist: {hub_config.repos_root}",
2613
- "Create the directory or update hub.repos_root in config.",
2614
- )
2615
- elif not hub_config.repos_root.is_dir():
2616
- _append_check(
2617
- checks,
2618
- "hub.repos_root",
2619
- "error",
2620
- f"Hub repos_root is not a directory: {hub_config.repos_root}",
2621
- "Point hub.repos_root at a directory.",
2622
- )
2623
- else:
2624
- _append_check(
2625
- checks,
2626
- "hub.repos_root",
2627
- "ok",
2628
- f"Hub repos_root exists: {hub_config.repos_root}",
2629
- )
2630
-
2631
- manifest_has_worktrees = (
2632
- hub_config.manifest_path.exists()
2633
- and _manifest_has_worktrees(hub_config.manifest_path)
2634
- )
2635
- worktrees_enabled = hub_config.worktrees_root.exists() or manifest_has_worktrees
2636
- if worktrees_enabled:
2637
- if ensure_executable("git"):
2638
- _append_check(
2639
- checks,
2640
- "hub.git",
2641
- "ok",
2642
- "git is available for hub worktrees.",
2643
- )
2644
- else:
2645
- _append_check(
2646
- checks,
2647
- "hub.git",
2648
- "error",
2649
- "git is not available but hub worktrees are enabled.",
2650
- "Install git or disable worktrees.",
2651
- )
2652
-
2653
- return DoctorReport(checks=checks)