codeframe-ai 0.9.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (197) hide show
  1. codeframe/__init__.py +11 -0
  2. codeframe/__main__.py +20 -0
  3. codeframe/adapters/__init__.py +5 -0
  4. codeframe/adapters/e2b/__init__.py +13 -0
  5. codeframe/adapters/e2b/adapter.py +342 -0
  6. codeframe/adapters/e2b/budget.py +71 -0
  7. codeframe/adapters/e2b/credential_scanner.py +134 -0
  8. codeframe/adapters/llm/__init__.py +92 -0
  9. codeframe/adapters/llm/anthropic.py +414 -0
  10. codeframe/adapters/llm/base.py +444 -0
  11. codeframe/adapters/llm/mock.py +281 -0
  12. codeframe/adapters/llm/openai.py +483 -0
  13. codeframe/agents/__init__.py +8 -0
  14. codeframe/agents/dependency_resolver.py +714 -0
  15. codeframe/auth/__init__.py +16 -0
  16. codeframe/auth/api_key_router.py +238 -0
  17. codeframe/auth/api_keys.py +156 -0
  18. codeframe/auth/dependencies.py +358 -0
  19. codeframe/auth/manager.py +178 -0
  20. codeframe/auth/models.py +30 -0
  21. codeframe/auth/router.py +93 -0
  22. codeframe/auth/schemas.py +15 -0
  23. codeframe/auth/scopes.py +53 -0
  24. codeframe/cli/__init__.py +12 -0
  25. codeframe/cli/__main__.py +20 -0
  26. codeframe/cli/api_client.py +275 -0
  27. codeframe/cli/app.py +5688 -0
  28. codeframe/cli/auth.py +122 -0
  29. codeframe/cli/auth_commands.py +958 -0
  30. codeframe/cli/commands/__init__.py +5 -0
  31. codeframe/cli/config_commands.py +79 -0
  32. codeframe/cli/dashboard_commands.py +67 -0
  33. codeframe/cli/engines_commands.py +205 -0
  34. codeframe/cli/env_commands.py +409 -0
  35. codeframe/cli/helpers.py +56 -0
  36. codeframe/cli/hooks_commands.py +208 -0
  37. codeframe/cli/import_commands.py +129 -0
  38. codeframe/cli/pr_commands.py +549 -0
  39. codeframe/cli/proof_commands.py +415 -0
  40. codeframe/cli/stats_commands.py +311 -0
  41. codeframe/cli/telemetry_runtime.py +153 -0
  42. codeframe/cli/validators.py +123 -0
  43. codeframe/config/rate_limits.py +165 -0
  44. codeframe/core/__init__.py +15 -0
  45. codeframe/core/adapters/__init__.py +43 -0
  46. codeframe/core/adapters/agent_adapter.py +114 -0
  47. codeframe/core/adapters/builtin.py +326 -0
  48. codeframe/core/adapters/claude_code.py +62 -0
  49. codeframe/core/adapters/codex.py +393 -0
  50. codeframe/core/adapters/git_utils.py +40 -0
  51. codeframe/core/adapters/kilocode.py +126 -0
  52. codeframe/core/adapters/opencode.py +48 -0
  53. codeframe/core/adapters/streaming_chat.py +483 -0
  54. codeframe/core/adapters/subprocess_adapter.py +213 -0
  55. codeframe/core/adapters/verification_wrapper.py +269 -0
  56. codeframe/core/agent.py +2183 -0
  57. codeframe/core/agents_config.py +569 -0
  58. codeframe/core/api_key_service.py +211 -0
  59. codeframe/core/artifacts.py +428 -0
  60. codeframe/core/blocker_detection.py +218 -0
  61. codeframe/core/blockers.py +433 -0
  62. codeframe/core/checkpoints.py +481 -0
  63. codeframe/core/conductor.py +2255 -0
  64. codeframe/core/config.py +827 -0
  65. codeframe/core/config_watcher.py +268 -0
  66. codeframe/core/context.py +542 -0
  67. codeframe/core/context_packager.py +234 -0
  68. codeframe/core/credentials.py +735 -0
  69. codeframe/core/dependency_analyzer.py +229 -0
  70. codeframe/core/dependency_graph.py +290 -0
  71. codeframe/core/diagnostic_agent.py +712 -0
  72. codeframe/core/diagnostics.py +616 -0
  73. codeframe/core/editor.py +556 -0
  74. codeframe/core/engine_registry.py +256 -0
  75. codeframe/core/engine_stats.py +231 -0
  76. codeframe/core/environment.py +697 -0
  77. codeframe/core/events.py +375 -0
  78. codeframe/core/executor.py +1005 -0
  79. codeframe/core/fix_tracker.py +480 -0
  80. codeframe/core/gates.py +1322 -0
  81. codeframe/core/git.py +477 -0
  82. codeframe/core/github_connect_service.py +178 -0
  83. codeframe/core/github_integration_config.py +118 -0
  84. codeframe/core/github_issues_service.py +449 -0
  85. codeframe/core/hooks.py +184 -0
  86. codeframe/core/importers/__init__.py +1 -0
  87. codeframe/core/importers/ralph.py +540 -0
  88. codeframe/core/installer.py +650 -0
  89. codeframe/core/models.py +1026 -0
  90. codeframe/core/notifications_config.py +183 -0
  91. codeframe/core/planner.py +437 -0
  92. codeframe/core/prd.py +670 -0
  93. codeframe/core/prd_discovery.py +1118 -0
  94. codeframe/core/prd_stress_test.py +499 -0
  95. codeframe/core/progress.py +126 -0
  96. codeframe/core/proof/__init__.py +34 -0
  97. codeframe/core/proof/capture.py +79 -0
  98. codeframe/core/proof/evidence.py +56 -0
  99. codeframe/core/proof/ledger.py +574 -0
  100. codeframe/core/proof/models.py +162 -0
  101. codeframe/core/proof/obligations.py +103 -0
  102. codeframe/core/proof/runner.py +233 -0
  103. codeframe/core/proof/scope.py +81 -0
  104. codeframe/core/proof/stubs.py +156 -0
  105. codeframe/core/quick_fixes.py +558 -0
  106. codeframe/core/react_agent.py +1650 -0
  107. codeframe/core/reconciliation.py +183 -0
  108. codeframe/core/replay.py +788 -0
  109. codeframe/core/review.py +285 -0
  110. codeframe/core/runtime.py +1134 -0
  111. codeframe/core/sandbox/__init__.py +27 -0
  112. codeframe/core/sandbox/context.py +98 -0
  113. codeframe/core/sandbox/worktree.py +20 -0
  114. codeframe/core/schedule.py +396 -0
  115. codeframe/core/stall_detector.py +71 -0
  116. codeframe/core/stall_monitor.py +134 -0
  117. codeframe/core/state_machine.py +121 -0
  118. codeframe/core/streaming.py +502 -0
  119. codeframe/core/task_tree.py +400 -0
  120. codeframe/core/tasks.py +1022 -0
  121. codeframe/core/telemetry.py +232 -0
  122. codeframe/core/templates.py +221 -0
  123. codeframe/core/tools.py +942 -0
  124. codeframe/core/workspace.py +887 -0
  125. codeframe/core/worktrees.py +276 -0
  126. codeframe/git/__init__.py +5 -0
  127. codeframe/git/github_integration.py +505 -0
  128. codeframe/lib/__init__.py +0 -0
  129. codeframe/lib/audit_logger.py +248 -0
  130. codeframe/lib/metrics_tracker.py +800 -0
  131. codeframe/lib/quality/__init__.py +7 -0
  132. codeframe/lib/quality/complexity_analyzer.py +316 -0
  133. codeframe/lib/quality/owasp_patterns.py +284 -0
  134. codeframe/lib/quality/security_scanner.py +250 -0
  135. codeframe/lib/rate_limiter.py +312 -0
  136. codeframe/notifications/__init__.py +0 -0
  137. codeframe/notifications/webhook.py +380 -0
  138. codeframe/planning/__init__.py +30 -0
  139. codeframe/planning/issue_generator.py +219 -0
  140. codeframe/planning/prd_template_functions.py +137 -0
  141. codeframe/planning/prd_templates.py +975 -0
  142. codeframe/planning/task_scheduler.py +511 -0
  143. codeframe/planning/task_templates.py +533 -0
  144. codeframe/platform_store/__init__.py +5 -0
  145. codeframe/platform_store/database.py +277 -0
  146. codeframe/platform_store/repositories/__init__.py +24 -0
  147. codeframe/platform_store/repositories/api_key_repository.py +245 -0
  148. codeframe/platform_store/repositories/audit_repository.py +67 -0
  149. codeframe/platform_store/repositories/base.py +295 -0
  150. codeframe/platform_store/repositories/interactive_sessions.py +165 -0
  151. codeframe/platform_store/repositories/token_repository.py +598 -0
  152. codeframe/platform_store/repositories/workspace_registry_repository.py +175 -0
  153. codeframe/platform_store/schema_manager.py +321 -0
  154. codeframe/templates/AGENTS.md.default +94 -0
  155. codeframe/tui/__init__.py +5 -0
  156. codeframe/tui/app.py +256 -0
  157. codeframe/tui/data_service.py +103 -0
  158. codeframe/ui/__init__.py +0 -0
  159. codeframe/ui/dependencies.py +103 -0
  160. codeframe/ui/models.py +999 -0
  161. codeframe/ui/response_models.py +201 -0
  162. codeframe/ui/routers/__init__.py +5 -0
  163. codeframe/ui/routers/_helpers.py +29 -0
  164. codeframe/ui/routers/batches_v2.py +315 -0
  165. codeframe/ui/routers/blockers_v2.py +320 -0
  166. codeframe/ui/routers/checkpoints_v2.py +310 -0
  167. codeframe/ui/routers/costs_v2.py +322 -0
  168. codeframe/ui/routers/diagnose_v2.py +225 -0
  169. codeframe/ui/routers/discovery_v2.py +417 -0
  170. codeframe/ui/routers/environment_v2.py +284 -0
  171. codeframe/ui/routers/events_v2.py +75 -0
  172. codeframe/ui/routers/gates_v2.py +166 -0
  173. codeframe/ui/routers/git_v2.py +284 -0
  174. codeframe/ui/routers/github_integrations_v2.py +532 -0
  175. codeframe/ui/routers/interactive_sessions_v2.py +238 -0
  176. codeframe/ui/routers/pr_v2.py +709 -0
  177. codeframe/ui/routers/prd_v2.py +695 -0
  178. codeframe/ui/routers/proof_v2.py +755 -0
  179. codeframe/ui/routers/review_v2.py +360 -0
  180. codeframe/ui/routers/schedule_v2.py +214 -0
  181. codeframe/ui/routers/session_chat_ws.py +354 -0
  182. codeframe/ui/routers/settings_v2.py +562 -0
  183. codeframe/ui/routers/streaming_v2.py +155 -0
  184. codeframe/ui/routers/tasks_v2.py +1098 -0
  185. codeframe/ui/routers/templates_v2.py +232 -0
  186. codeframe/ui/routers/terminal_ws.py +267 -0
  187. codeframe/ui/routers/workspace_v2.py +527 -0
  188. codeframe/ui/server.py +568 -0
  189. codeframe/ui/shared.py +241 -0
  190. codeframe/workspace/__init__.py +5 -0
  191. codeframe/workspace/manager.py +249 -0
  192. codeframe_ai-0.9.0.dist-info/METADATA +517 -0
  193. codeframe_ai-0.9.0.dist-info/RECORD +197 -0
  194. codeframe_ai-0.9.0.dist-info/WHEEL +5 -0
  195. codeframe_ai-0.9.0.dist-info/entry_points.txt +3 -0
  196. codeframe_ai-0.9.0.dist-info/licenses/LICENSE +661 -0
  197. codeframe_ai-0.9.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,2255 @@
1
+ """Batch execution conductor for CodeFRAME v2.
2
+
3
+ Orchestrates execution of multiple tasks, managing parallelization
4
+ and coordinating results.
5
+
6
+ This module is headless - no FastAPI or HTTP dependencies.
7
+ """
8
+
9
+ import hashlib
10
+ import json
11
+ import logging
12
+ import re
13
+ import subprocess
14
+ import sys
15
+ import threading
16
+ import uuid
17
+ from concurrent.futures import ThreadPoolExecutor, as_completed
18
+ from dataclasses import dataclass, field
19
+ from datetime import datetime, timezone
20
+ from pathlib import Path
21
+ from enum import Enum
22
+ from typing import TYPE_CHECKING, Callable, Optional
23
+
24
+ if TYPE_CHECKING:
25
+ from codeframe.core.config_watcher import ConfigReloadState
26
+
27
+ from codeframe.core.workspace import Workspace, get_db_connection
28
+ from codeframe.core import events, tasks, blockers
29
+ from codeframe.core.dependency_graph import create_execution_plan, CycleDetectedError
30
+ from codeframe.core.dependency_analyzer import analyze_dependencies, apply_inferred_dependencies
31
+ from codeframe.core.runtime import RunStatus, get_active_run, reset_blocked_run
32
+
33
+
34
+ logger = logging.getLogger(__name__)
35
+
36
+
37
+ def _dispatch_batch_completed_webhook(
38
+ workspace: Workspace,
39
+ event_type: "events.EventType",
40
+ batch_id: str,
41
+ task_count: int,
42
+ ) -> None:
43
+ """Best-effort outbound webhook for ``batch.completed`` (issue #560).
44
+
45
+ Only fires when the batch reached the full COMPLETED state — PARTIAL,
46
+ FAILED, and CANCELLED are intentionally out of scope for v1. Failures
47
+ never break the batch.
48
+ """
49
+ if event_type != events.EventType.BATCH_COMPLETED:
50
+ return
51
+ try:
52
+ from codeframe.core.notifications_config import is_webhook_active
53
+ from codeframe.notifications.webhook import (
54
+ WebhookNotificationService,
55
+ format_batch_payload,
56
+ )
57
+
58
+ url = is_webhook_active(workspace)
59
+ if url is None:
60
+ return
61
+ svc = WebhookNotificationService(webhook_url=url, timeout=5)
62
+ svc.send_event_background(format_batch_payload(batch_id, task_count))
63
+ except Exception:
64
+ logger.warning(
65
+ "Failed to dispatch batch.completed webhook for %s",
66
+ batch_id,
67
+ exc_info=True,
68
+ )
69
+
70
+
71
+ # Tactical patterns that the supervisor should auto-resolve
72
+ SUPERVISOR_TACTICAL_PATTERNS = [
73
+ # Virtual environment / package management
74
+ "virtual environment", "venv", "virtualenv",
75
+ "pip install", "npm install", "uv sync",
76
+ "break-system-packages", "pipx",
77
+ "package manager", "dependency installation",
78
+ "externally-managed", # PEP 668 error - use venv or uv
79
+ "externally managed",
80
+ # Module/import issues
81
+ "no module named", "__main__", "cannot be directly executed",
82
+ "modulenotfounderror", "importerror",
83
+ # Configuration
84
+ "pytest.ini", "pyproject.toml", "asyncio_default_fixture_loop_scope",
85
+ "fixture scope", "loop scope", "configuration file",
86
+ # Common tactical questions
87
+ "would you like me to", "would you prefer",
88
+ "should i create", "should i use",
89
+ "which approach", "which version",
90
+ "overwrite", "existing file",
91
+ ]
92
+
93
+ # Cache of resolved decisions to avoid duplicate LLM calls
94
+ _decision_cache: dict[str, str] = {}
95
+
96
+ # Track running subprocesses for force stop capability
97
+ # Structure: {batch_id: {task_id: Popen}}
98
+ _active_processes: dict[str, dict[str, subprocess.Popen]] = {}
99
+ # Lock for thread-safe access to _active_processes
100
+ _active_processes_lock = threading.Lock()
101
+ # Lock for thread-safe batch database writes
102
+ _batch_db_lock = threading.Lock()
103
+
104
+
105
+ class SupervisorResolver:
106
+ """Resolves tactical blockers at the conductor level.
107
+
108
+ Instead of each worker agent independently creating blockers,
109
+ the conductor uses this resolver to:
110
+ 1. Evaluate blockers with the supervision model (stronger)
111
+ 2. Auto-resolve tactical decisions
112
+ 3. Deduplicate similar questions across workers
113
+ 4. Only surface true human-required decisions
114
+ """
115
+
116
+ def __init__(self, workspace: Workspace):
117
+ self.workspace = workspace
118
+ self._llm = None # Lazy initialization
119
+
120
+ @property
121
+ def llm(self):
122
+ """Lazy-load LLM provider."""
123
+ if self._llm is None:
124
+ from codeframe.adapters.llm import get_provider
125
+ self._llm = get_provider()
126
+ return self._llm
127
+
128
+ def try_resolve_blocked_task(self, task_id: str) -> bool:
129
+ """Try to resolve a blocked task's blocker autonomously.
130
+
131
+ Args:
132
+ task_id: The blocked task ID
133
+
134
+ Returns:
135
+ True if blocker was resolved (task should retry),
136
+ False if blocker requires human input
137
+ """
138
+ # Get the task's open blockers
139
+ task_blockers = blockers.list_all(
140
+ self.workspace,
141
+ task_id=task_id,
142
+ status=blockers.BlockerStatus.OPEN,
143
+ )
144
+
145
+ if not task_blockers:
146
+ return False
147
+
148
+ blocker = task_blockers[0] # Most recent open blocker
149
+ question = blocker.question.lower()
150
+
151
+ # Check cache first
152
+ cache_key = self._get_cache_key(question)
153
+ if cache_key in _decision_cache:
154
+ print(" [Supervisor] Using cached decision for similar question")
155
+ self._auto_answer_blocker(blocker, _decision_cache[cache_key])
156
+ return True
157
+
158
+ # Check if question matches tactical patterns
159
+ if self._is_tactical_question(question):
160
+ print(" [Supervisor] Detected tactical question, auto-resolving")
161
+ resolution = self._generate_tactical_resolution(blocker.question)
162
+ _decision_cache[cache_key] = resolution
163
+ self._auto_answer_blocker(blocker, resolution)
164
+ return True
165
+
166
+ # Use supervision model to classify if uncertain
167
+ classification = self._classify_with_supervision(blocker.question)
168
+
169
+ if classification == "tactical":
170
+ print(" [Supervisor] Model classified as tactical, auto-resolving")
171
+ resolution = self._generate_tactical_resolution(blocker.question)
172
+ _decision_cache[cache_key] = resolution
173
+ self._auto_answer_blocker(blocker, resolution)
174
+ return True
175
+
176
+ # This is a genuine human-required decision
177
+ print(" [Supervisor] Question requires human input")
178
+ return False
179
+
180
+ def _is_tactical_question(self, question: str) -> bool:
181
+ """Check if question matches known tactical patterns."""
182
+ return any(pattern in question for pattern in SUPERVISOR_TACTICAL_PATTERNS)
183
+
184
+ def _get_cache_key(self, question: str) -> str:
185
+ """Generate a cache key for deduplication.
186
+
187
+ Normalizes similar questions to the same key.
188
+ """
189
+ # Simple normalization - could be improved with embeddings
190
+ q = question.lower()
191
+ if "virtual environment" in q or "venv" in q or "virtualenv" in q:
192
+ return "venv_creation"
193
+ if "fixture scope" in q or "asyncio" in q:
194
+ return "asyncio_fixture_scope"
195
+ if "package manager" in q or "pip" in q or "npm" in q:
196
+ return "package_manager"
197
+ if "pytest" in q and ("fail" in q or "verification" in q):
198
+ return "pytest_failure"
199
+ # Fallback to hash of first 50 chars
200
+ return f"blocker_{hash(q[:50])}"
201
+
202
+ def _classify_with_supervision(self, question: str) -> str:
203
+ """Use supervision model to classify the blocker question."""
204
+ from codeframe.adapters.llm import Purpose
205
+
206
+ prompt = f"""Classify this blocker question from a coding agent:
207
+
208
+ Question: {question}
209
+
210
+ Is this:
211
+ 1. TACTICAL - A decision the agent should make autonomously (venv, package manager, config options, test framework, file handling)
212
+ 2. HUMAN - Genuinely requires human input (conflicting requirements, missing credentials, business logic, security policy)
213
+
214
+ Respond with exactly one word: TACTICAL or HUMAN"""
215
+
216
+ try:
217
+ response = self.llm.complete(
218
+ messages=[{"role": "user", "content": prompt}],
219
+ purpose=Purpose.SUPERVISION,
220
+ max_tokens=10,
221
+ temperature=0.0,
222
+ )
223
+ result = response.content.strip().upper()
224
+ return "tactical" if "TACTICAL" in result else "human"
225
+ except Exception as e:
226
+ print(f" [Supervisor] Classification failed: {e}")
227
+ # Default to tactical for common patterns
228
+ return "tactical" if self._is_tactical_question(question.lower()) else "human"
229
+
230
+ def _generate_tactical_resolution(self, question: str) -> str:
231
+ """Generate an autonomous resolution for tactical questions."""
232
+ q = question.lower()
233
+
234
+ # Common resolutions
235
+ if "virtual environment" in q or "venv" in q:
236
+ return "Create a Python virtual environment using 'python -m venv .venv' or 'uv venv', then activate it and install dependencies."
237
+ if "externally-managed" in q or "externally managed" in q:
238
+ return "This system uses an externally-managed Python. Use 'uv pip install' instead of 'pip install', or create a virtual environment first with 'uv venv' then 'source .venv/bin/activate'."
239
+ if "__main__" in q or "cannot be directly executed" in q:
240
+ return "Create a __main__.py file in the package directory with the entry point code (e.g., 'from .cli import main; main()' or similar)."
241
+ if "no module named" in q or "modulenotfounderror" in q:
242
+ return "Install the missing module using 'uv pip install <module>' or add it to requirements.txt and run 'uv pip install -r requirements.txt'."
243
+ if "importerror" in q:
244
+ return "Check that the module is installed and the import path is correct. For local modules, ensure __init__.py exists in the package directory."
245
+ if "fixture scope" in q or "asyncio" in q or "asyncio_default_fixture_loop_scope" in q:
246
+ return "Add '[tool.pytest.ini_options]\nasyncio_default_fixture_loop_scope = \"function\"' to pyproject.toml or 'asyncio_default_fixture_loop_scope = function' to pytest.ini."
247
+ if "package manager" in q:
248
+ return "Use the project's default package manager (uv for Python, npm for JS). For Python, prefer 'uv pip install' or 'uv sync'."
249
+ if "pytest" in q and "fail" in q:
250
+ return "Fix the failing tests and retry."
251
+ if "overwrite" in q or "existing file" in q:
252
+ return "Overwrite the existing file with the new content."
253
+ if "which version" in q:
254
+ return "Use the latest stable version."
255
+
256
+ # Generic resolution
257
+ return "Proceed with the most appropriate approach based on best practices."
258
+
259
+ def _auto_answer_blocker(self, blocker: blockers.Blocker, answer: str) -> None:
260
+ """Auto-answer a blocker and reset the task for retry."""
261
+ # Answer the blocker
262
+ blockers.answer(self.workspace, blocker.id, f"[Auto-resolved by supervisor] {answer}")
263
+
264
+ # Reset the blocked run so task can retry
265
+ if blocker.task_id:
266
+ reset_blocked_run(self.workspace, blocker.task_id)
267
+
268
+
269
+ # Global supervisor instance per workspace (created lazily)
270
+ _supervisors: dict[str, SupervisorResolver] = {}
271
+
272
+
273
+ def get_supervisor(workspace: Workspace) -> SupervisorResolver:
274
+ """Get or create a supervisor resolver for a workspace."""
275
+ if workspace.id not in _supervisors:
276
+ _supervisors[workspace.id] = SupervisorResolver(workspace)
277
+ return _supervisors[workspace.id]
278
+
279
+
280
+ @dataclass
281
+ class GlobalFix:
282
+ """A global fix that affects project-wide state.
283
+
284
+ Attributes:
285
+ error_signature: Hash of the normalized error
286
+ fix_description: What the fix does
287
+ fix_type: Type of fix (shell, edit, create)
288
+ command: Shell command (for shell type)
289
+ file_path: File to modify (for edit/create)
290
+ status: pending, executing, completed, failed
291
+ """
292
+ error_signature: str
293
+ fix_description: str
294
+ fix_type: str
295
+ command: Optional[str] = None
296
+ file_path: Optional[str] = None
297
+ status: str = "pending"
298
+ result: Optional[str] = None
299
+
300
+
301
+ class GlobalFixCoordinator:
302
+ """Coordinates global fixes across parallel agents.
303
+
304
+ When multiple agents hit the same global issue (e.g., missing local package),
305
+ this coordinator ensures:
306
+ 1. Only ONE fix is executed (not N times for N agents)
307
+ 2. Other agents wait for the fix to complete
308
+ 3. Completed fixes are cached to skip redundant work
309
+
310
+ Thread-safe for parallel batch execution.
311
+ """
312
+
313
+ def __init__(self, workspace: Workspace):
314
+ self.workspace = workspace
315
+ self._lock = threading.Lock()
316
+ self._pending: dict[str, GlobalFix] = {} # error_sig -> GlobalFix
317
+ self._completed: dict[str, GlobalFix] = {} # error_sig -> GlobalFix (succeeded)
318
+ self._condition = threading.Condition(self._lock)
319
+
320
+ def _hash_error(self, error: str) -> str:
321
+ """Create a stable hash for an error message.
322
+
323
+ Normalizes variable parts (line numbers, paths) before hashing.
324
+ """
325
+ # Remove line numbers
326
+ normalized = re.sub(r"line \d+", "line N", error)
327
+ # Remove specific file paths but keep filename
328
+ normalized = re.sub(r"/[^\s:]+/([^/\s:]+)", r"\1", normalized)
329
+ # Remove memory addresses
330
+ normalized = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", normalized)
331
+ return hashlib.sha256(normalized.encode()).hexdigest()[:16]
332
+
333
+ def request_fix(
334
+ self,
335
+ error: str,
336
+ fix_type: str,
337
+ fix_description: str,
338
+ command: Optional[str] = None,
339
+ file_path: Optional[str] = None,
340
+ task_id: Optional[str] = None,
341
+ ) -> tuple[str, bool]:
342
+ """Request a global fix from the coordinator.
343
+
344
+ Returns:
345
+ Tuple of (status, should_execute):
346
+ - ("already_completed", False): Fix was already done, just retry verification
347
+ - ("pending", False): Another agent is handling it, wait and retry
348
+ - ("execute", True): You are responsible for executing this fix
349
+ """
350
+ error_sig = self._hash_error(error)
351
+
352
+ with self._lock:
353
+ # Already completed successfully?
354
+ if error_sig in self._completed:
355
+ print(" [GlobalFix] Fix already completed for this error")
356
+ return ("already_completed", False)
357
+
358
+ # Already being worked on?
359
+ if error_sig in self._pending:
360
+ print(" [GlobalFix] Another agent is fixing this, waiting...")
361
+ # Wait for completion (with timeout)
362
+ return ("pending", False)
363
+
364
+ # This agent will handle it
365
+ fix = GlobalFix(
366
+ error_signature=error_sig,
367
+ fix_description=fix_description,
368
+ fix_type=fix_type,
369
+ command=command,
370
+ file_path=file_path,
371
+ status="executing",
372
+ )
373
+ self._pending[error_sig] = fix
374
+ print(f" [GlobalFix] Agent taking ownership: {fix_description[:60]}...")
375
+ return ("execute", True)
376
+
377
+ def report_fix_result(
378
+ self,
379
+ error: str,
380
+ success: bool,
381
+ result_message: Optional[str] = None,
382
+ ) -> None:
383
+ """Report the result of executing a global fix.
384
+
385
+ Args:
386
+ error: Original error message
387
+ success: Whether the fix succeeded
388
+ result_message: Optional details about the result
389
+ """
390
+ error_sig = self._hash_error(error)
391
+
392
+ with self._lock:
393
+ if error_sig not in self._pending:
394
+ return # Not our fix
395
+
396
+ fix = self._pending.pop(error_sig)
397
+ fix.status = "completed" if success else "failed"
398
+ fix.result = result_message
399
+
400
+ if success:
401
+ self._completed[error_sig] = fix
402
+ print(" [GlobalFix] Fix completed successfully")
403
+ else:
404
+ print(f" [GlobalFix] Fix failed: {result_message}")
405
+
406
+ # Notify any waiting agents
407
+ self._condition.notify_all()
408
+
409
+ def wait_for_fix(self, error: str, timeout: float = 60.0) -> bool:
410
+ """Wait for another agent to complete a fix.
411
+
412
+ Args:
413
+ error: Error message we're waiting on
414
+ timeout: Max seconds to wait
415
+
416
+ Returns:
417
+ True if fix was completed successfully, False otherwise
418
+ """
419
+ error_sig = self._hash_error(error)
420
+
421
+ with self._condition:
422
+ start = datetime.now(timezone.utc)
423
+ while error_sig in self._pending:
424
+ remaining = timeout - (datetime.now(timezone.utc) - start).total_seconds()
425
+ if remaining <= 0:
426
+ print(" [GlobalFix] Timeout waiting for fix")
427
+ return False
428
+ self._condition.wait(timeout=remaining)
429
+
430
+ # Check if it completed successfully
431
+ return error_sig in self._completed
432
+
433
+ def is_fixed(self, error: str) -> bool:
434
+ """Check if an error has already been fixed."""
435
+ error_sig = self._hash_error(error)
436
+ with self._lock:
437
+ return error_sig in self._completed
438
+
439
+
440
+ # Global coordinator instances per workspace
441
+ _coordinators: dict[str, GlobalFixCoordinator] = {}
442
+
443
+
444
+ def get_fix_coordinator(workspace: Workspace) -> GlobalFixCoordinator:
445
+ """Get or create a global fix coordinator for a workspace."""
446
+ if workspace.id not in _coordinators:
447
+ _coordinators[workspace.id] = GlobalFixCoordinator(workspace)
448
+ return _coordinators[workspace.id]
449
+
450
+
451
+ def _utc_now() -> datetime:
452
+ """Get current UTC time as timezone-aware datetime."""
453
+ return datetime.now(timezone.utc)
454
+
455
+
456
+ @dataclass
457
+ class ConcurrencyConfig:
458
+ """Per-state concurrency limits for batch execution.
459
+
460
+ When ``by_status`` is non-empty, each status key caps the number of
461
+ concurrent workers for tasks in that state. Unspecified statuses
462
+ fall back to ``max_parallel``.
463
+ """
464
+
465
+ max_parallel: int = 4
466
+ by_status: dict[str, int] = field(default_factory=dict)
467
+
468
+ def get_limit_for_status(self, status: str) -> int:
469
+ """Return the concurrency limit for a given task status."""
470
+ return self.by_status.get(status, self.max_parallel)
471
+
472
+ def effective_workers(
473
+ self,
474
+ *,
475
+ statuses: list[str],
476
+ group_size: int,
477
+ global_running: int,
478
+ ) -> int:
479
+ """Compute the effective worker count for a group of tasks.
480
+
481
+ Takes the minimum of:
482
+ - Global slots remaining (max_parallel - global_running)
483
+ - Per-status limit for the most constrained status in the group
484
+ - Group size
485
+ """
486
+ global_slots = max(1, self.max_parallel - global_running)
487
+ if statuses and self.by_status:
488
+ per_status = min(self.get_limit_for_status(s) for s in statuses)
489
+ else:
490
+ per_status = self.max_parallel
491
+ return max(1, min(global_slots, per_status, group_size))
492
+
493
+
494
+ def parse_concurrency_by_status(value: str | None) -> dict[str, int]:
495
+ """Parse a --max-parallel-by-status string into a dict.
496
+
497
+ Format: "READY=3,IN_PROGRESS=2"
498
+
499
+ Raises:
500
+ ValueError: On invalid status names or format.
501
+ """
502
+ from codeframe.core.state_machine import TaskStatus
503
+
504
+ if not value:
505
+ return {}
506
+
507
+ valid_statuses = {s.value for s in TaskStatus}
508
+ result: dict[str, int] = {}
509
+
510
+ for pair in value.split(","):
511
+ pair = pair.strip()
512
+ if "=" not in pair:
513
+ raise ValueError(f"Invalid format '{pair}'. Expected STATUS=N")
514
+ key, val = pair.split("=", 1)
515
+ key = key.strip().upper()
516
+ if key not in valid_statuses:
517
+ raise ValueError(f"Invalid status '{key}'. Valid: {', '.join(sorted(valid_statuses))}")
518
+ try:
519
+ result[key] = int(val.strip())
520
+ except ValueError:
521
+ raise ValueError(f"Invalid value '{val.strip()}' for status '{key}'. Must be an integer.")
522
+
523
+ return result
524
+
525
+
526
+ class BatchStatus(str, Enum):
527
+ """Status of a batch execution."""
528
+
529
+ PENDING = "PENDING" # Created but not started
530
+ RUNNING = "RUNNING" # Tasks being processed
531
+ COMPLETED = "COMPLETED" # All tasks finished successfully
532
+ PARTIAL = "PARTIAL" # Some tasks completed, some failed/blocked
533
+ FAILED = "FAILED" # Critical failure
534
+ CANCELLED = "CANCELLED" # User cancelled
535
+
536
+
537
+ class OnFailure(str, Enum):
538
+ """Behavior when a task fails."""
539
+
540
+ CONTINUE = "continue" # Continue with remaining tasks
541
+ STOP = "stop" # Stop batch on first failure
542
+
543
+
544
+ @dataclass
545
+ class BatchRun:
546
+ """Represents a batch execution run.
547
+
548
+ Attributes:
549
+ id: Unique batch identifier (UUID)
550
+ workspace_id: Workspace this batch belongs to
551
+ task_ids: Ordered list of task IDs to execute
552
+ status: Current batch status
553
+ strategy: Execution strategy (serial, parallel)
554
+ max_parallel: Max concurrent tasks (for parallel strategy)
555
+ on_failure: Behavior on task failure
556
+ started_at: When the batch started
557
+ completed_at: When the batch finished (if finished)
558
+ results: Dict mapping task_id -> RunStatus value
559
+ """
560
+
561
+ id: str
562
+ workspace_id: str
563
+ task_ids: list[str]
564
+ status: BatchStatus
565
+ strategy: str
566
+ max_parallel: int
567
+ on_failure: OnFailure
568
+ started_at: datetime
569
+ completed_at: Optional[datetime]
570
+ results: dict[str, str] = field(default_factory=dict)
571
+ engine: str = "react"
572
+ stall_timeout_s: int = 300
573
+ stall_action: str = "blocker"
574
+ concurrency: ConcurrencyConfig = field(default_factory=ConcurrencyConfig)
575
+ isolate: bool = True
576
+ isolation: str = "none"
577
+ llm_provider: Optional[str] = None
578
+ llm_model: Optional[str] = None
579
+
580
+
581
+ def start_batch(
582
+ workspace: Workspace,
583
+ task_ids: list[str],
584
+ strategy: str = "serial",
585
+ max_parallel: int = 4,
586
+ on_failure: str = "continue",
587
+ dry_run: bool = False,
588
+ max_retries: int = 0,
589
+ on_event: Optional[Callable[[str, dict], None]] = None,
590
+ engine: str = "react",
591
+ stall_timeout_s: int = 300,
592
+ stall_action: str = "blocker",
593
+ concurrency_by_status: Optional[dict[str, int]] = None,
594
+ isolate: bool = True,
595
+ isolation: str = "none",
596
+ cloud_timeout_minutes: int = 30,
597
+ llm_provider: Optional[str] = None,
598
+ llm_model: Optional[str] = None,
599
+ ) -> BatchRun:
600
+ """Start a batch execution of multiple tasks.
601
+
602
+ Args:
603
+ workspace: Target workspace
604
+ task_ids: List of task IDs to execute (in order)
605
+ strategy: Execution strategy ("serial" or "parallel")
606
+ max_parallel: Max concurrent tasks for parallel strategy
607
+ on_failure: Behavior on task failure ("continue" or "stop")
608
+ dry_run: If True, don't actually execute tasks
609
+ max_retries: Max retry attempts for failed tasks (0 = no retries)
610
+ on_event: Optional callback for batch events
611
+ engine: Agent engine to use ("react" default, or "plan" for legacy)
612
+
613
+ Returns:
614
+ BatchRun with results populated
615
+
616
+ Raises:
617
+ ValueError: If task_ids is empty or contains invalid IDs
618
+ """
619
+ if not task_ids:
620
+ raise ValueError("task_ids cannot be empty")
621
+
622
+ # Validate all task IDs exist
623
+ for task_id in task_ids:
624
+ task = tasks.get(workspace, task_id)
625
+ if not task:
626
+ raise ValueError(f"Task not found: {task_id}")
627
+
628
+ # Create batch record
629
+ batch_id = str(uuid.uuid4())
630
+ now = _utc_now()
631
+ on_failure_enum = OnFailure(on_failure)
632
+
633
+ concurrency = ConcurrencyConfig(
634
+ max_parallel=max_parallel,
635
+ by_status=concurrency_by_status or {},
636
+ )
637
+
638
+ batch = BatchRun(
639
+ id=batch_id,
640
+ workspace_id=workspace.id,
641
+ task_ids=task_ids,
642
+ status=BatchStatus.PENDING,
643
+ strategy=strategy,
644
+ max_parallel=max_parallel,
645
+ on_failure=on_failure_enum,
646
+ started_at=now,
647
+ completed_at=None,
648
+ results={},
649
+ engine=engine,
650
+ stall_timeout_s=stall_timeout_s,
651
+ stall_action=stall_action,
652
+ concurrency=concurrency,
653
+ isolate=isolate,
654
+ isolation=isolation,
655
+ llm_provider=llm_provider,
656
+ llm_model=llm_model,
657
+ )
658
+
659
+ # Save to database
660
+ _save_batch(workspace, batch)
661
+
662
+ # Emit batch started event
663
+ events.emit_for_workspace(
664
+ workspace,
665
+ events.EventType.BATCH_STARTED,
666
+ {
667
+ "batch_id": batch_id,
668
+ "task_ids": task_ids,
669
+ "strategy": strategy,
670
+ "task_count": len(task_ids),
671
+ },
672
+ print_event=True,
673
+ )
674
+
675
+ if on_event:
676
+ on_event("batch_started", {"batch_id": batch_id, "task_count": len(task_ids)})
677
+
678
+ if dry_run:
679
+ batch.status = BatchStatus.COMPLETED
680
+ batch.completed_at = _utc_now()
681
+ _save_batch(workspace, batch)
682
+ return batch
683
+
684
+ # Update status to running
685
+ batch.status = BatchStatus.RUNNING
686
+ _save_batch(workspace, batch)
687
+
688
+ # Execute based on strategy
689
+ if strategy == "auto":
690
+ # Use LLM to infer dependencies, then execute in parallel
691
+ try:
692
+ print("\nAnalyzing task dependencies with LLM...")
693
+ dependencies = analyze_dependencies(workspace, task_ids)
694
+
695
+ # Show inferred dependencies
696
+ deps_with_values = {k: v for k, v in dependencies.items() if v}
697
+ if deps_with_values:
698
+ print("Inferred dependencies:")
699
+ for tid, deps in deps_with_values.items():
700
+ task = tasks.get(workspace, tid)
701
+ task_title = task.title[:40] if task else tid[:8]
702
+ dep_titles = []
703
+ for d in deps:
704
+ dep_task = tasks.get(workspace, d)
705
+ dep_titles.append(dep_task.title[:30] if dep_task else d[:8])
706
+ print(f" {task_title} <- {', '.join(dep_titles)}")
707
+ else:
708
+ print("No dependencies inferred - tasks appear independent")
709
+
710
+ # Apply inferred dependencies to task records
711
+ # Note: This persists the dependencies to the database so they're
712
+ # available for future executions and batch resumes
713
+ apply_inferred_dependencies(workspace, dependencies)
714
+
715
+ # Execute with parallel strategy
716
+ _execute_parallel(workspace, batch, on_event)
717
+ except CycleDetectedError as e:
718
+ print(f"Error: {e}")
719
+ print("Falling back to serial execution")
720
+ _execute_serial(workspace, batch, on_event)
721
+ except Exception as e:
722
+ print(f"Dependency analysis failed: {e}")
723
+ print("Falling back to serial execution")
724
+ _execute_serial(workspace, batch, on_event)
725
+ elif strategy == "parallel" and max_parallel > 1:
726
+ try:
727
+ _execute_parallel(workspace, batch, on_event)
728
+ except CycleDetectedError as e:
729
+ print(f"Error: {e}")
730
+ print("Falling back to serial execution")
731
+ _execute_serial(workspace, batch, on_event)
732
+ else:
733
+ _execute_serial(workspace, batch, on_event)
734
+
735
+ # Retry failed tasks if max_retries > 0
736
+ if max_retries > 0:
737
+ _execute_retries(workspace, batch, max_retries, on_event)
738
+
739
+ return batch
740
+
741
+
742
+ def get_batch(workspace: Workspace, batch_id: str) -> Optional[BatchRun]:
743
+ """Get a batch by ID.
744
+
745
+ Args:
746
+ workspace: Workspace to query
747
+ batch_id: Batch identifier
748
+
749
+ Returns:
750
+ BatchRun if found, None otherwise
751
+ """
752
+ conn = get_db_connection(workspace)
753
+ cursor = conn.cursor()
754
+
755
+ cursor.execute(
756
+ """
757
+ SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
758
+ on_failure, started_at, completed_at, results, engine
759
+ FROM batch_runs
760
+ WHERE workspace_id = ? AND id = ?
761
+ """,
762
+ (workspace.id, batch_id),
763
+ )
764
+ row = cursor.fetchone()
765
+ conn.close()
766
+
767
+ if not row:
768
+ return None
769
+
770
+ return _row_to_batch(row)
771
+
772
+
773
+ def list_batches(
774
+ workspace: Workspace,
775
+ status: Optional[BatchStatus] = None,
776
+ limit: int = 20,
777
+ ) -> list[BatchRun]:
778
+ """List batches in a workspace.
779
+
780
+ Args:
781
+ workspace: Workspace to query
782
+ status: Optional status filter
783
+ limit: Maximum batches to return
784
+
785
+ Returns:
786
+ List of BatchRuns, newest first
787
+ """
788
+ conn = get_db_connection(workspace)
789
+ cursor = conn.cursor()
790
+
791
+ if status:
792
+ cursor.execute(
793
+ """
794
+ SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
795
+ on_failure, started_at, completed_at, results, engine
796
+ FROM batch_runs
797
+ WHERE workspace_id = ? AND status = ?
798
+ ORDER BY started_at DESC
799
+ LIMIT ?
800
+ """,
801
+ (workspace.id, status.value, limit),
802
+ )
803
+ else:
804
+ cursor.execute(
805
+ """
806
+ SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
807
+ on_failure, started_at, completed_at, results, engine
808
+ FROM batch_runs
809
+ WHERE workspace_id = ?
810
+ ORDER BY started_at DESC
811
+ LIMIT ?
812
+ """,
813
+ (workspace.id, limit),
814
+ )
815
+
816
+ rows = cursor.fetchall()
817
+ conn.close()
818
+
819
+ return [_row_to_batch(row) for row in rows]
820
+
821
+
822
+ def cancel_batch(workspace: Workspace, batch_id: str) -> BatchRun:
823
+ """Cancel a running batch.
824
+
825
+ Sends SIGTERM to any running subprocesses and marks the batch as cancelled.
826
+
827
+ Args:
828
+ workspace: Target workspace
829
+ batch_id: Batch to cancel
830
+
831
+ Returns:
832
+ Updated BatchRun
833
+
834
+ Raises:
835
+ ValueError: If batch not found or not in a cancellable state
836
+ """
837
+ batch = get_batch(workspace, batch_id)
838
+ if not batch:
839
+ raise ValueError(f"Batch not found: {batch_id}")
840
+
841
+ if batch.status not in (BatchStatus.PENDING, BatchStatus.RUNNING):
842
+ raise ValueError(f"Batch cannot be cancelled: {batch.status}")
843
+
844
+ # Update status
845
+ batch.status = BatchStatus.CANCELLED
846
+ batch.completed_at = _utc_now()
847
+ _save_batch(workspace, batch)
848
+
849
+ # Emit event
850
+ events.emit_for_workspace(
851
+ workspace,
852
+ events.EventType.BATCH_CANCELLED,
853
+ {"batch_id": batch_id},
854
+ print_event=True,
855
+ )
856
+
857
+ return batch
858
+
859
+
860
+ def stop_batch(workspace: Workspace, batch_id: str, force: bool = False) -> BatchRun:
861
+ """Stop a running batch.
862
+
863
+ Graceful stop (force=False):
864
+ - Marks batch as CANCELLED
865
+ - Execution loops will exit after current task completes
866
+ - Running tasks are allowed to finish naturally
867
+
868
+ Force stop (force=True):
869
+ - Marks batch as CANCELLED immediately
870
+ - Sends SIGTERM to all running subprocesses
871
+ - Processes have a brief window to cleanup before termination
872
+
873
+ Args:
874
+ workspace: Target workspace
875
+ batch_id: Batch to stop
876
+ force: If True, terminate running processes immediately
877
+
878
+ Returns:
879
+ Updated BatchRun
880
+
881
+ Raises:
882
+ ValueError: If batch not found or not stoppable
883
+ """
884
+ batch = get_batch(workspace, batch_id)
885
+ if not batch:
886
+ raise ValueError(f"Batch not found: {batch_id}")
887
+
888
+ if batch.status not in (BatchStatus.PENDING, BatchStatus.RUNNING):
889
+ raise ValueError(f"Batch cannot be stopped (status={batch.status})")
890
+
891
+ # Update status first - execution loops check this
892
+ batch.status = BatchStatus.CANCELLED
893
+ batch.completed_at = _utc_now()
894
+ _save_batch(workspace, batch)
895
+
896
+ terminated_count = 0
897
+ with _active_processes_lock:
898
+ if force and batch_id in _active_processes:
899
+ # Terminate all running processes for this batch
900
+ processes = _active_processes.get(batch_id, {})
901
+ for task_id, process in list(processes.items()):
902
+ try:
903
+ if process.poll() is None: # Still running
904
+ process.terminate() # SIGTERM
905
+ terminated_count += 1
906
+ except (ProcessLookupError, OSError):
907
+ pass # Process already exited
908
+
909
+ # Cleanup tracking
910
+ _active_processes.pop(batch_id, None)
911
+
912
+ # Emit event
913
+ event_data = {"batch_id": batch_id, "force": force}
914
+ if terminated_count > 0:
915
+ event_data["terminated_processes"] = terminated_count
916
+
917
+ events.emit_for_workspace(
918
+ workspace,
919
+ events.EventType.BATCH_CANCELLED,
920
+ event_data,
921
+ print_event=True,
922
+ )
923
+
924
+ return batch
925
+
926
+
927
+ def resume_batch(
928
+ workspace: Workspace,
929
+ batch_id: str,
930
+ force: bool = False,
931
+ on_event: Optional[Callable[[str, dict], None]] = None,
932
+ ) -> BatchRun:
933
+ """Resume a batch by re-running failed/blocked tasks.
934
+
935
+ Args:
936
+ workspace: Target workspace
937
+ batch_id: Batch to resume
938
+ force: If True, re-run all tasks including completed ones
939
+ on_event: Optional callback for batch events
940
+
941
+ Returns:
942
+ Updated BatchRun with new results
943
+
944
+ Raises:
945
+ ValueError: If batch not found or not in a resumable state
946
+ """
947
+ batch = get_batch(workspace, batch_id)
948
+ if not batch:
949
+ raise ValueError(f"Batch not found: {batch_id}")
950
+
951
+ # Check if batch is in a resumable state
952
+ resumable_statuses = (
953
+ BatchStatus.PARTIAL,
954
+ BatchStatus.FAILED,
955
+ BatchStatus.CANCELLED,
956
+ )
957
+ if batch.status not in resumable_statuses:
958
+ raise ValueError(
959
+ f"Batch cannot be resumed (status={batch.status}). "
960
+ f"Only {', '.join(s.value for s in resumable_statuses)} batches can be resumed."
961
+ )
962
+
963
+ # Determine which tasks to re-run
964
+ if force:
965
+ # Re-run all tasks
966
+ tasks_to_run = batch.task_ids
967
+ print(f"Resuming batch {batch_id[:8]}... (force mode: re-running all {len(tasks_to_run)} tasks)")
968
+ else:
969
+ # Only re-run failed/blocked tasks
970
+ failed_statuses = {"FAILED", "BLOCKED"}
971
+ tasks_to_run = [
972
+ tid for tid in batch.task_ids
973
+ if batch.results.get(tid) in failed_statuses or tid not in batch.results
974
+ ]
975
+ if not tasks_to_run:
976
+ print(f"No failed or blocked tasks to resume in batch {batch_id[:8]}")
977
+ return batch
978
+ print(f"Resuming batch {batch_id[:8]}... (re-running {len(tasks_to_run)} failed/blocked tasks)")
979
+
980
+ # Emit batch resumed event
981
+ events.emit_for_workspace(
982
+ workspace,
983
+ events.EventType.BATCH_STARTED, # Reuse BATCH_STARTED for resume
984
+ {
985
+ "batch_id": batch_id,
986
+ "task_ids": tasks_to_run,
987
+ "strategy": batch.strategy,
988
+ "task_count": len(tasks_to_run),
989
+ "is_resume": True,
990
+ },
991
+ print_event=True,
992
+ )
993
+
994
+ if on_event:
995
+ on_event("batch_resumed", {"batch_id": batch_id, "task_count": len(tasks_to_run)})
996
+
997
+ # Update status to running
998
+ batch.status = BatchStatus.RUNNING
999
+ batch.completed_at = None # Clear completed_at since we're running again
1000
+ _save_batch(workspace, batch)
1001
+
1002
+ # Execute the tasks
1003
+ _execute_serial_resume(workspace, batch, tasks_to_run, on_event)
1004
+
1005
+ return batch
1006
+
1007
+
1008
+ def _execute_serial_resume(
1009
+ workspace: Workspace,
1010
+ batch: BatchRun,
1011
+ tasks_to_run: list[str],
1012
+ on_event: Optional[Callable[[str, dict], None]] = None,
1013
+ ) -> None:
1014
+ """Execute a subset of tasks serially for resume.
1015
+
1016
+ Similar to _execute_serial but only runs specified tasks and
1017
+ merges results with existing batch results.
1018
+ """
1019
+ completed_count = 0
1020
+ failed_count = 0
1021
+ blocked_count = 0
1022
+
1023
+ # Count existing successful results (tasks not being re-run)
1024
+ for task_id in batch.task_ids:
1025
+ if task_id not in tasks_to_run:
1026
+ if batch.results.get(task_id) == RunStatus.COMPLETED.value:
1027
+ completed_count += 1
1028
+
1029
+ for i, task_id in enumerate(tasks_to_run):
1030
+ # Check if batch was cancelled
1031
+ current_batch = get_batch(workspace, batch.id)
1032
+ if current_batch and current_batch.status == BatchStatus.CANCELLED:
1033
+ break
1034
+
1035
+ # Emit task queued event
1036
+ events.emit_for_workspace(
1037
+ workspace,
1038
+ events.EventType.BATCH_TASK_QUEUED,
1039
+ {"batch_id": batch.id, "task_id": task_id, "position": i + 1},
1040
+ print_event=True,
1041
+ )
1042
+
1043
+ # Get task info for display
1044
+ task = tasks.get(workspace, task_id)
1045
+ task_title = task.title if task else task_id
1046
+ previous_status = batch.results.get(task_id, "N/A")
1047
+
1048
+ print(f"\n[{i + 1}/{len(tasks_to_run)}] Retrying task {task_id}: {task_title}")
1049
+ print(f" Previous status: {previous_status}")
1050
+
1051
+ # Emit task started event
1052
+ events.emit_for_workspace(
1053
+ workspace,
1054
+ events.EventType.BATCH_TASK_STARTED,
1055
+ {"batch_id": batch.id, "task_id": task_id, "is_retry": True},
1056
+ print_event=True,
1057
+ )
1058
+
1059
+ if on_event:
1060
+ on_event("batch_task_started", {"task_id": task_id, "position": i + 1, "is_retry": True})
1061
+
1062
+ # Execute task via subprocess (with isolation context)
1063
+ from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
1064
+ exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
1065
+ try:
1066
+ result_status = _execute_task_subprocess(
1067
+ workspace, task_id, batch.id, engine=batch.engine,
1068
+ stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
1069
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1070
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1071
+ )
1072
+ finally:
1073
+ exec_ctx.cleanup()
1074
+
1075
+ # Record result (overwrites previous result)
1076
+ batch.results[task_id] = result_status
1077
+ _save_batch(workspace, batch)
1078
+
1079
+ # Emit appropriate event based on result
1080
+ if result_status == RunStatus.COMPLETED.value:
1081
+ completed_count += 1
1082
+ events.emit_for_workspace(
1083
+ workspace,
1084
+ events.EventType.BATCH_TASK_COMPLETED,
1085
+ {"batch_id": batch.id, "task_id": task_id},
1086
+ print_event=True,
1087
+ )
1088
+ print(f" ✓ Completed (was: {previous_status})")
1089
+ elif result_status == RunStatus.BLOCKED.value:
1090
+ blocked_count += 1
1091
+ events.emit_for_workspace(
1092
+ workspace,
1093
+ events.EventType.BATCH_TASK_BLOCKED,
1094
+ {"batch_id": batch.id, "task_id": task_id},
1095
+ print_event=True,
1096
+ )
1097
+ print(" ⊘ Still blocked")
1098
+ else:
1099
+ failed_count += 1
1100
+ events.emit_for_workspace(
1101
+ workspace,
1102
+ events.EventType.BATCH_TASK_FAILED,
1103
+ {"batch_id": batch.id, "task_id": task_id, "status": result_status},
1104
+ print_event=True,
1105
+ )
1106
+ print(f" ✗ Still failed: {result_status}")
1107
+
1108
+ # Note: resume doesn't stop on failure, always continues
1109
+ # to give all failed tasks a chance
1110
+
1111
+ if on_event:
1112
+ on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
1113
+
1114
+ # Determine final batch status based on ALL results
1115
+ total = len(batch.task_ids)
1116
+
1117
+ # Recount from results
1118
+ final_completed = sum(1 for s in batch.results.values() if s == RunStatus.COMPLETED.value)
1119
+ final_failed = sum(1 for s in batch.results.values() if s == RunStatus.FAILED.value)
1120
+ final_blocked = sum(1 for s in batch.results.values() if s == RunStatus.BLOCKED.value)
1121
+
1122
+ if final_completed == total:
1123
+ batch.status = BatchStatus.COMPLETED
1124
+ event_type = events.EventType.BATCH_COMPLETED
1125
+
1126
+ # Run batch-level validation (full gate sweep)
1127
+ validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
1128
+
1129
+ if not validation_passed:
1130
+ # Gates failed - change status to PARTIAL (tasks done, integration broken)
1131
+ batch.status = BatchStatus.PARTIAL
1132
+ event_type = events.EventType.BATCH_PARTIAL
1133
+ print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
1134
+ print(f"Validation error: {validation_error}")
1135
+
1136
+ elif final_completed == 0 and (final_failed > 0 or final_blocked > 0):
1137
+ batch.status = BatchStatus.FAILED
1138
+ event_type = events.EventType.BATCH_FAILED
1139
+ elif final_completed > 0:
1140
+ batch.status = BatchStatus.PARTIAL
1141
+ event_type = events.EventType.BATCH_PARTIAL
1142
+ else:
1143
+ batch.status = BatchStatus.CANCELLED
1144
+ event_type = events.EventType.BATCH_CANCELLED
1145
+
1146
+ batch.completed_at = _utc_now()
1147
+ _save_batch(workspace, batch)
1148
+
1149
+ # Emit batch completion event
1150
+ events.emit_for_workspace(
1151
+ workspace,
1152
+ event_type,
1153
+ {
1154
+ "batch_id": batch.id,
1155
+ "completed": final_completed,
1156
+ "failed": final_failed,
1157
+ "blocked": final_blocked,
1158
+ "total": total,
1159
+ "is_resume": True,
1160
+ },
1161
+ print_event=True,
1162
+ )
1163
+ _dispatch_batch_completed_webhook(workspace, event_type, batch.id, final_completed)
1164
+
1165
+ # Print summary
1166
+ print(f"\nBatch resume {batch.status.value.lower()}: {final_completed}/{total} tasks completed")
1167
+ if final_failed > 0:
1168
+ print(f" Failed: {final_failed}")
1169
+ if final_blocked > 0:
1170
+ print(f" Blocked: {final_blocked}")
1171
+
1172
+
1173
+ def _execute_retries(
1174
+ workspace: Workspace,
1175
+ batch: BatchRun,
1176
+ max_retries: int,
1177
+ on_event: Optional[Callable[[str, dict], None]] = None,
1178
+ ) -> None:
1179
+ """Retry failed tasks up to max_retries times.
1180
+
1181
+ After the initial execution pass, this function re-runs any failed
1182
+ tasks, continuing until all tasks succeed or max_retries is exhausted.
1183
+
1184
+ Args:
1185
+ workspace: Target workspace
1186
+ batch: BatchRun with initial results
1187
+ max_retries: Maximum retry attempts per task
1188
+ on_event: Optional callback for events
1189
+ """
1190
+ failed_statuses = {RunStatus.FAILED.value} # Only retry FAILED, not BLOCKED
1191
+
1192
+ for retry_num in range(1, max_retries + 1):
1193
+ # Find tasks that failed
1194
+ failed_tasks = [
1195
+ tid for tid in batch.task_ids
1196
+ if batch.results.get(tid) in failed_statuses
1197
+ ]
1198
+
1199
+ if not failed_tasks:
1200
+ # All tasks succeeded, no retries needed
1201
+ break
1202
+
1203
+ print(f"\n{'='*60}")
1204
+ print(f"Retry attempt {retry_num}/{max_retries}: {len(failed_tasks)} failed task(s)")
1205
+ print(f"{'='*60}")
1206
+
1207
+ # Emit retry event
1208
+ events.emit_for_workspace(
1209
+ workspace,
1210
+ events.EventType.BATCH_STARTED, # Reuse for retry
1211
+ {
1212
+ "batch_id": batch.id,
1213
+ "task_ids": failed_tasks,
1214
+ "retry_attempt": retry_num,
1215
+ "max_retries": max_retries,
1216
+ },
1217
+ print_event=True,
1218
+ )
1219
+
1220
+ if on_event:
1221
+ on_event("batch_retry_started", {
1222
+ "batch_id": batch.id,
1223
+ "retry_attempt": retry_num,
1224
+ "task_count": len(failed_tasks),
1225
+ })
1226
+
1227
+ # Re-run each failed task
1228
+ for i, task_id in enumerate(failed_tasks):
1229
+ # Check if batch was cancelled
1230
+ current_batch = get_batch(workspace, batch.id)
1231
+ if current_batch and current_batch.status == BatchStatus.CANCELLED:
1232
+ return
1233
+
1234
+ task = tasks.get(workspace, task_id)
1235
+ task_title = task.title if task else task_id
1236
+ previous_status = batch.results.get(task_id, "N/A")
1237
+
1238
+ print(f"\n[Retry {retry_num}, {i + 1}/{len(failed_tasks)}] {task_id}: {task_title}")
1239
+ print(f" Previous: {previous_status}")
1240
+
1241
+ # Execute task (with isolation context)
1242
+ from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
1243
+ exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
1244
+ try:
1245
+ result_status = _execute_task_subprocess(
1246
+ workspace, task_id, batch.id, engine=batch.engine,
1247
+ stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
1248
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1249
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1250
+ )
1251
+ finally:
1252
+ exec_ctx.cleanup()
1253
+
1254
+ # Update result
1255
+ batch.results[task_id] = result_status
1256
+ _save_batch(workspace, batch)
1257
+
1258
+ if result_status == RunStatus.COMPLETED.value:
1259
+ print(f" ✓ Succeeded on retry {retry_num}")
1260
+ else:
1261
+ remaining = max_retries - retry_num
1262
+ if remaining > 0:
1263
+ print(f" ✗ Still failed ({remaining} retries left)")
1264
+ else:
1265
+ print(f" ✗ Failed after {max_retries} retries")
1266
+
1267
+ if on_event:
1268
+ on_event("batch_task_retried", {
1269
+ "task_id": task_id,
1270
+ "retry_attempt": retry_num,
1271
+ "status": result_status,
1272
+ })
1273
+
1274
+ # Recalculate final batch status after all retries
1275
+ total = len(batch.task_ids)
1276
+ final_completed = sum(1 for s in batch.results.values() if s == RunStatus.COMPLETED.value)
1277
+ final_failed = sum(1 for s in batch.results.values() if s == RunStatus.FAILED.value)
1278
+ final_blocked = sum(1 for s in batch.results.values() if s == RunStatus.BLOCKED.value)
1279
+
1280
+ if final_completed == total:
1281
+ batch.status = BatchStatus.COMPLETED
1282
+ event_type = events.EventType.BATCH_COMPLETED
1283
+
1284
+ # Run batch-level validation (full gate sweep)
1285
+ validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
1286
+
1287
+ if not validation_passed:
1288
+ # Gates failed - change status to PARTIAL (tasks done, integration broken)
1289
+ batch.status = BatchStatus.PARTIAL
1290
+ event_type = events.EventType.BATCH_PARTIAL
1291
+ print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
1292
+ print(f"Validation error: {validation_error}")
1293
+
1294
+ elif final_completed == 0 and (final_failed > 0 or final_blocked > 0):
1295
+ batch.status = BatchStatus.FAILED
1296
+ event_type = events.EventType.BATCH_FAILED
1297
+ elif final_completed > 0:
1298
+ batch.status = BatchStatus.PARTIAL
1299
+ event_type = events.EventType.BATCH_PARTIAL
1300
+ else:
1301
+ batch.status = BatchStatus.CANCELLED
1302
+ event_type = events.EventType.BATCH_CANCELLED
1303
+
1304
+ batch.completed_at = _utc_now()
1305
+ _save_batch(workspace, batch)
1306
+
1307
+ # Emit final status event
1308
+ events.emit_for_workspace(
1309
+ workspace,
1310
+ event_type,
1311
+ {
1312
+ "batch_id": batch.id,
1313
+ "completed": final_completed,
1314
+ "failed": final_failed,
1315
+ "blocked": final_blocked,
1316
+ "total": total,
1317
+ "retries_used": max_retries,
1318
+ },
1319
+ print_event=True,
1320
+ )
1321
+ _dispatch_batch_completed_webhook(workspace, event_type, batch.id, final_completed)
1322
+
1323
+ # Print retry summary
1324
+ if final_failed == 0:
1325
+ print("\n✓ All tasks succeeded after retries")
1326
+ else:
1327
+ print(f"\n⚠ {final_failed} task(s) still failing after {max_retries} retries")
1328
+
1329
+
1330
+ def _run_batch_level_validation(workspace: Workspace, batch: BatchRun) -> tuple[bool, Optional[str]]:
1331
+ """Run full gate sweep after all tasks complete to catch cross-task inconsistencies.
1332
+
1333
+ Args:
1334
+ workspace: The workspace
1335
+ batch: The batch run that just completed
1336
+
1337
+ Returns:
1338
+ Tuple of (passed: bool, failure_summary: Optional[str])
1339
+ - passed=True means all gates passed
1340
+ - passed=False means gates failed, with summary of failures
1341
+ """
1342
+ from codeframe.core import gates
1343
+
1344
+ print("\n[Conductor] Running batch-level validation (full gate sweep)...")
1345
+
1346
+ # Run all auto-detected gates against the full workspace
1347
+ result = gates.run(workspace, gates=None, verbose=False, auto_install_deps=True)
1348
+
1349
+ if result.passed:
1350
+ print("[Conductor] ✓ Batch-level validation passed")
1351
+ return True, None
1352
+ else:
1353
+ # Extract failure summary
1354
+ failure_summary = result.get_error_summary()
1355
+ if not failure_summary:
1356
+ failure_summary = "Gates failed (see individual gate outputs)"
1357
+
1358
+ print(f"[Conductor] ✗ Batch-level validation failed:\n{failure_summary}")
1359
+
1360
+ # Emit batch validation failed event
1361
+ events.emit_for_workspace(
1362
+ workspace,
1363
+ events.EventType.BATCH_VALIDATION_FAILED,
1364
+ {
1365
+ "batch_id": batch.id,
1366
+ "failed_gates": [check.name for check in result.checks if check.status == gates.GateStatus.FAILED],
1367
+ "summary": failure_summary[:500], # Truncate for event payload
1368
+ },
1369
+ print_event=True,
1370
+ )
1371
+
1372
+ return False, failure_summary
1373
+
1374
+
1375
+ def _apply_pending_config_reload(
1376
+ batch: BatchRun,
1377
+ workspace: Workspace,
1378
+ reload_state: "ConfigReloadState",
1379
+ last_seen_reload: datetime,
1380
+ ) -> datetime:
1381
+ """Check for config reloads and record them in batch results.
1382
+
1383
+ Args:
1384
+ batch: Current batch run.
1385
+ workspace: Target workspace.
1386
+ reload_state: Shared reload state from ConfigFileWatcher.
1387
+ last_seen_reload: Timestamp of the last check.
1388
+
1389
+ Returns:
1390
+ Updated last_seen_reload timestamp.
1391
+ """
1392
+ if reload_state.has_reloaded_since(last_seen_reload):
1393
+ now = datetime.now(timezone.utc)
1394
+ reloads = batch.results.setdefault("__config_reloads__", [])
1395
+ reloads.append(now.isoformat())
1396
+ _save_batch(workspace, batch)
1397
+ print(f" [config] Configuration reloaded at {now.strftime('%H:%M:%S')}")
1398
+ return now
1399
+ return last_seen_reload
1400
+
1401
+
1402
+ def _execute_serial(
1403
+ workspace: Workspace,
1404
+ batch: BatchRun,
1405
+ on_event: Optional[Callable[[str, dict], None]] = None,
1406
+ ) -> None:
1407
+ """Execute tasks serially (one at a time).
1408
+
1409
+ Updates batch.results and batch.status as tasks complete.
1410
+ """
1411
+ # Start reconciliation thread for continuous state checking
1412
+ from codeframe.core.config import load_environment_config
1413
+ env_config = load_environment_config(workspace.repo_path)
1414
+ interval = env_config.reconciliation_interval_seconds if env_config else 30
1415
+ reconcile_stop = _start_reconciliation_thread(workspace, batch, interval_seconds=interval)
1416
+
1417
+ # Start config file watcher (optional — failure must not break batch)
1418
+ config_watcher = None
1419
+ reload_state = None
1420
+ _last_seen_reload = datetime.now(timezone.utc)
1421
+ try:
1422
+ from codeframe.core.config_watcher import ConfigFileWatcher
1423
+ from codeframe.core.agents_config import load_preferences
1424
+
1425
+ config_watcher = ConfigFileWatcher(Path(workspace.repo_path))
1426
+ reload_state = config_watcher.start(load_preferences(Path(workspace.repo_path)))
1427
+ except Exception:
1428
+ logger.debug("Config watcher failed to start in serial execution", exc_info=True)
1429
+
1430
+ completed_count = 0
1431
+ failed_count = 0
1432
+ blocked_count = 0
1433
+
1434
+ try:
1435
+ for i, task_id in enumerate(batch.task_ids):
1436
+ # Check if batch was cancelled
1437
+ current_batch = get_batch(workspace, batch.id)
1438
+ if current_batch and current_batch.status == BatchStatus.CANCELLED:
1439
+ break
1440
+
1441
+ # Check for config reloads between tasks
1442
+ if reload_state is not None:
1443
+ _last_seen_reload = _apply_pending_config_reload(
1444
+ batch, workspace, reload_state, _last_seen_reload
1445
+ )
1446
+
1447
+ # Emit task queued event
1448
+ events.emit_for_workspace(
1449
+ workspace,
1450
+ events.EventType.BATCH_TASK_QUEUED,
1451
+ {"batch_id": batch.id, "task_id": task_id, "position": i + 1},
1452
+ print_event=True,
1453
+ )
1454
+
1455
+ # Get task info for display
1456
+ task = tasks.get(workspace, task_id)
1457
+ task_title = task.title if task else task_id
1458
+
1459
+ print(f"\n[{i + 1}/{len(batch.task_ids)}] Starting task {task_id}: {task_title}")
1460
+
1461
+ # Emit task started event
1462
+ events.emit_for_workspace(
1463
+ workspace,
1464
+ events.EventType.BATCH_TASK_STARTED,
1465
+ {"batch_id": batch.id, "task_id": task_id},
1466
+ print_event=True,
1467
+ )
1468
+
1469
+ if on_event:
1470
+ on_event("batch_task_started", {"task_id": task_id, "position": i + 1})
1471
+
1472
+ # Execute task via subprocess (with isolation context)
1473
+ from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
1474
+ exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
1475
+ try:
1476
+ result_status = _execute_task_subprocess(
1477
+ workspace, task_id, batch.id, engine=batch.engine,
1478
+ stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
1479
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1480
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1481
+ )
1482
+
1483
+ # If task is BLOCKED, try supervisor resolution
1484
+ if result_status == RunStatus.BLOCKED.value:
1485
+ supervisor = get_supervisor(workspace)
1486
+ if supervisor.try_resolve_blocked_task(task_id):
1487
+ # Supervisor resolved the blocker - retry the task
1488
+ print(" [Supervisor] Retrying task after auto-resolution...")
1489
+ result_status = _execute_task_subprocess(
1490
+ workspace, task_id, batch.id, engine=batch.engine,
1491
+ stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
1492
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1493
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1494
+ )
1495
+ finally:
1496
+ exec_ctx.cleanup()
1497
+
1498
+ # Record result
1499
+ batch.results[task_id] = result_status
1500
+ _save_batch(workspace, batch)
1501
+
1502
+ # Emit appropriate event based on result
1503
+ if result_status == RunStatus.COMPLETED.value:
1504
+ completed_count += 1
1505
+ events.emit_for_workspace(
1506
+ workspace,
1507
+ events.EventType.BATCH_TASK_COMPLETED,
1508
+ {"batch_id": batch.id, "task_id": task_id},
1509
+ print_event=True,
1510
+ )
1511
+ print(" ✓ Completed")
1512
+ elif result_status == RunStatus.BLOCKED.value:
1513
+ blocked_count += 1
1514
+ events.emit_for_workspace(
1515
+ workspace,
1516
+ events.EventType.BATCH_TASK_BLOCKED,
1517
+ {"batch_id": batch.id, "task_id": task_id},
1518
+ print_event=True,
1519
+ )
1520
+ print(" ⊘ Blocked (requires human input)")
1521
+ else:
1522
+ failed_count += 1
1523
+ events.emit_for_workspace(
1524
+ workspace,
1525
+ events.EventType.BATCH_TASK_FAILED,
1526
+ {"batch_id": batch.id, "task_id": task_id, "status": result_status},
1527
+ print_event=True,
1528
+ )
1529
+ print(f" ✗ Failed: {result_status}")
1530
+
1531
+ # Check on_failure behavior
1532
+ if batch.on_failure == OnFailure.STOP:
1533
+ print("\nStopping batch due to --on-failure=stop")
1534
+ break
1535
+
1536
+ if on_event:
1537
+ on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
1538
+
1539
+ # Determine final batch status
1540
+ total = len(batch.task_ids)
1541
+ completed_count + failed_count + blocked_count
1542
+
1543
+ if completed_count == total:
1544
+ batch.status = BatchStatus.COMPLETED
1545
+ event_type = events.EventType.BATCH_COMPLETED
1546
+
1547
+ # Run batch-level validation (full gate sweep)
1548
+ validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
1549
+
1550
+ if not validation_passed:
1551
+ # Gates failed - change status to PARTIAL (tasks done, integration broken)
1552
+ batch.status = BatchStatus.PARTIAL
1553
+ event_type = events.EventType.BATCH_PARTIAL
1554
+ print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
1555
+ print(f"Validation error: {validation_error}")
1556
+
1557
+ elif completed_count == 0 and (failed_count > 0 or blocked_count > 0):
1558
+ batch.status = BatchStatus.FAILED
1559
+ event_type = events.EventType.BATCH_FAILED
1560
+ elif completed_count > 0:
1561
+ batch.status = BatchStatus.PARTIAL
1562
+ event_type = events.EventType.BATCH_PARTIAL
1563
+ else:
1564
+ # Nothing executed (e.g., cancelled before start)
1565
+ batch.status = BatchStatus.CANCELLED
1566
+ event_type = events.EventType.BATCH_CANCELLED
1567
+
1568
+ batch.completed_at = _utc_now()
1569
+ _save_batch(workspace, batch)
1570
+
1571
+ # Emit batch completion event
1572
+ events.emit_for_workspace(
1573
+ workspace,
1574
+ event_type,
1575
+ {
1576
+ "batch_id": batch.id,
1577
+ "completed": completed_count,
1578
+ "failed": failed_count,
1579
+ "blocked": blocked_count,
1580
+ "total": total,
1581
+ },
1582
+ print_event=True,
1583
+ )
1584
+ _dispatch_batch_completed_webhook(workspace, event_type, batch.id, completed_count)
1585
+
1586
+ # Stop reconciliation thread
1587
+ reconcile_stop.set()
1588
+
1589
+ # Print summary
1590
+ print(f"\nBatch {batch.status.value.lower()}: {completed_count}/{total} tasks completed")
1591
+ if failed_count > 0:
1592
+ print(f" Failed: {failed_count}")
1593
+ if blocked_count > 0:
1594
+ print(f" Blocked: {blocked_count}")
1595
+ finally:
1596
+ if config_watcher is not None:
1597
+ config_watcher.stop()
1598
+
1599
+
1600
+ def _execute_parallel(
1601
+ workspace: Workspace,
1602
+ batch: BatchRun,
1603
+ on_event: Optional[Callable[[str, dict], None]] = None,
1604
+ ) -> None:
1605
+ """Execute tasks in parallel using dependency-aware groups.
1606
+
1607
+ Creates an execution plan from task dependencies and executes each group
1608
+ in sequence. Tasks within a group run in parallel, up to max_parallel.
1609
+
1610
+ Args:
1611
+ workspace: Target workspace
1612
+ batch: BatchRun to execute
1613
+ on_event: Optional callback for batch events
1614
+
1615
+ Raises:
1616
+ CycleDetectedError: If circular dependencies are detected
1617
+ """
1618
+ # Clean up orphaned worktrees from crashed workers on previous runs
1619
+ from codeframe.core.sandbox.context import IsolationLevel as _IL
1620
+ if batch.isolation == _IL.WORKTREE.value:
1621
+ from codeframe.core.worktrees import WorktreeRegistry
1622
+ WorktreeRegistry().cleanup_stale(workspace.repo_path)
1623
+
1624
+ # Create execution plan based on dependencies
1625
+ plan = create_execution_plan(workspace, batch.task_ids)
1626
+
1627
+ print(f"\nExecution plan: {plan.num_groups} groups, {plan.total_tasks} tasks")
1628
+ if plan.can_run_parallel():
1629
+ print(f"Parallelizable groups found - using max {batch.max_parallel} workers")
1630
+ else:
1631
+ print("All tasks are sequential (chain dependencies)")
1632
+
1633
+ # Start reconciliation thread for continuous state checking
1634
+ from codeframe.core.config import load_environment_config as _load_env_config
1635
+ _env_config_p = _load_env_config(workspace.repo_path)
1636
+ _interval_p = _env_config_p.reconciliation_interval_seconds if _env_config_p else 30
1637
+ _reconcile_stop_p = _start_reconciliation_thread(workspace, batch, interval_seconds=_interval_p)
1638
+
1639
+ # Start config file watcher (optional — failure must not break batch)
1640
+ config_watcher_p = None
1641
+ reload_state_p = None
1642
+ _last_seen_reload_p = datetime.now(timezone.utc)
1643
+ try:
1644
+ from codeframe.core.config_watcher import ConfigFileWatcher as _CFW
1645
+ from codeframe.core.agents_config import load_preferences as _load_prefs
1646
+
1647
+ config_watcher_p = _CFW(Path(workspace.repo_path))
1648
+ reload_state_p = config_watcher_p.start(_load_prefs(Path(workspace.repo_path)))
1649
+ except Exception:
1650
+ logger.debug("Config watcher failed to start in parallel execution", exc_info=True)
1651
+
1652
+ completed_count = 0
1653
+ failed_count = 0
1654
+ blocked_count = 0
1655
+ task_index = 0 # Global task counter for progress display
1656
+
1657
+ try:
1658
+ for group_idx, group in enumerate(plan.groups):
1659
+ # Check if batch was cancelled
1660
+ current_batch = get_batch(workspace, batch.id)
1661
+ if current_batch and current_batch.status == BatchStatus.CANCELLED:
1662
+ break
1663
+
1664
+ # Check for config reloads between groups
1665
+ if reload_state_p is not None:
1666
+ _last_seen_reload_p = _apply_pending_config_reload(
1667
+ batch, workspace, reload_state_p, _last_seen_reload_p
1668
+ )
1669
+
1670
+ # Check if any previous failure should stop execution
1671
+ if batch.on_failure == OnFailure.STOP and failed_count > 0:
1672
+ print("\nStopping batch due to --on-failure=stop")
1673
+ break
1674
+
1675
+ group_size = len(group)
1676
+ print(f"\n{'─'*60}")
1677
+ print(f"Group {group_idx + 1}/{plan.num_groups}: {group_size} task(s)")
1678
+
1679
+ if group_size == 1:
1680
+ # Single task - run directly
1681
+ task_id = group[0]
1682
+ task_index += 1
1683
+ result = _execute_single_task(
1684
+ workspace, batch, task_id, task_index, len(batch.task_ids), on_event
1685
+ )
1686
+ if result == RunStatus.COMPLETED.value:
1687
+ completed_count += 1
1688
+ elif result == RunStatus.BLOCKED.value:
1689
+ blocked_count += 1
1690
+ else:
1691
+ failed_count += 1
1692
+ else:
1693
+ # Multiple tasks - run in parallel (use per-status limits if configured)
1694
+ if batch.concurrency.by_status:
1695
+ group_statuses = []
1696
+ for tid in group:
1697
+ t = tasks.get(workspace, tid)
1698
+ if t:
1699
+ group_statuses.append(t.status.value)
1700
+ effective_workers = batch.concurrency.effective_workers(
1701
+ statuses=group_statuses, group_size=group_size, global_running=0,
1702
+ )
1703
+ else:
1704
+ effective_workers = min(group_size, batch.max_parallel)
1705
+ print(f"Running {group_size} tasks with {effective_workers} workers")
1706
+
1707
+ # Execute group in parallel
1708
+ results = _execute_group_parallel(
1709
+ workspace, batch, group, task_index, len(batch.task_ids),
1710
+ effective_workers, on_event
1711
+ )
1712
+
1713
+ # Process results
1714
+ for task_id, result_status in results.items():
1715
+ task_index += 1
1716
+ if result_status == RunStatus.COMPLETED.value:
1717
+ completed_count += 1
1718
+ elif result_status == RunStatus.BLOCKED.value:
1719
+ blocked_count += 1
1720
+ else:
1721
+ failed_count += 1
1722
+
1723
+ # Check stop on failure within parallel group
1724
+ if batch.on_failure == OnFailure.STOP and failed_count > 0:
1725
+ # Can't stop mid-group, but will stop after group completes
1726
+ pass
1727
+
1728
+ # Determine final batch status
1729
+ total = len(batch.task_ids)
1730
+ completed_count + failed_count + blocked_count
1731
+
1732
+ if completed_count == total:
1733
+ batch.status = BatchStatus.COMPLETED
1734
+ event_type = events.EventType.BATCH_COMPLETED
1735
+
1736
+ # Run batch-level validation (full gate sweep)
1737
+ validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
1738
+
1739
+ if not validation_passed:
1740
+ # Gates failed - change status to PARTIAL (tasks done, integration broken)
1741
+ batch.status = BatchStatus.PARTIAL
1742
+ event_type = events.EventType.BATCH_PARTIAL
1743
+ print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
1744
+ print(f"Validation error: {validation_error}")
1745
+
1746
+ elif completed_count == 0 and (failed_count > 0 or blocked_count > 0):
1747
+ batch.status = BatchStatus.FAILED
1748
+ event_type = events.EventType.BATCH_FAILED
1749
+ elif completed_count > 0:
1750
+ batch.status = BatchStatus.PARTIAL
1751
+ event_type = events.EventType.BATCH_PARTIAL
1752
+ else:
1753
+ batch.status = BatchStatus.CANCELLED
1754
+ event_type = events.EventType.BATCH_CANCELLED
1755
+
1756
+ batch.completed_at = _utc_now()
1757
+ _save_batch(workspace, batch)
1758
+
1759
+ # Emit batch completion event
1760
+ events.emit_for_workspace(
1761
+ workspace,
1762
+ event_type,
1763
+ {
1764
+ "batch_id": batch.id,
1765
+ "completed": completed_count,
1766
+ "failed": failed_count,
1767
+ "blocked": blocked_count,
1768
+ "total": total,
1769
+ "strategy": "parallel",
1770
+ "groups": plan.num_groups,
1771
+ },
1772
+ print_event=True,
1773
+ )
1774
+ _dispatch_batch_completed_webhook(workspace, event_type, batch.id, completed_count)
1775
+
1776
+ # Stop reconciliation thread
1777
+ _reconcile_stop_p.set()
1778
+
1779
+ # Print summary
1780
+ print(f"\nBatch {batch.status.value.lower()}: {completed_count}/{total} tasks completed")
1781
+ print(f" Execution: {plan.num_groups} groups (parallel strategy)")
1782
+ if failed_count > 0:
1783
+ print(f" Failed: {failed_count}")
1784
+ if blocked_count > 0:
1785
+ print(f" Blocked: {blocked_count}")
1786
+ finally:
1787
+ if config_watcher_p is not None:
1788
+ config_watcher_p.stop()
1789
+
1790
+
1791
+ def _start_reconciliation_thread(
1792
+ workspace: Workspace,
1793
+ batch: BatchRun,
1794
+ interval_seconds: int = 30,
1795
+ github_checker: Optional[Callable] = None,
1796
+ ) -> threading.Event:
1797
+ """Start a daemon thread that periodically reconciles batch state.
1798
+
1799
+ The thread checks all active tasks for external state changes every
1800
+ ``interval_seconds`` and applies adjustments (skip completed tasks,
1801
+ re-queue unblocked tasks).
1802
+
1803
+ Returns a threading.Event that can be set to stop the thread.
1804
+ """
1805
+ from codeframe.core.reconciliation import ReconciliationEngine
1806
+
1807
+ stop_event = threading.Event()
1808
+ engine = ReconciliationEngine(workspace, github_checker=github_checker)
1809
+
1810
+ def _loop() -> None:
1811
+ while not stop_event.wait(timeout=interval_seconds):
1812
+ try:
1813
+ # Get currently active task IDs from the batch
1814
+ active_ids = [
1815
+ tid for tid in batch.task_ids
1816
+ if batch.results.get(tid) is None
1817
+ or batch.results.get(tid) == "RUNNING"
1818
+ ]
1819
+ if not active_ids:
1820
+ continue
1821
+
1822
+ result = engine.check_all_active(active_ids)
1823
+ if result.changes_detected:
1824
+ with _active_processes_lock:
1825
+ procs = _active_processes.get(batch.id, {})
1826
+ engine.apply_changes(result, batch, procs)
1827
+
1828
+ # Emit events for changes
1829
+ for tid in result.tasks_skipped:
1830
+ events.emit_for_workspace(
1831
+ workspace,
1832
+ events.EventType.RECONCILIATION_TASK_SKIPPED,
1833
+ {"batch_id": batch.id, "task_id": tid},
1834
+ )
1835
+ for tid in result.tasks_requeued:
1836
+ events.emit_for_workspace(
1837
+ workspace,
1838
+ events.EventType.RECONCILIATION_TASK_REQUEUED,
1839
+ {"batch_id": batch.id, "task_id": tid},
1840
+ )
1841
+ if result.errors:
1842
+ for err in result.errors:
1843
+ events.emit_for_workspace(
1844
+ workspace,
1845
+ events.EventType.RECONCILIATION_ERROR,
1846
+ {"batch_id": batch.id, "error": err},
1847
+ )
1848
+ except Exception as exc:
1849
+ logger.warning("Reconciliation loop error: %s", exc)
1850
+
1851
+ thread = threading.Thread(target=_loop, daemon=True, name=f"reconcile-{batch.id[:8]}")
1852
+ thread.start()
1853
+ return stop_event
1854
+
1855
+
1856
+ def _execute_single_task(
1857
+ workspace: Workspace,
1858
+ batch: BatchRun,
1859
+ task_id: str,
1860
+ position: int,
1861
+ total: int,
1862
+ on_event: Optional[Callable[[str, dict], None]] = None,
1863
+ ) -> str:
1864
+ """Execute a single task and update batch results.
1865
+
1866
+ Helper for parallel execution that handles one task.
1867
+
1868
+ Returns:
1869
+ RunStatus value string
1870
+ """
1871
+ # Emit task queued event
1872
+ events.emit_for_workspace(
1873
+ workspace,
1874
+ events.EventType.BATCH_TASK_QUEUED,
1875
+ {"batch_id": batch.id, "task_id": task_id, "position": position},
1876
+ print_event=True,
1877
+ )
1878
+
1879
+ # Get task info for display
1880
+ task = tasks.get(workspace, task_id)
1881
+ task_title = task.title if task else task_id
1882
+
1883
+ print(f"\n[{position}/{total}] Starting task {task_id}: {task_title}")
1884
+
1885
+ # Emit task started event
1886
+ events.emit_for_workspace(
1887
+ workspace,
1888
+ events.EventType.BATCH_TASK_STARTED,
1889
+ {"batch_id": batch.id, "task_id": task_id},
1890
+ print_event=True,
1891
+ )
1892
+
1893
+ if on_event:
1894
+ on_event("batch_task_started", {"task_id": task_id, "position": position})
1895
+
1896
+ # Create execution context (handles isolation level; NONE is a no-op)
1897
+ from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
1898
+ exec_ctx = create_execution_context(
1899
+ task_id, IsolationLevel(batch.isolation), workspace.repo_path
1900
+ )
1901
+
1902
+ try:
1903
+ # Execute task via subprocess
1904
+ result_status = _execute_task_subprocess(
1905
+ workspace, task_id, batch.id,
1906
+ engine=batch.engine,
1907
+ stall_timeout_s=batch.stall_timeout_s,
1908
+ stall_action=batch.stall_action,
1909
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1910
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1911
+ )
1912
+
1913
+ # If task is BLOCKED, try supervisor resolution
1914
+ if result_status == RunStatus.BLOCKED.value:
1915
+ supervisor = get_supervisor(workspace)
1916
+ if supervisor.try_resolve_blocked_task(task_id):
1917
+ # Supervisor resolved the blocker - retry the task
1918
+ print(" [Supervisor] Retrying task after auto-resolution...")
1919
+ result_status = _execute_task_subprocess(
1920
+ workspace, task_id, batch.id,
1921
+ engine=batch.engine,
1922
+ stall_timeout_s=batch.stall_timeout_s,
1923
+ stall_action=batch.stall_action,
1924
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
1925
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
1926
+ )
1927
+ finally:
1928
+ exec_ctx.cleanup()
1929
+
1930
+ # Record result
1931
+ batch.results[task_id] = result_status
1932
+ _save_batch(workspace, batch)
1933
+
1934
+ # Emit appropriate event based on result
1935
+ if result_status == RunStatus.COMPLETED.value:
1936
+ events.emit_for_workspace(
1937
+ workspace,
1938
+ events.EventType.BATCH_TASK_COMPLETED,
1939
+ {"batch_id": batch.id, "task_id": task_id},
1940
+ print_event=True,
1941
+ )
1942
+ print(" ✓ Completed")
1943
+ elif result_status == RunStatus.BLOCKED.value:
1944
+ events.emit_for_workspace(
1945
+ workspace,
1946
+ events.EventType.BATCH_TASK_BLOCKED,
1947
+ {"batch_id": batch.id, "task_id": task_id},
1948
+ print_event=True,
1949
+ )
1950
+ print(" ⊘ Blocked (requires human input)")
1951
+ else:
1952
+ events.emit_for_workspace(
1953
+ workspace,
1954
+ events.EventType.BATCH_TASK_FAILED,
1955
+ {"batch_id": batch.id, "task_id": task_id, "status": result_status},
1956
+ print_event=True,
1957
+ )
1958
+ print(f" ✗ Failed: {result_status}")
1959
+
1960
+ if on_event:
1961
+ on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
1962
+
1963
+ return result_status
1964
+
1965
+
1966
+ def _execute_group_parallel(
1967
+ workspace: Workspace,
1968
+ batch: BatchRun,
1969
+ group: list[str],
1970
+ start_index: int,
1971
+ total: int,
1972
+ max_workers: int,
1973
+ on_event: Optional[Callable[[str, dict], None]] = None,
1974
+ ) -> dict[str, str]:
1975
+ """Execute a group of tasks in parallel.
1976
+
1977
+ Args:
1978
+ workspace: Target workspace
1979
+ batch: BatchRun being executed
1980
+ group: List of task IDs to execute concurrently
1981
+ start_index: Starting task index for progress display
1982
+ total: Total tasks in batch
1983
+ max_workers: Maximum concurrent workers
1984
+ on_event: Optional callback for events
1985
+
1986
+ Returns:
1987
+ Dict mapping task_id -> RunStatus value
1988
+ """
1989
+ results: dict[str, str] = {}
1990
+
1991
+ # Show which tasks are starting
1992
+ for i, task_id in enumerate(group):
1993
+ task = tasks.get(workspace, task_id)
1994
+ task_title = task.title if task else task_id
1995
+ print(f" [{start_index + i + 1}/{total}] Queued: {task_id}: {task_title}")
1996
+
1997
+ # Emit task queued event
1998
+ events.emit_for_workspace(
1999
+ workspace,
2000
+ events.EventType.BATCH_TASK_QUEUED,
2001
+ {"batch_id": batch.id, "task_id": task_id, "position": start_index + i + 1, "parallel": True},
2002
+ print_event=True,
2003
+ )
2004
+
2005
+ def execute_task(task_id: str) -> tuple[str, str]:
2006
+ """Execute a single task and return (task_id, status)."""
2007
+ # Emit task started event
2008
+ events.emit_for_workspace(
2009
+ workspace,
2010
+ events.EventType.BATCH_TASK_STARTED,
2011
+ {"batch_id": batch.id, "task_id": task_id, "parallel": True},
2012
+ print_event=True,
2013
+ )
2014
+
2015
+ if on_event:
2016
+ on_event("batch_task_started", {"task_id": task_id, "parallel": True})
2017
+
2018
+ # Create execution context for this task
2019
+ from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
2020
+ exec_ctx = create_execution_context(
2021
+ task_id, IsolationLevel(batch.isolation), workspace.repo_path
2022
+ )
2023
+
2024
+ try:
2025
+ # Execute via subprocess
2026
+ result_status = _execute_task_subprocess(
2027
+ workspace, task_id, batch.id,
2028
+ engine=batch.engine,
2029
+ stall_timeout_s=batch.stall_timeout_s,
2030
+ stall_action=batch.stall_action,
2031
+ worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
2032
+ llm_provider=batch.llm_provider, llm_model=batch.llm_model,
2033
+ )
2034
+ finally:
2035
+ exec_ctx.cleanup()
2036
+
2037
+ # Record result (thread-safe due to GIL for simple dict operations)
2038
+ batch.results[task_id] = result_status
2039
+ _save_batch(workspace, batch)
2040
+
2041
+ return task_id, result_status
2042
+
2043
+ # Execute tasks in parallel using thread pool
2044
+ with ThreadPoolExecutor(max_workers=max_workers) as executor:
2045
+ futures = {executor.submit(execute_task, tid): tid for tid in group}
2046
+
2047
+ for future in as_completed(futures):
2048
+ task_id, result_status = future.result()
2049
+ results[task_id] = result_status
2050
+
2051
+ # Get task info for display
2052
+ task = tasks.get(workspace, task_id)
2053
+ task_title = task.title if task else task_id
2054
+
2055
+ # Emit appropriate event and print result
2056
+ if result_status == RunStatus.COMPLETED.value:
2057
+ events.emit_for_workspace(
2058
+ workspace,
2059
+ events.EventType.BATCH_TASK_COMPLETED,
2060
+ {"batch_id": batch.id, "task_id": task_id, "parallel": True},
2061
+ print_event=True,
2062
+ )
2063
+ print(f" ✓ {task_id}: Completed")
2064
+ elif result_status == RunStatus.BLOCKED.value:
2065
+ events.emit_for_workspace(
2066
+ workspace,
2067
+ events.EventType.BATCH_TASK_BLOCKED,
2068
+ {"batch_id": batch.id, "task_id": task_id, "parallel": True},
2069
+ print_event=True,
2070
+ )
2071
+ print(f" ⊘ {task_id}: Blocked")
2072
+ else:
2073
+ events.emit_for_workspace(
2074
+ workspace,
2075
+ events.EventType.BATCH_TASK_FAILED,
2076
+ {"batch_id": batch.id, "task_id": task_id, "status": result_status, "parallel": True},
2077
+ print_event=True,
2078
+ )
2079
+ print(f" ✗ {task_id}: Failed ({result_status})")
2080
+
2081
+ if on_event:
2082
+ on_event("batch_task_completed", {"task_id": task_id, "status": result_status, "parallel": True})
2083
+
2084
+ return results
2085
+
2086
+
2087
+ def _execute_task_subprocess(
2088
+ workspace: Workspace,
2089
+ task_id: str,
2090
+ batch_id: Optional[str] = None,
2091
+ engine: str = "react",
2092
+ stall_timeout_s: int = 300,
2093
+ stall_action: str = "blocker",
2094
+ worktree_path: Optional[Path] = None,
2095
+ cloud_timeout_minutes: int = 30,
2096
+ llm_provider: Optional[str] = None,
2097
+ llm_model: Optional[str] = None,
2098
+ ) -> str:
2099
+ """Execute a single task via subprocess.
2100
+
2101
+ Runs `cf work start <task_id> --execute --engine <engine>` as a subprocess.
2102
+
2103
+ Args:
2104
+ workspace: Target workspace
2105
+ task_id: Task to execute
2106
+ batch_id: Optional batch ID for process tracking (enables force stop)
2107
+ engine: Agent engine to use ("plan", "react", "cloud", etc.)
2108
+ stall_timeout_s: Stall detection timeout in seconds (0 = disabled)
2109
+ stall_action: Recovery action on stall ("blocker", "retry", or "fail")
2110
+ cloud_timeout_minutes: Sandbox timeout for cloud engine (1-60)
2111
+
2112
+ Returns:
2113
+ RunStatus value string (COMPLETED, FAILED, BLOCKED)
2114
+ """
2115
+ # Build command
2116
+ cmd = [
2117
+ sys.executable, "-m", "codeframe.cli.app",
2118
+ "work", "start", task_id, "--execute",
2119
+ "--engine", engine,
2120
+ "--stall-timeout", str(stall_timeout_s),
2121
+ "--stall-action", stall_action,
2122
+ ]
2123
+ if engine == "cloud":
2124
+ cmd += ["--cloud-timeout", str(cloud_timeout_minutes)]
2125
+ if llm_provider:
2126
+ cmd += ["--llm-provider", llm_provider]
2127
+ if llm_model:
2128
+ cmd += ["--llm-model", llm_model]
2129
+
2130
+ process = None
2131
+ try:
2132
+ # Use Popen instead of run for process tracking
2133
+ process = subprocess.Popen(
2134
+ cmd,
2135
+ cwd=str(worktree_path) if worktree_path else workspace.repo_path,
2136
+ stdout=None, # Let output flow to terminal
2137
+ stderr=None,
2138
+ text=True,
2139
+ )
2140
+
2141
+ # Track process if batch_id provided (thread-safe)
2142
+ if batch_id:
2143
+ with _active_processes_lock:
2144
+ if batch_id not in _active_processes:
2145
+ _active_processes[batch_id] = {}
2146
+ _active_processes[batch_id][task_id] = process
2147
+
2148
+ # Wait for completion (outside lock to avoid blocking)
2149
+ returncode = process.wait()
2150
+
2151
+ # Untrack process (thread-safe)
2152
+ if batch_id:
2153
+ with _active_processes_lock:
2154
+ if batch_id in _active_processes:
2155
+ _active_processes[batch_id].pop(task_id, None)
2156
+ if not _active_processes[batch_id]:
2157
+ _active_processes.pop(batch_id, None)
2158
+
2159
+ # Check the run status from database
2160
+ # The subprocess should have updated the run record
2161
+ run = get_active_run(workspace, task_id)
2162
+ if run:
2163
+ return run.status.value
2164
+
2165
+ # If no active run, check if there's a recent completed/failed run
2166
+ # by listing runs for this task
2167
+ from codeframe.core.runtime import list_runs
2168
+ runs = list_runs(workspace, task_id=task_id, limit=1)
2169
+ if runs:
2170
+ return runs[0].status.value
2171
+
2172
+ # Fallback based on subprocess exit code
2173
+ if returncode == 0:
2174
+ return RunStatus.COMPLETED.value
2175
+ else:
2176
+ return RunStatus.FAILED.value
2177
+
2178
+ except Exception as e:
2179
+ print(f" Error executing task: {e}")
2180
+ # Cleanup on exception (thread-safe)
2181
+ if batch_id:
2182
+ with _active_processes_lock:
2183
+ if batch_id in _active_processes:
2184
+ _active_processes[batch_id].pop(task_id, None)
2185
+ if not _active_processes[batch_id]:
2186
+ _active_processes.pop(batch_id, None)
2187
+ return RunStatus.FAILED.value
2188
+
2189
+
2190
+ def _save_batch(workspace: Workspace, batch: BatchRun) -> None:
2191
+ """Save or update a batch record in the database.
2192
+
2193
+ Uses a thread lock to prevent SQLite "database is locked" errors
2194
+ when multiple workers try to update batch status concurrently.
2195
+ """
2196
+ task_ids_json = json.dumps(batch.task_ids)
2197
+ results_json = json.dumps(batch.results) if batch.results else None
2198
+ completed_at = batch.completed_at.isoformat() if batch.completed_at else None
2199
+
2200
+ with _batch_db_lock:
2201
+ conn = get_db_connection(workspace)
2202
+ try:
2203
+ cursor = conn.cursor()
2204
+ # Ensure isolation column exists (migration for existing databases)
2205
+ try:
2206
+ cursor.execute(
2207
+ "ALTER TABLE batch_runs ADD COLUMN isolation TEXT DEFAULT 'none'"
2208
+ )
2209
+ conn.commit()
2210
+ except Exception:
2211
+ pass # Column already exists
2212
+
2213
+ cursor.execute(
2214
+ """
2215
+ INSERT OR REPLACE INTO batch_runs
2216
+ (id, workspace_id, task_ids, status, strategy, max_parallel, on_failure,
2217
+ started_at, completed_at, results, engine, isolation)
2218
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
2219
+ """,
2220
+ (
2221
+ batch.id,
2222
+ batch.workspace_id,
2223
+ task_ids_json,
2224
+ batch.status.value,
2225
+ batch.strategy,
2226
+ batch.max_parallel,
2227
+ batch.on_failure.value,
2228
+ batch.started_at.isoformat(),
2229
+ completed_at,
2230
+ results_json,
2231
+ batch.engine,
2232
+ batch.isolation,
2233
+ ),
2234
+ )
2235
+ conn.commit()
2236
+ finally:
2237
+ conn.close()
2238
+
2239
+
2240
+ def _row_to_batch(row: tuple) -> BatchRun:
2241
+ """Convert a database row to a BatchRun object."""
2242
+ return BatchRun(
2243
+ id=row[0],
2244
+ workspace_id=row[1],
2245
+ task_ids=json.loads(row[2]),
2246
+ status=BatchStatus(row[3]),
2247
+ strategy=row[4],
2248
+ max_parallel=row[5],
2249
+ on_failure=OnFailure(row[6]),
2250
+ started_at=datetime.fromisoformat(row[7]),
2251
+ completed_at=datetime.fromisoformat(row[8]) if row[8] else None,
2252
+ results=json.loads(row[9]) if row[9] else {},
2253
+ engine=row[10] if len(row) > 10 and row[10] else "plan",
2254
+ isolation=row[11] if len(row) > 11 and row[11] else "none",
2255
+ )