codeframe-ai 0.9.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- codeframe/__init__.py +11 -0
- codeframe/__main__.py +20 -0
- codeframe/adapters/__init__.py +5 -0
- codeframe/adapters/e2b/__init__.py +13 -0
- codeframe/adapters/e2b/adapter.py +342 -0
- codeframe/adapters/e2b/budget.py +71 -0
- codeframe/adapters/e2b/credential_scanner.py +134 -0
- codeframe/adapters/llm/__init__.py +92 -0
- codeframe/adapters/llm/anthropic.py +414 -0
- codeframe/adapters/llm/base.py +444 -0
- codeframe/adapters/llm/mock.py +281 -0
- codeframe/adapters/llm/openai.py +483 -0
- codeframe/agents/__init__.py +8 -0
- codeframe/agents/dependency_resolver.py +714 -0
- codeframe/auth/__init__.py +16 -0
- codeframe/auth/api_key_router.py +238 -0
- codeframe/auth/api_keys.py +156 -0
- codeframe/auth/dependencies.py +358 -0
- codeframe/auth/manager.py +178 -0
- codeframe/auth/models.py +30 -0
- codeframe/auth/router.py +93 -0
- codeframe/auth/schemas.py +15 -0
- codeframe/auth/scopes.py +53 -0
- codeframe/cli/__init__.py +12 -0
- codeframe/cli/__main__.py +20 -0
- codeframe/cli/api_client.py +275 -0
- codeframe/cli/app.py +5688 -0
- codeframe/cli/auth.py +122 -0
- codeframe/cli/auth_commands.py +958 -0
- codeframe/cli/commands/__init__.py +5 -0
- codeframe/cli/config_commands.py +79 -0
- codeframe/cli/dashboard_commands.py +67 -0
- codeframe/cli/engines_commands.py +205 -0
- codeframe/cli/env_commands.py +409 -0
- codeframe/cli/helpers.py +56 -0
- codeframe/cli/hooks_commands.py +208 -0
- codeframe/cli/import_commands.py +129 -0
- codeframe/cli/pr_commands.py +549 -0
- codeframe/cli/proof_commands.py +415 -0
- codeframe/cli/stats_commands.py +311 -0
- codeframe/cli/telemetry_runtime.py +153 -0
- codeframe/cli/validators.py +123 -0
- codeframe/config/rate_limits.py +165 -0
- codeframe/core/__init__.py +15 -0
- codeframe/core/adapters/__init__.py +43 -0
- codeframe/core/adapters/agent_adapter.py +114 -0
- codeframe/core/adapters/builtin.py +326 -0
- codeframe/core/adapters/claude_code.py +62 -0
- codeframe/core/adapters/codex.py +393 -0
- codeframe/core/adapters/git_utils.py +40 -0
- codeframe/core/adapters/kilocode.py +126 -0
- codeframe/core/adapters/opencode.py +48 -0
- codeframe/core/adapters/streaming_chat.py +483 -0
- codeframe/core/adapters/subprocess_adapter.py +213 -0
- codeframe/core/adapters/verification_wrapper.py +269 -0
- codeframe/core/agent.py +2183 -0
- codeframe/core/agents_config.py +569 -0
- codeframe/core/api_key_service.py +211 -0
- codeframe/core/artifacts.py +428 -0
- codeframe/core/blocker_detection.py +218 -0
- codeframe/core/blockers.py +433 -0
- codeframe/core/checkpoints.py +481 -0
- codeframe/core/conductor.py +2255 -0
- codeframe/core/config.py +827 -0
- codeframe/core/config_watcher.py +268 -0
- codeframe/core/context.py +542 -0
- codeframe/core/context_packager.py +234 -0
- codeframe/core/credentials.py +735 -0
- codeframe/core/dependency_analyzer.py +229 -0
- codeframe/core/dependency_graph.py +290 -0
- codeframe/core/diagnostic_agent.py +712 -0
- codeframe/core/diagnostics.py +616 -0
- codeframe/core/editor.py +556 -0
- codeframe/core/engine_registry.py +256 -0
- codeframe/core/engine_stats.py +231 -0
- codeframe/core/environment.py +697 -0
- codeframe/core/events.py +375 -0
- codeframe/core/executor.py +1005 -0
- codeframe/core/fix_tracker.py +480 -0
- codeframe/core/gates.py +1322 -0
- codeframe/core/git.py +477 -0
- codeframe/core/github_connect_service.py +178 -0
- codeframe/core/github_integration_config.py +118 -0
- codeframe/core/github_issues_service.py +449 -0
- codeframe/core/hooks.py +184 -0
- codeframe/core/importers/__init__.py +1 -0
- codeframe/core/importers/ralph.py +540 -0
- codeframe/core/installer.py +650 -0
- codeframe/core/models.py +1026 -0
- codeframe/core/notifications_config.py +183 -0
- codeframe/core/planner.py +437 -0
- codeframe/core/prd.py +670 -0
- codeframe/core/prd_discovery.py +1118 -0
- codeframe/core/prd_stress_test.py +499 -0
- codeframe/core/progress.py +126 -0
- codeframe/core/proof/__init__.py +34 -0
- codeframe/core/proof/capture.py +79 -0
- codeframe/core/proof/evidence.py +56 -0
- codeframe/core/proof/ledger.py +574 -0
- codeframe/core/proof/models.py +162 -0
- codeframe/core/proof/obligations.py +103 -0
- codeframe/core/proof/runner.py +233 -0
- codeframe/core/proof/scope.py +81 -0
- codeframe/core/proof/stubs.py +156 -0
- codeframe/core/quick_fixes.py +558 -0
- codeframe/core/react_agent.py +1650 -0
- codeframe/core/reconciliation.py +183 -0
- codeframe/core/replay.py +788 -0
- codeframe/core/review.py +285 -0
- codeframe/core/runtime.py +1134 -0
- codeframe/core/sandbox/__init__.py +27 -0
- codeframe/core/sandbox/context.py +98 -0
- codeframe/core/sandbox/worktree.py +20 -0
- codeframe/core/schedule.py +396 -0
- codeframe/core/stall_detector.py +71 -0
- codeframe/core/stall_monitor.py +134 -0
- codeframe/core/state_machine.py +121 -0
- codeframe/core/streaming.py +502 -0
- codeframe/core/task_tree.py +400 -0
- codeframe/core/tasks.py +1022 -0
- codeframe/core/telemetry.py +232 -0
- codeframe/core/templates.py +221 -0
- codeframe/core/tools.py +942 -0
- codeframe/core/workspace.py +887 -0
- codeframe/core/worktrees.py +276 -0
- codeframe/git/__init__.py +5 -0
- codeframe/git/github_integration.py +505 -0
- codeframe/lib/__init__.py +0 -0
- codeframe/lib/audit_logger.py +248 -0
- codeframe/lib/metrics_tracker.py +800 -0
- codeframe/lib/quality/__init__.py +7 -0
- codeframe/lib/quality/complexity_analyzer.py +316 -0
- codeframe/lib/quality/owasp_patterns.py +284 -0
- codeframe/lib/quality/security_scanner.py +250 -0
- codeframe/lib/rate_limiter.py +312 -0
- codeframe/notifications/__init__.py +0 -0
- codeframe/notifications/webhook.py +380 -0
- codeframe/planning/__init__.py +30 -0
- codeframe/planning/issue_generator.py +219 -0
- codeframe/planning/prd_template_functions.py +137 -0
- codeframe/planning/prd_templates.py +975 -0
- codeframe/planning/task_scheduler.py +511 -0
- codeframe/planning/task_templates.py +533 -0
- codeframe/platform_store/__init__.py +5 -0
- codeframe/platform_store/database.py +277 -0
- codeframe/platform_store/repositories/__init__.py +24 -0
- codeframe/platform_store/repositories/api_key_repository.py +245 -0
- codeframe/platform_store/repositories/audit_repository.py +67 -0
- codeframe/platform_store/repositories/base.py +295 -0
- codeframe/platform_store/repositories/interactive_sessions.py +165 -0
- codeframe/platform_store/repositories/token_repository.py +598 -0
- codeframe/platform_store/repositories/workspace_registry_repository.py +175 -0
- codeframe/platform_store/schema_manager.py +321 -0
- codeframe/templates/AGENTS.md.default +94 -0
- codeframe/tui/__init__.py +5 -0
- codeframe/tui/app.py +256 -0
- codeframe/tui/data_service.py +103 -0
- codeframe/ui/__init__.py +0 -0
- codeframe/ui/dependencies.py +103 -0
- codeframe/ui/models.py +999 -0
- codeframe/ui/response_models.py +201 -0
- codeframe/ui/routers/__init__.py +5 -0
- codeframe/ui/routers/_helpers.py +29 -0
- codeframe/ui/routers/batches_v2.py +315 -0
- codeframe/ui/routers/blockers_v2.py +320 -0
- codeframe/ui/routers/checkpoints_v2.py +310 -0
- codeframe/ui/routers/costs_v2.py +322 -0
- codeframe/ui/routers/diagnose_v2.py +225 -0
- codeframe/ui/routers/discovery_v2.py +417 -0
- codeframe/ui/routers/environment_v2.py +284 -0
- codeframe/ui/routers/events_v2.py +75 -0
- codeframe/ui/routers/gates_v2.py +166 -0
- codeframe/ui/routers/git_v2.py +284 -0
- codeframe/ui/routers/github_integrations_v2.py +532 -0
- codeframe/ui/routers/interactive_sessions_v2.py +238 -0
- codeframe/ui/routers/pr_v2.py +709 -0
- codeframe/ui/routers/prd_v2.py +695 -0
- codeframe/ui/routers/proof_v2.py +755 -0
- codeframe/ui/routers/review_v2.py +360 -0
- codeframe/ui/routers/schedule_v2.py +214 -0
- codeframe/ui/routers/session_chat_ws.py +354 -0
- codeframe/ui/routers/settings_v2.py +562 -0
- codeframe/ui/routers/streaming_v2.py +155 -0
- codeframe/ui/routers/tasks_v2.py +1098 -0
- codeframe/ui/routers/templates_v2.py +232 -0
- codeframe/ui/routers/terminal_ws.py +267 -0
- codeframe/ui/routers/workspace_v2.py +527 -0
- codeframe/ui/server.py +568 -0
- codeframe/ui/shared.py +241 -0
- codeframe/workspace/__init__.py +5 -0
- codeframe/workspace/manager.py +249 -0
- codeframe_ai-0.9.0.dist-info/METADATA +517 -0
- codeframe_ai-0.9.0.dist-info/RECORD +197 -0
- codeframe_ai-0.9.0.dist-info/WHEEL +5 -0
- codeframe_ai-0.9.0.dist-info/entry_points.txt +3 -0
- codeframe_ai-0.9.0.dist-info/licenses/LICENSE +661 -0
- codeframe_ai-0.9.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2255 @@
|
|
|
1
|
+
"""Batch execution conductor for CodeFRAME v2.
|
|
2
|
+
|
|
3
|
+
Orchestrates execution of multiple tasks, managing parallelization
|
|
4
|
+
and coordinating results.
|
|
5
|
+
|
|
6
|
+
This module is headless - no FastAPI or HTTP dependencies.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import logging
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import threading
|
|
16
|
+
import uuid
|
|
17
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
18
|
+
from dataclasses import dataclass, field
|
|
19
|
+
from datetime import datetime, timezone
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from enum import Enum
|
|
22
|
+
from typing import TYPE_CHECKING, Callable, Optional
|
|
23
|
+
|
|
24
|
+
if TYPE_CHECKING:
|
|
25
|
+
from codeframe.core.config_watcher import ConfigReloadState
|
|
26
|
+
|
|
27
|
+
from codeframe.core.workspace import Workspace, get_db_connection
|
|
28
|
+
from codeframe.core import events, tasks, blockers
|
|
29
|
+
from codeframe.core.dependency_graph import create_execution_plan, CycleDetectedError
|
|
30
|
+
from codeframe.core.dependency_analyzer import analyze_dependencies, apply_inferred_dependencies
|
|
31
|
+
from codeframe.core.runtime import RunStatus, get_active_run, reset_blocked_run
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
logger = logging.getLogger(__name__)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _dispatch_batch_completed_webhook(
|
|
38
|
+
workspace: Workspace,
|
|
39
|
+
event_type: "events.EventType",
|
|
40
|
+
batch_id: str,
|
|
41
|
+
task_count: int,
|
|
42
|
+
) -> None:
|
|
43
|
+
"""Best-effort outbound webhook for ``batch.completed`` (issue #560).
|
|
44
|
+
|
|
45
|
+
Only fires when the batch reached the full COMPLETED state — PARTIAL,
|
|
46
|
+
FAILED, and CANCELLED are intentionally out of scope for v1. Failures
|
|
47
|
+
never break the batch.
|
|
48
|
+
"""
|
|
49
|
+
if event_type != events.EventType.BATCH_COMPLETED:
|
|
50
|
+
return
|
|
51
|
+
try:
|
|
52
|
+
from codeframe.core.notifications_config import is_webhook_active
|
|
53
|
+
from codeframe.notifications.webhook import (
|
|
54
|
+
WebhookNotificationService,
|
|
55
|
+
format_batch_payload,
|
|
56
|
+
)
|
|
57
|
+
|
|
58
|
+
url = is_webhook_active(workspace)
|
|
59
|
+
if url is None:
|
|
60
|
+
return
|
|
61
|
+
svc = WebhookNotificationService(webhook_url=url, timeout=5)
|
|
62
|
+
svc.send_event_background(format_batch_payload(batch_id, task_count))
|
|
63
|
+
except Exception:
|
|
64
|
+
logger.warning(
|
|
65
|
+
"Failed to dispatch batch.completed webhook for %s",
|
|
66
|
+
batch_id,
|
|
67
|
+
exc_info=True,
|
|
68
|
+
)
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
# Tactical patterns that the supervisor should auto-resolve
|
|
72
|
+
SUPERVISOR_TACTICAL_PATTERNS = [
|
|
73
|
+
# Virtual environment / package management
|
|
74
|
+
"virtual environment", "venv", "virtualenv",
|
|
75
|
+
"pip install", "npm install", "uv sync",
|
|
76
|
+
"break-system-packages", "pipx",
|
|
77
|
+
"package manager", "dependency installation",
|
|
78
|
+
"externally-managed", # PEP 668 error - use venv or uv
|
|
79
|
+
"externally managed",
|
|
80
|
+
# Module/import issues
|
|
81
|
+
"no module named", "__main__", "cannot be directly executed",
|
|
82
|
+
"modulenotfounderror", "importerror",
|
|
83
|
+
# Configuration
|
|
84
|
+
"pytest.ini", "pyproject.toml", "asyncio_default_fixture_loop_scope",
|
|
85
|
+
"fixture scope", "loop scope", "configuration file",
|
|
86
|
+
# Common tactical questions
|
|
87
|
+
"would you like me to", "would you prefer",
|
|
88
|
+
"should i create", "should i use",
|
|
89
|
+
"which approach", "which version",
|
|
90
|
+
"overwrite", "existing file",
|
|
91
|
+
]
|
|
92
|
+
|
|
93
|
+
# Cache of resolved decisions to avoid duplicate LLM calls
|
|
94
|
+
_decision_cache: dict[str, str] = {}
|
|
95
|
+
|
|
96
|
+
# Track running subprocesses for force stop capability
|
|
97
|
+
# Structure: {batch_id: {task_id: Popen}}
|
|
98
|
+
_active_processes: dict[str, dict[str, subprocess.Popen]] = {}
|
|
99
|
+
# Lock for thread-safe access to _active_processes
|
|
100
|
+
_active_processes_lock = threading.Lock()
|
|
101
|
+
# Lock for thread-safe batch database writes
|
|
102
|
+
_batch_db_lock = threading.Lock()
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
class SupervisorResolver:
|
|
106
|
+
"""Resolves tactical blockers at the conductor level.
|
|
107
|
+
|
|
108
|
+
Instead of each worker agent independently creating blockers,
|
|
109
|
+
the conductor uses this resolver to:
|
|
110
|
+
1. Evaluate blockers with the supervision model (stronger)
|
|
111
|
+
2. Auto-resolve tactical decisions
|
|
112
|
+
3. Deduplicate similar questions across workers
|
|
113
|
+
4. Only surface true human-required decisions
|
|
114
|
+
"""
|
|
115
|
+
|
|
116
|
+
def __init__(self, workspace: Workspace):
|
|
117
|
+
self.workspace = workspace
|
|
118
|
+
self._llm = None # Lazy initialization
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def llm(self):
|
|
122
|
+
"""Lazy-load LLM provider."""
|
|
123
|
+
if self._llm is None:
|
|
124
|
+
from codeframe.adapters.llm import get_provider
|
|
125
|
+
self._llm = get_provider()
|
|
126
|
+
return self._llm
|
|
127
|
+
|
|
128
|
+
def try_resolve_blocked_task(self, task_id: str) -> bool:
|
|
129
|
+
"""Try to resolve a blocked task's blocker autonomously.
|
|
130
|
+
|
|
131
|
+
Args:
|
|
132
|
+
task_id: The blocked task ID
|
|
133
|
+
|
|
134
|
+
Returns:
|
|
135
|
+
True if blocker was resolved (task should retry),
|
|
136
|
+
False if blocker requires human input
|
|
137
|
+
"""
|
|
138
|
+
# Get the task's open blockers
|
|
139
|
+
task_blockers = blockers.list_all(
|
|
140
|
+
self.workspace,
|
|
141
|
+
task_id=task_id,
|
|
142
|
+
status=blockers.BlockerStatus.OPEN,
|
|
143
|
+
)
|
|
144
|
+
|
|
145
|
+
if not task_blockers:
|
|
146
|
+
return False
|
|
147
|
+
|
|
148
|
+
blocker = task_blockers[0] # Most recent open blocker
|
|
149
|
+
question = blocker.question.lower()
|
|
150
|
+
|
|
151
|
+
# Check cache first
|
|
152
|
+
cache_key = self._get_cache_key(question)
|
|
153
|
+
if cache_key in _decision_cache:
|
|
154
|
+
print(" [Supervisor] Using cached decision for similar question")
|
|
155
|
+
self._auto_answer_blocker(blocker, _decision_cache[cache_key])
|
|
156
|
+
return True
|
|
157
|
+
|
|
158
|
+
# Check if question matches tactical patterns
|
|
159
|
+
if self._is_tactical_question(question):
|
|
160
|
+
print(" [Supervisor] Detected tactical question, auto-resolving")
|
|
161
|
+
resolution = self._generate_tactical_resolution(blocker.question)
|
|
162
|
+
_decision_cache[cache_key] = resolution
|
|
163
|
+
self._auto_answer_blocker(blocker, resolution)
|
|
164
|
+
return True
|
|
165
|
+
|
|
166
|
+
# Use supervision model to classify if uncertain
|
|
167
|
+
classification = self._classify_with_supervision(blocker.question)
|
|
168
|
+
|
|
169
|
+
if classification == "tactical":
|
|
170
|
+
print(" [Supervisor] Model classified as tactical, auto-resolving")
|
|
171
|
+
resolution = self._generate_tactical_resolution(blocker.question)
|
|
172
|
+
_decision_cache[cache_key] = resolution
|
|
173
|
+
self._auto_answer_blocker(blocker, resolution)
|
|
174
|
+
return True
|
|
175
|
+
|
|
176
|
+
# This is a genuine human-required decision
|
|
177
|
+
print(" [Supervisor] Question requires human input")
|
|
178
|
+
return False
|
|
179
|
+
|
|
180
|
+
def _is_tactical_question(self, question: str) -> bool:
|
|
181
|
+
"""Check if question matches known tactical patterns."""
|
|
182
|
+
return any(pattern in question for pattern in SUPERVISOR_TACTICAL_PATTERNS)
|
|
183
|
+
|
|
184
|
+
def _get_cache_key(self, question: str) -> str:
|
|
185
|
+
"""Generate a cache key for deduplication.
|
|
186
|
+
|
|
187
|
+
Normalizes similar questions to the same key.
|
|
188
|
+
"""
|
|
189
|
+
# Simple normalization - could be improved with embeddings
|
|
190
|
+
q = question.lower()
|
|
191
|
+
if "virtual environment" in q or "venv" in q or "virtualenv" in q:
|
|
192
|
+
return "venv_creation"
|
|
193
|
+
if "fixture scope" in q or "asyncio" in q:
|
|
194
|
+
return "asyncio_fixture_scope"
|
|
195
|
+
if "package manager" in q or "pip" in q or "npm" in q:
|
|
196
|
+
return "package_manager"
|
|
197
|
+
if "pytest" in q and ("fail" in q or "verification" in q):
|
|
198
|
+
return "pytest_failure"
|
|
199
|
+
# Fallback to hash of first 50 chars
|
|
200
|
+
return f"blocker_{hash(q[:50])}"
|
|
201
|
+
|
|
202
|
+
def _classify_with_supervision(self, question: str) -> str:
|
|
203
|
+
"""Use supervision model to classify the blocker question."""
|
|
204
|
+
from codeframe.adapters.llm import Purpose
|
|
205
|
+
|
|
206
|
+
prompt = f"""Classify this blocker question from a coding agent:
|
|
207
|
+
|
|
208
|
+
Question: {question}
|
|
209
|
+
|
|
210
|
+
Is this:
|
|
211
|
+
1. TACTICAL - A decision the agent should make autonomously (venv, package manager, config options, test framework, file handling)
|
|
212
|
+
2. HUMAN - Genuinely requires human input (conflicting requirements, missing credentials, business logic, security policy)
|
|
213
|
+
|
|
214
|
+
Respond with exactly one word: TACTICAL or HUMAN"""
|
|
215
|
+
|
|
216
|
+
try:
|
|
217
|
+
response = self.llm.complete(
|
|
218
|
+
messages=[{"role": "user", "content": prompt}],
|
|
219
|
+
purpose=Purpose.SUPERVISION,
|
|
220
|
+
max_tokens=10,
|
|
221
|
+
temperature=0.0,
|
|
222
|
+
)
|
|
223
|
+
result = response.content.strip().upper()
|
|
224
|
+
return "tactical" if "TACTICAL" in result else "human"
|
|
225
|
+
except Exception as e:
|
|
226
|
+
print(f" [Supervisor] Classification failed: {e}")
|
|
227
|
+
# Default to tactical for common patterns
|
|
228
|
+
return "tactical" if self._is_tactical_question(question.lower()) else "human"
|
|
229
|
+
|
|
230
|
+
def _generate_tactical_resolution(self, question: str) -> str:
|
|
231
|
+
"""Generate an autonomous resolution for tactical questions."""
|
|
232
|
+
q = question.lower()
|
|
233
|
+
|
|
234
|
+
# Common resolutions
|
|
235
|
+
if "virtual environment" in q or "venv" in q:
|
|
236
|
+
return "Create a Python virtual environment using 'python -m venv .venv' or 'uv venv', then activate it and install dependencies."
|
|
237
|
+
if "externally-managed" in q or "externally managed" in q:
|
|
238
|
+
return "This system uses an externally-managed Python. Use 'uv pip install' instead of 'pip install', or create a virtual environment first with 'uv venv' then 'source .venv/bin/activate'."
|
|
239
|
+
if "__main__" in q or "cannot be directly executed" in q:
|
|
240
|
+
return "Create a __main__.py file in the package directory with the entry point code (e.g., 'from .cli import main; main()' or similar)."
|
|
241
|
+
if "no module named" in q or "modulenotfounderror" in q:
|
|
242
|
+
return "Install the missing module using 'uv pip install <module>' or add it to requirements.txt and run 'uv pip install -r requirements.txt'."
|
|
243
|
+
if "importerror" in q:
|
|
244
|
+
return "Check that the module is installed and the import path is correct. For local modules, ensure __init__.py exists in the package directory."
|
|
245
|
+
if "fixture scope" in q or "asyncio" in q or "asyncio_default_fixture_loop_scope" in q:
|
|
246
|
+
return "Add '[tool.pytest.ini_options]\nasyncio_default_fixture_loop_scope = \"function\"' to pyproject.toml or 'asyncio_default_fixture_loop_scope = function' to pytest.ini."
|
|
247
|
+
if "package manager" in q:
|
|
248
|
+
return "Use the project's default package manager (uv for Python, npm for JS). For Python, prefer 'uv pip install' or 'uv sync'."
|
|
249
|
+
if "pytest" in q and "fail" in q:
|
|
250
|
+
return "Fix the failing tests and retry."
|
|
251
|
+
if "overwrite" in q or "existing file" in q:
|
|
252
|
+
return "Overwrite the existing file with the new content."
|
|
253
|
+
if "which version" in q:
|
|
254
|
+
return "Use the latest stable version."
|
|
255
|
+
|
|
256
|
+
# Generic resolution
|
|
257
|
+
return "Proceed with the most appropriate approach based on best practices."
|
|
258
|
+
|
|
259
|
+
def _auto_answer_blocker(self, blocker: blockers.Blocker, answer: str) -> None:
|
|
260
|
+
"""Auto-answer a blocker and reset the task for retry."""
|
|
261
|
+
# Answer the blocker
|
|
262
|
+
blockers.answer(self.workspace, blocker.id, f"[Auto-resolved by supervisor] {answer}")
|
|
263
|
+
|
|
264
|
+
# Reset the blocked run so task can retry
|
|
265
|
+
if blocker.task_id:
|
|
266
|
+
reset_blocked_run(self.workspace, blocker.task_id)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# Global supervisor instance per workspace (created lazily)
|
|
270
|
+
_supervisors: dict[str, SupervisorResolver] = {}
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def get_supervisor(workspace: Workspace) -> SupervisorResolver:
|
|
274
|
+
"""Get or create a supervisor resolver for a workspace."""
|
|
275
|
+
if workspace.id not in _supervisors:
|
|
276
|
+
_supervisors[workspace.id] = SupervisorResolver(workspace)
|
|
277
|
+
return _supervisors[workspace.id]
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
@dataclass
|
|
281
|
+
class GlobalFix:
|
|
282
|
+
"""A global fix that affects project-wide state.
|
|
283
|
+
|
|
284
|
+
Attributes:
|
|
285
|
+
error_signature: Hash of the normalized error
|
|
286
|
+
fix_description: What the fix does
|
|
287
|
+
fix_type: Type of fix (shell, edit, create)
|
|
288
|
+
command: Shell command (for shell type)
|
|
289
|
+
file_path: File to modify (for edit/create)
|
|
290
|
+
status: pending, executing, completed, failed
|
|
291
|
+
"""
|
|
292
|
+
error_signature: str
|
|
293
|
+
fix_description: str
|
|
294
|
+
fix_type: str
|
|
295
|
+
command: Optional[str] = None
|
|
296
|
+
file_path: Optional[str] = None
|
|
297
|
+
status: str = "pending"
|
|
298
|
+
result: Optional[str] = None
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
class GlobalFixCoordinator:
|
|
302
|
+
"""Coordinates global fixes across parallel agents.
|
|
303
|
+
|
|
304
|
+
When multiple agents hit the same global issue (e.g., missing local package),
|
|
305
|
+
this coordinator ensures:
|
|
306
|
+
1. Only ONE fix is executed (not N times for N agents)
|
|
307
|
+
2. Other agents wait for the fix to complete
|
|
308
|
+
3. Completed fixes are cached to skip redundant work
|
|
309
|
+
|
|
310
|
+
Thread-safe for parallel batch execution.
|
|
311
|
+
"""
|
|
312
|
+
|
|
313
|
+
def __init__(self, workspace: Workspace):
|
|
314
|
+
self.workspace = workspace
|
|
315
|
+
self._lock = threading.Lock()
|
|
316
|
+
self._pending: dict[str, GlobalFix] = {} # error_sig -> GlobalFix
|
|
317
|
+
self._completed: dict[str, GlobalFix] = {} # error_sig -> GlobalFix (succeeded)
|
|
318
|
+
self._condition = threading.Condition(self._lock)
|
|
319
|
+
|
|
320
|
+
def _hash_error(self, error: str) -> str:
|
|
321
|
+
"""Create a stable hash for an error message.
|
|
322
|
+
|
|
323
|
+
Normalizes variable parts (line numbers, paths) before hashing.
|
|
324
|
+
"""
|
|
325
|
+
# Remove line numbers
|
|
326
|
+
normalized = re.sub(r"line \d+", "line N", error)
|
|
327
|
+
# Remove specific file paths but keep filename
|
|
328
|
+
normalized = re.sub(r"/[^\s:]+/([^/\s:]+)", r"\1", normalized)
|
|
329
|
+
# Remove memory addresses
|
|
330
|
+
normalized = re.sub(r"0x[0-9a-fA-F]+", "0xADDR", normalized)
|
|
331
|
+
return hashlib.sha256(normalized.encode()).hexdigest()[:16]
|
|
332
|
+
|
|
333
|
+
def request_fix(
|
|
334
|
+
self,
|
|
335
|
+
error: str,
|
|
336
|
+
fix_type: str,
|
|
337
|
+
fix_description: str,
|
|
338
|
+
command: Optional[str] = None,
|
|
339
|
+
file_path: Optional[str] = None,
|
|
340
|
+
task_id: Optional[str] = None,
|
|
341
|
+
) -> tuple[str, bool]:
|
|
342
|
+
"""Request a global fix from the coordinator.
|
|
343
|
+
|
|
344
|
+
Returns:
|
|
345
|
+
Tuple of (status, should_execute):
|
|
346
|
+
- ("already_completed", False): Fix was already done, just retry verification
|
|
347
|
+
- ("pending", False): Another agent is handling it, wait and retry
|
|
348
|
+
- ("execute", True): You are responsible for executing this fix
|
|
349
|
+
"""
|
|
350
|
+
error_sig = self._hash_error(error)
|
|
351
|
+
|
|
352
|
+
with self._lock:
|
|
353
|
+
# Already completed successfully?
|
|
354
|
+
if error_sig in self._completed:
|
|
355
|
+
print(" [GlobalFix] Fix already completed for this error")
|
|
356
|
+
return ("already_completed", False)
|
|
357
|
+
|
|
358
|
+
# Already being worked on?
|
|
359
|
+
if error_sig in self._pending:
|
|
360
|
+
print(" [GlobalFix] Another agent is fixing this, waiting...")
|
|
361
|
+
# Wait for completion (with timeout)
|
|
362
|
+
return ("pending", False)
|
|
363
|
+
|
|
364
|
+
# This agent will handle it
|
|
365
|
+
fix = GlobalFix(
|
|
366
|
+
error_signature=error_sig,
|
|
367
|
+
fix_description=fix_description,
|
|
368
|
+
fix_type=fix_type,
|
|
369
|
+
command=command,
|
|
370
|
+
file_path=file_path,
|
|
371
|
+
status="executing",
|
|
372
|
+
)
|
|
373
|
+
self._pending[error_sig] = fix
|
|
374
|
+
print(f" [GlobalFix] Agent taking ownership: {fix_description[:60]}...")
|
|
375
|
+
return ("execute", True)
|
|
376
|
+
|
|
377
|
+
def report_fix_result(
|
|
378
|
+
self,
|
|
379
|
+
error: str,
|
|
380
|
+
success: bool,
|
|
381
|
+
result_message: Optional[str] = None,
|
|
382
|
+
) -> None:
|
|
383
|
+
"""Report the result of executing a global fix.
|
|
384
|
+
|
|
385
|
+
Args:
|
|
386
|
+
error: Original error message
|
|
387
|
+
success: Whether the fix succeeded
|
|
388
|
+
result_message: Optional details about the result
|
|
389
|
+
"""
|
|
390
|
+
error_sig = self._hash_error(error)
|
|
391
|
+
|
|
392
|
+
with self._lock:
|
|
393
|
+
if error_sig not in self._pending:
|
|
394
|
+
return # Not our fix
|
|
395
|
+
|
|
396
|
+
fix = self._pending.pop(error_sig)
|
|
397
|
+
fix.status = "completed" if success else "failed"
|
|
398
|
+
fix.result = result_message
|
|
399
|
+
|
|
400
|
+
if success:
|
|
401
|
+
self._completed[error_sig] = fix
|
|
402
|
+
print(" [GlobalFix] Fix completed successfully")
|
|
403
|
+
else:
|
|
404
|
+
print(f" [GlobalFix] Fix failed: {result_message}")
|
|
405
|
+
|
|
406
|
+
# Notify any waiting agents
|
|
407
|
+
self._condition.notify_all()
|
|
408
|
+
|
|
409
|
+
def wait_for_fix(self, error: str, timeout: float = 60.0) -> bool:
|
|
410
|
+
"""Wait for another agent to complete a fix.
|
|
411
|
+
|
|
412
|
+
Args:
|
|
413
|
+
error: Error message we're waiting on
|
|
414
|
+
timeout: Max seconds to wait
|
|
415
|
+
|
|
416
|
+
Returns:
|
|
417
|
+
True if fix was completed successfully, False otherwise
|
|
418
|
+
"""
|
|
419
|
+
error_sig = self._hash_error(error)
|
|
420
|
+
|
|
421
|
+
with self._condition:
|
|
422
|
+
start = datetime.now(timezone.utc)
|
|
423
|
+
while error_sig in self._pending:
|
|
424
|
+
remaining = timeout - (datetime.now(timezone.utc) - start).total_seconds()
|
|
425
|
+
if remaining <= 0:
|
|
426
|
+
print(" [GlobalFix] Timeout waiting for fix")
|
|
427
|
+
return False
|
|
428
|
+
self._condition.wait(timeout=remaining)
|
|
429
|
+
|
|
430
|
+
# Check if it completed successfully
|
|
431
|
+
return error_sig in self._completed
|
|
432
|
+
|
|
433
|
+
def is_fixed(self, error: str) -> bool:
|
|
434
|
+
"""Check if an error has already been fixed."""
|
|
435
|
+
error_sig = self._hash_error(error)
|
|
436
|
+
with self._lock:
|
|
437
|
+
return error_sig in self._completed
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
# Global coordinator instances per workspace
|
|
441
|
+
_coordinators: dict[str, GlobalFixCoordinator] = {}
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def get_fix_coordinator(workspace: Workspace) -> GlobalFixCoordinator:
|
|
445
|
+
"""Get or create a global fix coordinator for a workspace."""
|
|
446
|
+
if workspace.id not in _coordinators:
|
|
447
|
+
_coordinators[workspace.id] = GlobalFixCoordinator(workspace)
|
|
448
|
+
return _coordinators[workspace.id]
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _utc_now() -> datetime:
|
|
452
|
+
"""Get current UTC time as timezone-aware datetime."""
|
|
453
|
+
return datetime.now(timezone.utc)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
@dataclass
|
|
457
|
+
class ConcurrencyConfig:
|
|
458
|
+
"""Per-state concurrency limits for batch execution.
|
|
459
|
+
|
|
460
|
+
When ``by_status`` is non-empty, each status key caps the number of
|
|
461
|
+
concurrent workers for tasks in that state. Unspecified statuses
|
|
462
|
+
fall back to ``max_parallel``.
|
|
463
|
+
"""
|
|
464
|
+
|
|
465
|
+
max_parallel: int = 4
|
|
466
|
+
by_status: dict[str, int] = field(default_factory=dict)
|
|
467
|
+
|
|
468
|
+
def get_limit_for_status(self, status: str) -> int:
|
|
469
|
+
"""Return the concurrency limit for a given task status."""
|
|
470
|
+
return self.by_status.get(status, self.max_parallel)
|
|
471
|
+
|
|
472
|
+
def effective_workers(
|
|
473
|
+
self,
|
|
474
|
+
*,
|
|
475
|
+
statuses: list[str],
|
|
476
|
+
group_size: int,
|
|
477
|
+
global_running: int,
|
|
478
|
+
) -> int:
|
|
479
|
+
"""Compute the effective worker count for a group of tasks.
|
|
480
|
+
|
|
481
|
+
Takes the minimum of:
|
|
482
|
+
- Global slots remaining (max_parallel - global_running)
|
|
483
|
+
- Per-status limit for the most constrained status in the group
|
|
484
|
+
- Group size
|
|
485
|
+
"""
|
|
486
|
+
global_slots = max(1, self.max_parallel - global_running)
|
|
487
|
+
if statuses and self.by_status:
|
|
488
|
+
per_status = min(self.get_limit_for_status(s) for s in statuses)
|
|
489
|
+
else:
|
|
490
|
+
per_status = self.max_parallel
|
|
491
|
+
return max(1, min(global_slots, per_status, group_size))
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def parse_concurrency_by_status(value: str | None) -> dict[str, int]:
|
|
495
|
+
"""Parse a --max-parallel-by-status string into a dict.
|
|
496
|
+
|
|
497
|
+
Format: "READY=3,IN_PROGRESS=2"
|
|
498
|
+
|
|
499
|
+
Raises:
|
|
500
|
+
ValueError: On invalid status names or format.
|
|
501
|
+
"""
|
|
502
|
+
from codeframe.core.state_machine import TaskStatus
|
|
503
|
+
|
|
504
|
+
if not value:
|
|
505
|
+
return {}
|
|
506
|
+
|
|
507
|
+
valid_statuses = {s.value for s in TaskStatus}
|
|
508
|
+
result: dict[str, int] = {}
|
|
509
|
+
|
|
510
|
+
for pair in value.split(","):
|
|
511
|
+
pair = pair.strip()
|
|
512
|
+
if "=" not in pair:
|
|
513
|
+
raise ValueError(f"Invalid format '{pair}'. Expected STATUS=N")
|
|
514
|
+
key, val = pair.split("=", 1)
|
|
515
|
+
key = key.strip().upper()
|
|
516
|
+
if key not in valid_statuses:
|
|
517
|
+
raise ValueError(f"Invalid status '{key}'. Valid: {', '.join(sorted(valid_statuses))}")
|
|
518
|
+
try:
|
|
519
|
+
result[key] = int(val.strip())
|
|
520
|
+
except ValueError:
|
|
521
|
+
raise ValueError(f"Invalid value '{val.strip()}' for status '{key}'. Must be an integer.")
|
|
522
|
+
|
|
523
|
+
return result
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
class BatchStatus(str, Enum):
|
|
527
|
+
"""Status of a batch execution."""
|
|
528
|
+
|
|
529
|
+
PENDING = "PENDING" # Created but not started
|
|
530
|
+
RUNNING = "RUNNING" # Tasks being processed
|
|
531
|
+
COMPLETED = "COMPLETED" # All tasks finished successfully
|
|
532
|
+
PARTIAL = "PARTIAL" # Some tasks completed, some failed/blocked
|
|
533
|
+
FAILED = "FAILED" # Critical failure
|
|
534
|
+
CANCELLED = "CANCELLED" # User cancelled
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
class OnFailure(str, Enum):
|
|
538
|
+
"""Behavior when a task fails."""
|
|
539
|
+
|
|
540
|
+
CONTINUE = "continue" # Continue with remaining tasks
|
|
541
|
+
STOP = "stop" # Stop batch on first failure
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
@dataclass
|
|
545
|
+
class BatchRun:
|
|
546
|
+
"""Represents a batch execution run.
|
|
547
|
+
|
|
548
|
+
Attributes:
|
|
549
|
+
id: Unique batch identifier (UUID)
|
|
550
|
+
workspace_id: Workspace this batch belongs to
|
|
551
|
+
task_ids: Ordered list of task IDs to execute
|
|
552
|
+
status: Current batch status
|
|
553
|
+
strategy: Execution strategy (serial, parallel)
|
|
554
|
+
max_parallel: Max concurrent tasks (for parallel strategy)
|
|
555
|
+
on_failure: Behavior on task failure
|
|
556
|
+
started_at: When the batch started
|
|
557
|
+
completed_at: When the batch finished (if finished)
|
|
558
|
+
results: Dict mapping task_id -> RunStatus value
|
|
559
|
+
"""
|
|
560
|
+
|
|
561
|
+
id: str
|
|
562
|
+
workspace_id: str
|
|
563
|
+
task_ids: list[str]
|
|
564
|
+
status: BatchStatus
|
|
565
|
+
strategy: str
|
|
566
|
+
max_parallel: int
|
|
567
|
+
on_failure: OnFailure
|
|
568
|
+
started_at: datetime
|
|
569
|
+
completed_at: Optional[datetime]
|
|
570
|
+
results: dict[str, str] = field(default_factory=dict)
|
|
571
|
+
engine: str = "react"
|
|
572
|
+
stall_timeout_s: int = 300
|
|
573
|
+
stall_action: str = "blocker"
|
|
574
|
+
concurrency: ConcurrencyConfig = field(default_factory=ConcurrencyConfig)
|
|
575
|
+
isolate: bool = True
|
|
576
|
+
isolation: str = "none"
|
|
577
|
+
llm_provider: Optional[str] = None
|
|
578
|
+
llm_model: Optional[str] = None
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def start_batch(
|
|
582
|
+
workspace: Workspace,
|
|
583
|
+
task_ids: list[str],
|
|
584
|
+
strategy: str = "serial",
|
|
585
|
+
max_parallel: int = 4,
|
|
586
|
+
on_failure: str = "continue",
|
|
587
|
+
dry_run: bool = False,
|
|
588
|
+
max_retries: int = 0,
|
|
589
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
590
|
+
engine: str = "react",
|
|
591
|
+
stall_timeout_s: int = 300,
|
|
592
|
+
stall_action: str = "blocker",
|
|
593
|
+
concurrency_by_status: Optional[dict[str, int]] = None,
|
|
594
|
+
isolate: bool = True,
|
|
595
|
+
isolation: str = "none",
|
|
596
|
+
cloud_timeout_minutes: int = 30,
|
|
597
|
+
llm_provider: Optional[str] = None,
|
|
598
|
+
llm_model: Optional[str] = None,
|
|
599
|
+
) -> BatchRun:
|
|
600
|
+
"""Start a batch execution of multiple tasks.
|
|
601
|
+
|
|
602
|
+
Args:
|
|
603
|
+
workspace: Target workspace
|
|
604
|
+
task_ids: List of task IDs to execute (in order)
|
|
605
|
+
strategy: Execution strategy ("serial" or "parallel")
|
|
606
|
+
max_parallel: Max concurrent tasks for parallel strategy
|
|
607
|
+
on_failure: Behavior on task failure ("continue" or "stop")
|
|
608
|
+
dry_run: If True, don't actually execute tasks
|
|
609
|
+
max_retries: Max retry attempts for failed tasks (0 = no retries)
|
|
610
|
+
on_event: Optional callback for batch events
|
|
611
|
+
engine: Agent engine to use ("react" default, or "plan" for legacy)
|
|
612
|
+
|
|
613
|
+
Returns:
|
|
614
|
+
BatchRun with results populated
|
|
615
|
+
|
|
616
|
+
Raises:
|
|
617
|
+
ValueError: If task_ids is empty or contains invalid IDs
|
|
618
|
+
"""
|
|
619
|
+
if not task_ids:
|
|
620
|
+
raise ValueError("task_ids cannot be empty")
|
|
621
|
+
|
|
622
|
+
# Validate all task IDs exist
|
|
623
|
+
for task_id in task_ids:
|
|
624
|
+
task = tasks.get(workspace, task_id)
|
|
625
|
+
if not task:
|
|
626
|
+
raise ValueError(f"Task not found: {task_id}")
|
|
627
|
+
|
|
628
|
+
# Create batch record
|
|
629
|
+
batch_id = str(uuid.uuid4())
|
|
630
|
+
now = _utc_now()
|
|
631
|
+
on_failure_enum = OnFailure(on_failure)
|
|
632
|
+
|
|
633
|
+
concurrency = ConcurrencyConfig(
|
|
634
|
+
max_parallel=max_parallel,
|
|
635
|
+
by_status=concurrency_by_status or {},
|
|
636
|
+
)
|
|
637
|
+
|
|
638
|
+
batch = BatchRun(
|
|
639
|
+
id=batch_id,
|
|
640
|
+
workspace_id=workspace.id,
|
|
641
|
+
task_ids=task_ids,
|
|
642
|
+
status=BatchStatus.PENDING,
|
|
643
|
+
strategy=strategy,
|
|
644
|
+
max_parallel=max_parallel,
|
|
645
|
+
on_failure=on_failure_enum,
|
|
646
|
+
started_at=now,
|
|
647
|
+
completed_at=None,
|
|
648
|
+
results={},
|
|
649
|
+
engine=engine,
|
|
650
|
+
stall_timeout_s=stall_timeout_s,
|
|
651
|
+
stall_action=stall_action,
|
|
652
|
+
concurrency=concurrency,
|
|
653
|
+
isolate=isolate,
|
|
654
|
+
isolation=isolation,
|
|
655
|
+
llm_provider=llm_provider,
|
|
656
|
+
llm_model=llm_model,
|
|
657
|
+
)
|
|
658
|
+
|
|
659
|
+
# Save to database
|
|
660
|
+
_save_batch(workspace, batch)
|
|
661
|
+
|
|
662
|
+
# Emit batch started event
|
|
663
|
+
events.emit_for_workspace(
|
|
664
|
+
workspace,
|
|
665
|
+
events.EventType.BATCH_STARTED,
|
|
666
|
+
{
|
|
667
|
+
"batch_id": batch_id,
|
|
668
|
+
"task_ids": task_ids,
|
|
669
|
+
"strategy": strategy,
|
|
670
|
+
"task_count": len(task_ids),
|
|
671
|
+
},
|
|
672
|
+
print_event=True,
|
|
673
|
+
)
|
|
674
|
+
|
|
675
|
+
if on_event:
|
|
676
|
+
on_event("batch_started", {"batch_id": batch_id, "task_count": len(task_ids)})
|
|
677
|
+
|
|
678
|
+
if dry_run:
|
|
679
|
+
batch.status = BatchStatus.COMPLETED
|
|
680
|
+
batch.completed_at = _utc_now()
|
|
681
|
+
_save_batch(workspace, batch)
|
|
682
|
+
return batch
|
|
683
|
+
|
|
684
|
+
# Update status to running
|
|
685
|
+
batch.status = BatchStatus.RUNNING
|
|
686
|
+
_save_batch(workspace, batch)
|
|
687
|
+
|
|
688
|
+
# Execute based on strategy
|
|
689
|
+
if strategy == "auto":
|
|
690
|
+
# Use LLM to infer dependencies, then execute in parallel
|
|
691
|
+
try:
|
|
692
|
+
print("\nAnalyzing task dependencies with LLM...")
|
|
693
|
+
dependencies = analyze_dependencies(workspace, task_ids)
|
|
694
|
+
|
|
695
|
+
# Show inferred dependencies
|
|
696
|
+
deps_with_values = {k: v for k, v in dependencies.items() if v}
|
|
697
|
+
if deps_with_values:
|
|
698
|
+
print("Inferred dependencies:")
|
|
699
|
+
for tid, deps in deps_with_values.items():
|
|
700
|
+
task = tasks.get(workspace, tid)
|
|
701
|
+
task_title = task.title[:40] if task else tid[:8]
|
|
702
|
+
dep_titles = []
|
|
703
|
+
for d in deps:
|
|
704
|
+
dep_task = tasks.get(workspace, d)
|
|
705
|
+
dep_titles.append(dep_task.title[:30] if dep_task else d[:8])
|
|
706
|
+
print(f" {task_title} <- {', '.join(dep_titles)}")
|
|
707
|
+
else:
|
|
708
|
+
print("No dependencies inferred - tasks appear independent")
|
|
709
|
+
|
|
710
|
+
# Apply inferred dependencies to task records
|
|
711
|
+
# Note: This persists the dependencies to the database so they're
|
|
712
|
+
# available for future executions and batch resumes
|
|
713
|
+
apply_inferred_dependencies(workspace, dependencies)
|
|
714
|
+
|
|
715
|
+
# Execute with parallel strategy
|
|
716
|
+
_execute_parallel(workspace, batch, on_event)
|
|
717
|
+
except CycleDetectedError as e:
|
|
718
|
+
print(f"Error: {e}")
|
|
719
|
+
print("Falling back to serial execution")
|
|
720
|
+
_execute_serial(workspace, batch, on_event)
|
|
721
|
+
except Exception as e:
|
|
722
|
+
print(f"Dependency analysis failed: {e}")
|
|
723
|
+
print("Falling back to serial execution")
|
|
724
|
+
_execute_serial(workspace, batch, on_event)
|
|
725
|
+
elif strategy == "parallel" and max_parallel > 1:
|
|
726
|
+
try:
|
|
727
|
+
_execute_parallel(workspace, batch, on_event)
|
|
728
|
+
except CycleDetectedError as e:
|
|
729
|
+
print(f"Error: {e}")
|
|
730
|
+
print("Falling back to serial execution")
|
|
731
|
+
_execute_serial(workspace, batch, on_event)
|
|
732
|
+
else:
|
|
733
|
+
_execute_serial(workspace, batch, on_event)
|
|
734
|
+
|
|
735
|
+
# Retry failed tasks if max_retries > 0
|
|
736
|
+
if max_retries > 0:
|
|
737
|
+
_execute_retries(workspace, batch, max_retries, on_event)
|
|
738
|
+
|
|
739
|
+
return batch
|
|
740
|
+
|
|
741
|
+
|
|
742
|
+
def get_batch(workspace: Workspace, batch_id: str) -> Optional[BatchRun]:
|
|
743
|
+
"""Get a batch by ID.
|
|
744
|
+
|
|
745
|
+
Args:
|
|
746
|
+
workspace: Workspace to query
|
|
747
|
+
batch_id: Batch identifier
|
|
748
|
+
|
|
749
|
+
Returns:
|
|
750
|
+
BatchRun if found, None otherwise
|
|
751
|
+
"""
|
|
752
|
+
conn = get_db_connection(workspace)
|
|
753
|
+
cursor = conn.cursor()
|
|
754
|
+
|
|
755
|
+
cursor.execute(
|
|
756
|
+
"""
|
|
757
|
+
SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
|
|
758
|
+
on_failure, started_at, completed_at, results, engine
|
|
759
|
+
FROM batch_runs
|
|
760
|
+
WHERE workspace_id = ? AND id = ?
|
|
761
|
+
""",
|
|
762
|
+
(workspace.id, batch_id),
|
|
763
|
+
)
|
|
764
|
+
row = cursor.fetchone()
|
|
765
|
+
conn.close()
|
|
766
|
+
|
|
767
|
+
if not row:
|
|
768
|
+
return None
|
|
769
|
+
|
|
770
|
+
return _row_to_batch(row)
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def list_batches(
|
|
774
|
+
workspace: Workspace,
|
|
775
|
+
status: Optional[BatchStatus] = None,
|
|
776
|
+
limit: int = 20,
|
|
777
|
+
) -> list[BatchRun]:
|
|
778
|
+
"""List batches in a workspace.
|
|
779
|
+
|
|
780
|
+
Args:
|
|
781
|
+
workspace: Workspace to query
|
|
782
|
+
status: Optional status filter
|
|
783
|
+
limit: Maximum batches to return
|
|
784
|
+
|
|
785
|
+
Returns:
|
|
786
|
+
List of BatchRuns, newest first
|
|
787
|
+
"""
|
|
788
|
+
conn = get_db_connection(workspace)
|
|
789
|
+
cursor = conn.cursor()
|
|
790
|
+
|
|
791
|
+
if status:
|
|
792
|
+
cursor.execute(
|
|
793
|
+
"""
|
|
794
|
+
SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
|
|
795
|
+
on_failure, started_at, completed_at, results, engine
|
|
796
|
+
FROM batch_runs
|
|
797
|
+
WHERE workspace_id = ? AND status = ?
|
|
798
|
+
ORDER BY started_at DESC
|
|
799
|
+
LIMIT ?
|
|
800
|
+
""",
|
|
801
|
+
(workspace.id, status.value, limit),
|
|
802
|
+
)
|
|
803
|
+
else:
|
|
804
|
+
cursor.execute(
|
|
805
|
+
"""
|
|
806
|
+
SELECT id, workspace_id, task_ids, status, strategy, max_parallel,
|
|
807
|
+
on_failure, started_at, completed_at, results, engine
|
|
808
|
+
FROM batch_runs
|
|
809
|
+
WHERE workspace_id = ?
|
|
810
|
+
ORDER BY started_at DESC
|
|
811
|
+
LIMIT ?
|
|
812
|
+
""",
|
|
813
|
+
(workspace.id, limit),
|
|
814
|
+
)
|
|
815
|
+
|
|
816
|
+
rows = cursor.fetchall()
|
|
817
|
+
conn.close()
|
|
818
|
+
|
|
819
|
+
return [_row_to_batch(row) for row in rows]
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def cancel_batch(workspace: Workspace, batch_id: str) -> BatchRun:
|
|
823
|
+
"""Cancel a running batch.
|
|
824
|
+
|
|
825
|
+
Sends SIGTERM to any running subprocesses and marks the batch as cancelled.
|
|
826
|
+
|
|
827
|
+
Args:
|
|
828
|
+
workspace: Target workspace
|
|
829
|
+
batch_id: Batch to cancel
|
|
830
|
+
|
|
831
|
+
Returns:
|
|
832
|
+
Updated BatchRun
|
|
833
|
+
|
|
834
|
+
Raises:
|
|
835
|
+
ValueError: If batch not found or not in a cancellable state
|
|
836
|
+
"""
|
|
837
|
+
batch = get_batch(workspace, batch_id)
|
|
838
|
+
if not batch:
|
|
839
|
+
raise ValueError(f"Batch not found: {batch_id}")
|
|
840
|
+
|
|
841
|
+
if batch.status not in (BatchStatus.PENDING, BatchStatus.RUNNING):
|
|
842
|
+
raise ValueError(f"Batch cannot be cancelled: {batch.status}")
|
|
843
|
+
|
|
844
|
+
# Update status
|
|
845
|
+
batch.status = BatchStatus.CANCELLED
|
|
846
|
+
batch.completed_at = _utc_now()
|
|
847
|
+
_save_batch(workspace, batch)
|
|
848
|
+
|
|
849
|
+
# Emit event
|
|
850
|
+
events.emit_for_workspace(
|
|
851
|
+
workspace,
|
|
852
|
+
events.EventType.BATCH_CANCELLED,
|
|
853
|
+
{"batch_id": batch_id},
|
|
854
|
+
print_event=True,
|
|
855
|
+
)
|
|
856
|
+
|
|
857
|
+
return batch
|
|
858
|
+
|
|
859
|
+
|
|
860
|
+
def stop_batch(workspace: Workspace, batch_id: str, force: bool = False) -> BatchRun:
|
|
861
|
+
"""Stop a running batch.
|
|
862
|
+
|
|
863
|
+
Graceful stop (force=False):
|
|
864
|
+
- Marks batch as CANCELLED
|
|
865
|
+
- Execution loops will exit after current task completes
|
|
866
|
+
- Running tasks are allowed to finish naturally
|
|
867
|
+
|
|
868
|
+
Force stop (force=True):
|
|
869
|
+
- Marks batch as CANCELLED immediately
|
|
870
|
+
- Sends SIGTERM to all running subprocesses
|
|
871
|
+
- Processes have a brief window to cleanup before termination
|
|
872
|
+
|
|
873
|
+
Args:
|
|
874
|
+
workspace: Target workspace
|
|
875
|
+
batch_id: Batch to stop
|
|
876
|
+
force: If True, terminate running processes immediately
|
|
877
|
+
|
|
878
|
+
Returns:
|
|
879
|
+
Updated BatchRun
|
|
880
|
+
|
|
881
|
+
Raises:
|
|
882
|
+
ValueError: If batch not found or not stoppable
|
|
883
|
+
"""
|
|
884
|
+
batch = get_batch(workspace, batch_id)
|
|
885
|
+
if not batch:
|
|
886
|
+
raise ValueError(f"Batch not found: {batch_id}")
|
|
887
|
+
|
|
888
|
+
if batch.status not in (BatchStatus.PENDING, BatchStatus.RUNNING):
|
|
889
|
+
raise ValueError(f"Batch cannot be stopped (status={batch.status})")
|
|
890
|
+
|
|
891
|
+
# Update status first - execution loops check this
|
|
892
|
+
batch.status = BatchStatus.CANCELLED
|
|
893
|
+
batch.completed_at = _utc_now()
|
|
894
|
+
_save_batch(workspace, batch)
|
|
895
|
+
|
|
896
|
+
terminated_count = 0
|
|
897
|
+
with _active_processes_lock:
|
|
898
|
+
if force and batch_id in _active_processes:
|
|
899
|
+
# Terminate all running processes for this batch
|
|
900
|
+
processes = _active_processes.get(batch_id, {})
|
|
901
|
+
for task_id, process in list(processes.items()):
|
|
902
|
+
try:
|
|
903
|
+
if process.poll() is None: # Still running
|
|
904
|
+
process.terminate() # SIGTERM
|
|
905
|
+
terminated_count += 1
|
|
906
|
+
except (ProcessLookupError, OSError):
|
|
907
|
+
pass # Process already exited
|
|
908
|
+
|
|
909
|
+
# Cleanup tracking
|
|
910
|
+
_active_processes.pop(batch_id, None)
|
|
911
|
+
|
|
912
|
+
# Emit event
|
|
913
|
+
event_data = {"batch_id": batch_id, "force": force}
|
|
914
|
+
if terminated_count > 0:
|
|
915
|
+
event_data["terminated_processes"] = terminated_count
|
|
916
|
+
|
|
917
|
+
events.emit_for_workspace(
|
|
918
|
+
workspace,
|
|
919
|
+
events.EventType.BATCH_CANCELLED,
|
|
920
|
+
event_data,
|
|
921
|
+
print_event=True,
|
|
922
|
+
)
|
|
923
|
+
|
|
924
|
+
return batch
|
|
925
|
+
|
|
926
|
+
|
|
927
|
+
def resume_batch(
|
|
928
|
+
workspace: Workspace,
|
|
929
|
+
batch_id: str,
|
|
930
|
+
force: bool = False,
|
|
931
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
932
|
+
) -> BatchRun:
|
|
933
|
+
"""Resume a batch by re-running failed/blocked tasks.
|
|
934
|
+
|
|
935
|
+
Args:
|
|
936
|
+
workspace: Target workspace
|
|
937
|
+
batch_id: Batch to resume
|
|
938
|
+
force: If True, re-run all tasks including completed ones
|
|
939
|
+
on_event: Optional callback for batch events
|
|
940
|
+
|
|
941
|
+
Returns:
|
|
942
|
+
Updated BatchRun with new results
|
|
943
|
+
|
|
944
|
+
Raises:
|
|
945
|
+
ValueError: If batch not found or not in a resumable state
|
|
946
|
+
"""
|
|
947
|
+
batch = get_batch(workspace, batch_id)
|
|
948
|
+
if not batch:
|
|
949
|
+
raise ValueError(f"Batch not found: {batch_id}")
|
|
950
|
+
|
|
951
|
+
# Check if batch is in a resumable state
|
|
952
|
+
resumable_statuses = (
|
|
953
|
+
BatchStatus.PARTIAL,
|
|
954
|
+
BatchStatus.FAILED,
|
|
955
|
+
BatchStatus.CANCELLED,
|
|
956
|
+
)
|
|
957
|
+
if batch.status not in resumable_statuses:
|
|
958
|
+
raise ValueError(
|
|
959
|
+
f"Batch cannot be resumed (status={batch.status}). "
|
|
960
|
+
f"Only {', '.join(s.value for s in resumable_statuses)} batches can be resumed."
|
|
961
|
+
)
|
|
962
|
+
|
|
963
|
+
# Determine which tasks to re-run
|
|
964
|
+
if force:
|
|
965
|
+
# Re-run all tasks
|
|
966
|
+
tasks_to_run = batch.task_ids
|
|
967
|
+
print(f"Resuming batch {batch_id[:8]}... (force mode: re-running all {len(tasks_to_run)} tasks)")
|
|
968
|
+
else:
|
|
969
|
+
# Only re-run failed/blocked tasks
|
|
970
|
+
failed_statuses = {"FAILED", "BLOCKED"}
|
|
971
|
+
tasks_to_run = [
|
|
972
|
+
tid for tid in batch.task_ids
|
|
973
|
+
if batch.results.get(tid) in failed_statuses or tid not in batch.results
|
|
974
|
+
]
|
|
975
|
+
if not tasks_to_run:
|
|
976
|
+
print(f"No failed or blocked tasks to resume in batch {batch_id[:8]}")
|
|
977
|
+
return batch
|
|
978
|
+
print(f"Resuming batch {batch_id[:8]}... (re-running {len(tasks_to_run)} failed/blocked tasks)")
|
|
979
|
+
|
|
980
|
+
# Emit batch resumed event
|
|
981
|
+
events.emit_for_workspace(
|
|
982
|
+
workspace,
|
|
983
|
+
events.EventType.BATCH_STARTED, # Reuse BATCH_STARTED for resume
|
|
984
|
+
{
|
|
985
|
+
"batch_id": batch_id,
|
|
986
|
+
"task_ids": tasks_to_run,
|
|
987
|
+
"strategy": batch.strategy,
|
|
988
|
+
"task_count": len(tasks_to_run),
|
|
989
|
+
"is_resume": True,
|
|
990
|
+
},
|
|
991
|
+
print_event=True,
|
|
992
|
+
)
|
|
993
|
+
|
|
994
|
+
if on_event:
|
|
995
|
+
on_event("batch_resumed", {"batch_id": batch_id, "task_count": len(tasks_to_run)})
|
|
996
|
+
|
|
997
|
+
# Update status to running
|
|
998
|
+
batch.status = BatchStatus.RUNNING
|
|
999
|
+
batch.completed_at = None # Clear completed_at since we're running again
|
|
1000
|
+
_save_batch(workspace, batch)
|
|
1001
|
+
|
|
1002
|
+
# Execute the tasks
|
|
1003
|
+
_execute_serial_resume(workspace, batch, tasks_to_run, on_event)
|
|
1004
|
+
|
|
1005
|
+
return batch
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def _execute_serial_resume(
|
|
1009
|
+
workspace: Workspace,
|
|
1010
|
+
batch: BatchRun,
|
|
1011
|
+
tasks_to_run: list[str],
|
|
1012
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1013
|
+
) -> None:
|
|
1014
|
+
"""Execute a subset of tasks serially for resume.
|
|
1015
|
+
|
|
1016
|
+
Similar to _execute_serial but only runs specified tasks and
|
|
1017
|
+
merges results with existing batch results.
|
|
1018
|
+
"""
|
|
1019
|
+
completed_count = 0
|
|
1020
|
+
failed_count = 0
|
|
1021
|
+
blocked_count = 0
|
|
1022
|
+
|
|
1023
|
+
# Count existing successful results (tasks not being re-run)
|
|
1024
|
+
for task_id in batch.task_ids:
|
|
1025
|
+
if task_id not in tasks_to_run:
|
|
1026
|
+
if batch.results.get(task_id) == RunStatus.COMPLETED.value:
|
|
1027
|
+
completed_count += 1
|
|
1028
|
+
|
|
1029
|
+
for i, task_id in enumerate(tasks_to_run):
|
|
1030
|
+
# Check if batch was cancelled
|
|
1031
|
+
current_batch = get_batch(workspace, batch.id)
|
|
1032
|
+
if current_batch and current_batch.status == BatchStatus.CANCELLED:
|
|
1033
|
+
break
|
|
1034
|
+
|
|
1035
|
+
# Emit task queued event
|
|
1036
|
+
events.emit_for_workspace(
|
|
1037
|
+
workspace,
|
|
1038
|
+
events.EventType.BATCH_TASK_QUEUED,
|
|
1039
|
+
{"batch_id": batch.id, "task_id": task_id, "position": i + 1},
|
|
1040
|
+
print_event=True,
|
|
1041
|
+
)
|
|
1042
|
+
|
|
1043
|
+
# Get task info for display
|
|
1044
|
+
task = tasks.get(workspace, task_id)
|
|
1045
|
+
task_title = task.title if task else task_id
|
|
1046
|
+
previous_status = batch.results.get(task_id, "N/A")
|
|
1047
|
+
|
|
1048
|
+
print(f"\n[{i + 1}/{len(tasks_to_run)}] Retrying task {task_id}: {task_title}")
|
|
1049
|
+
print(f" Previous status: {previous_status}")
|
|
1050
|
+
|
|
1051
|
+
# Emit task started event
|
|
1052
|
+
events.emit_for_workspace(
|
|
1053
|
+
workspace,
|
|
1054
|
+
events.EventType.BATCH_TASK_STARTED,
|
|
1055
|
+
{"batch_id": batch.id, "task_id": task_id, "is_retry": True},
|
|
1056
|
+
print_event=True,
|
|
1057
|
+
)
|
|
1058
|
+
|
|
1059
|
+
if on_event:
|
|
1060
|
+
on_event("batch_task_started", {"task_id": task_id, "position": i + 1, "is_retry": True})
|
|
1061
|
+
|
|
1062
|
+
# Execute task via subprocess (with isolation context)
|
|
1063
|
+
from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
|
|
1064
|
+
exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
|
|
1065
|
+
try:
|
|
1066
|
+
result_status = _execute_task_subprocess(
|
|
1067
|
+
workspace, task_id, batch.id, engine=batch.engine,
|
|
1068
|
+
stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
|
|
1069
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1070
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1071
|
+
)
|
|
1072
|
+
finally:
|
|
1073
|
+
exec_ctx.cleanup()
|
|
1074
|
+
|
|
1075
|
+
# Record result (overwrites previous result)
|
|
1076
|
+
batch.results[task_id] = result_status
|
|
1077
|
+
_save_batch(workspace, batch)
|
|
1078
|
+
|
|
1079
|
+
# Emit appropriate event based on result
|
|
1080
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
1081
|
+
completed_count += 1
|
|
1082
|
+
events.emit_for_workspace(
|
|
1083
|
+
workspace,
|
|
1084
|
+
events.EventType.BATCH_TASK_COMPLETED,
|
|
1085
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1086
|
+
print_event=True,
|
|
1087
|
+
)
|
|
1088
|
+
print(f" ✓ Completed (was: {previous_status})")
|
|
1089
|
+
elif result_status == RunStatus.BLOCKED.value:
|
|
1090
|
+
blocked_count += 1
|
|
1091
|
+
events.emit_for_workspace(
|
|
1092
|
+
workspace,
|
|
1093
|
+
events.EventType.BATCH_TASK_BLOCKED,
|
|
1094
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1095
|
+
print_event=True,
|
|
1096
|
+
)
|
|
1097
|
+
print(" ⊘ Still blocked")
|
|
1098
|
+
else:
|
|
1099
|
+
failed_count += 1
|
|
1100
|
+
events.emit_for_workspace(
|
|
1101
|
+
workspace,
|
|
1102
|
+
events.EventType.BATCH_TASK_FAILED,
|
|
1103
|
+
{"batch_id": batch.id, "task_id": task_id, "status": result_status},
|
|
1104
|
+
print_event=True,
|
|
1105
|
+
)
|
|
1106
|
+
print(f" ✗ Still failed: {result_status}")
|
|
1107
|
+
|
|
1108
|
+
# Note: resume doesn't stop on failure, always continues
|
|
1109
|
+
# to give all failed tasks a chance
|
|
1110
|
+
|
|
1111
|
+
if on_event:
|
|
1112
|
+
on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
|
|
1113
|
+
|
|
1114
|
+
# Determine final batch status based on ALL results
|
|
1115
|
+
total = len(batch.task_ids)
|
|
1116
|
+
|
|
1117
|
+
# Recount from results
|
|
1118
|
+
final_completed = sum(1 for s in batch.results.values() if s == RunStatus.COMPLETED.value)
|
|
1119
|
+
final_failed = sum(1 for s in batch.results.values() if s == RunStatus.FAILED.value)
|
|
1120
|
+
final_blocked = sum(1 for s in batch.results.values() if s == RunStatus.BLOCKED.value)
|
|
1121
|
+
|
|
1122
|
+
if final_completed == total:
|
|
1123
|
+
batch.status = BatchStatus.COMPLETED
|
|
1124
|
+
event_type = events.EventType.BATCH_COMPLETED
|
|
1125
|
+
|
|
1126
|
+
# Run batch-level validation (full gate sweep)
|
|
1127
|
+
validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
|
|
1128
|
+
|
|
1129
|
+
if not validation_passed:
|
|
1130
|
+
# Gates failed - change status to PARTIAL (tasks done, integration broken)
|
|
1131
|
+
batch.status = BatchStatus.PARTIAL
|
|
1132
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1133
|
+
print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
|
|
1134
|
+
print(f"Validation error: {validation_error}")
|
|
1135
|
+
|
|
1136
|
+
elif final_completed == 0 and (final_failed > 0 or final_blocked > 0):
|
|
1137
|
+
batch.status = BatchStatus.FAILED
|
|
1138
|
+
event_type = events.EventType.BATCH_FAILED
|
|
1139
|
+
elif final_completed > 0:
|
|
1140
|
+
batch.status = BatchStatus.PARTIAL
|
|
1141
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1142
|
+
else:
|
|
1143
|
+
batch.status = BatchStatus.CANCELLED
|
|
1144
|
+
event_type = events.EventType.BATCH_CANCELLED
|
|
1145
|
+
|
|
1146
|
+
batch.completed_at = _utc_now()
|
|
1147
|
+
_save_batch(workspace, batch)
|
|
1148
|
+
|
|
1149
|
+
# Emit batch completion event
|
|
1150
|
+
events.emit_for_workspace(
|
|
1151
|
+
workspace,
|
|
1152
|
+
event_type,
|
|
1153
|
+
{
|
|
1154
|
+
"batch_id": batch.id,
|
|
1155
|
+
"completed": final_completed,
|
|
1156
|
+
"failed": final_failed,
|
|
1157
|
+
"blocked": final_blocked,
|
|
1158
|
+
"total": total,
|
|
1159
|
+
"is_resume": True,
|
|
1160
|
+
},
|
|
1161
|
+
print_event=True,
|
|
1162
|
+
)
|
|
1163
|
+
_dispatch_batch_completed_webhook(workspace, event_type, batch.id, final_completed)
|
|
1164
|
+
|
|
1165
|
+
# Print summary
|
|
1166
|
+
print(f"\nBatch resume {batch.status.value.lower()}: {final_completed}/{total} tasks completed")
|
|
1167
|
+
if final_failed > 0:
|
|
1168
|
+
print(f" Failed: {final_failed}")
|
|
1169
|
+
if final_blocked > 0:
|
|
1170
|
+
print(f" Blocked: {final_blocked}")
|
|
1171
|
+
|
|
1172
|
+
|
|
1173
|
+
def _execute_retries(
|
|
1174
|
+
workspace: Workspace,
|
|
1175
|
+
batch: BatchRun,
|
|
1176
|
+
max_retries: int,
|
|
1177
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1178
|
+
) -> None:
|
|
1179
|
+
"""Retry failed tasks up to max_retries times.
|
|
1180
|
+
|
|
1181
|
+
After the initial execution pass, this function re-runs any failed
|
|
1182
|
+
tasks, continuing until all tasks succeed or max_retries is exhausted.
|
|
1183
|
+
|
|
1184
|
+
Args:
|
|
1185
|
+
workspace: Target workspace
|
|
1186
|
+
batch: BatchRun with initial results
|
|
1187
|
+
max_retries: Maximum retry attempts per task
|
|
1188
|
+
on_event: Optional callback for events
|
|
1189
|
+
"""
|
|
1190
|
+
failed_statuses = {RunStatus.FAILED.value} # Only retry FAILED, not BLOCKED
|
|
1191
|
+
|
|
1192
|
+
for retry_num in range(1, max_retries + 1):
|
|
1193
|
+
# Find tasks that failed
|
|
1194
|
+
failed_tasks = [
|
|
1195
|
+
tid for tid in batch.task_ids
|
|
1196
|
+
if batch.results.get(tid) in failed_statuses
|
|
1197
|
+
]
|
|
1198
|
+
|
|
1199
|
+
if not failed_tasks:
|
|
1200
|
+
# All tasks succeeded, no retries needed
|
|
1201
|
+
break
|
|
1202
|
+
|
|
1203
|
+
print(f"\n{'='*60}")
|
|
1204
|
+
print(f"Retry attempt {retry_num}/{max_retries}: {len(failed_tasks)} failed task(s)")
|
|
1205
|
+
print(f"{'='*60}")
|
|
1206
|
+
|
|
1207
|
+
# Emit retry event
|
|
1208
|
+
events.emit_for_workspace(
|
|
1209
|
+
workspace,
|
|
1210
|
+
events.EventType.BATCH_STARTED, # Reuse for retry
|
|
1211
|
+
{
|
|
1212
|
+
"batch_id": batch.id,
|
|
1213
|
+
"task_ids": failed_tasks,
|
|
1214
|
+
"retry_attempt": retry_num,
|
|
1215
|
+
"max_retries": max_retries,
|
|
1216
|
+
},
|
|
1217
|
+
print_event=True,
|
|
1218
|
+
)
|
|
1219
|
+
|
|
1220
|
+
if on_event:
|
|
1221
|
+
on_event("batch_retry_started", {
|
|
1222
|
+
"batch_id": batch.id,
|
|
1223
|
+
"retry_attempt": retry_num,
|
|
1224
|
+
"task_count": len(failed_tasks),
|
|
1225
|
+
})
|
|
1226
|
+
|
|
1227
|
+
# Re-run each failed task
|
|
1228
|
+
for i, task_id in enumerate(failed_tasks):
|
|
1229
|
+
# Check if batch was cancelled
|
|
1230
|
+
current_batch = get_batch(workspace, batch.id)
|
|
1231
|
+
if current_batch and current_batch.status == BatchStatus.CANCELLED:
|
|
1232
|
+
return
|
|
1233
|
+
|
|
1234
|
+
task = tasks.get(workspace, task_id)
|
|
1235
|
+
task_title = task.title if task else task_id
|
|
1236
|
+
previous_status = batch.results.get(task_id, "N/A")
|
|
1237
|
+
|
|
1238
|
+
print(f"\n[Retry {retry_num}, {i + 1}/{len(failed_tasks)}] {task_id}: {task_title}")
|
|
1239
|
+
print(f" Previous: {previous_status}")
|
|
1240
|
+
|
|
1241
|
+
# Execute task (with isolation context)
|
|
1242
|
+
from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
|
|
1243
|
+
exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
|
|
1244
|
+
try:
|
|
1245
|
+
result_status = _execute_task_subprocess(
|
|
1246
|
+
workspace, task_id, batch.id, engine=batch.engine,
|
|
1247
|
+
stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
|
|
1248
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1249
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1250
|
+
)
|
|
1251
|
+
finally:
|
|
1252
|
+
exec_ctx.cleanup()
|
|
1253
|
+
|
|
1254
|
+
# Update result
|
|
1255
|
+
batch.results[task_id] = result_status
|
|
1256
|
+
_save_batch(workspace, batch)
|
|
1257
|
+
|
|
1258
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
1259
|
+
print(f" ✓ Succeeded on retry {retry_num}")
|
|
1260
|
+
else:
|
|
1261
|
+
remaining = max_retries - retry_num
|
|
1262
|
+
if remaining > 0:
|
|
1263
|
+
print(f" ✗ Still failed ({remaining} retries left)")
|
|
1264
|
+
else:
|
|
1265
|
+
print(f" ✗ Failed after {max_retries} retries")
|
|
1266
|
+
|
|
1267
|
+
if on_event:
|
|
1268
|
+
on_event("batch_task_retried", {
|
|
1269
|
+
"task_id": task_id,
|
|
1270
|
+
"retry_attempt": retry_num,
|
|
1271
|
+
"status": result_status,
|
|
1272
|
+
})
|
|
1273
|
+
|
|
1274
|
+
# Recalculate final batch status after all retries
|
|
1275
|
+
total = len(batch.task_ids)
|
|
1276
|
+
final_completed = sum(1 for s in batch.results.values() if s == RunStatus.COMPLETED.value)
|
|
1277
|
+
final_failed = sum(1 for s in batch.results.values() if s == RunStatus.FAILED.value)
|
|
1278
|
+
final_blocked = sum(1 for s in batch.results.values() if s == RunStatus.BLOCKED.value)
|
|
1279
|
+
|
|
1280
|
+
if final_completed == total:
|
|
1281
|
+
batch.status = BatchStatus.COMPLETED
|
|
1282
|
+
event_type = events.EventType.BATCH_COMPLETED
|
|
1283
|
+
|
|
1284
|
+
# Run batch-level validation (full gate sweep)
|
|
1285
|
+
validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
|
|
1286
|
+
|
|
1287
|
+
if not validation_passed:
|
|
1288
|
+
# Gates failed - change status to PARTIAL (tasks done, integration broken)
|
|
1289
|
+
batch.status = BatchStatus.PARTIAL
|
|
1290
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1291
|
+
print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
|
|
1292
|
+
print(f"Validation error: {validation_error}")
|
|
1293
|
+
|
|
1294
|
+
elif final_completed == 0 and (final_failed > 0 or final_blocked > 0):
|
|
1295
|
+
batch.status = BatchStatus.FAILED
|
|
1296
|
+
event_type = events.EventType.BATCH_FAILED
|
|
1297
|
+
elif final_completed > 0:
|
|
1298
|
+
batch.status = BatchStatus.PARTIAL
|
|
1299
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1300
|
+
else:
|
|
1301
|
+
batch.status = BatchStatus.CANCELLED
|
|
1302
|
+
event_type = events.EventType.BATCH_CANCELLED
|
|
1303
|
+
|
|
1304
|
+
batch.completed_at = _utc_now()
|
|
1305
|
+
_save_batch(workspace, batch)
|
|
1306
|
+
|
|
1307
|
+
# Emit final status event
|
|
1308
|
+
events.emit_for_workspace(
|
|
1309
|
+
workspace,
|
|
1310
|
+
event_type,
|
|
1311
|
+
{
|
|
1312
|
+
"batch_id": batch.id,
|
|
1313
|
+
"completed": final_completed,
|
|
1314
|
+
"failed": final_failed,
|
|
1315
|
+
"blocked": final_blocked,
|
|
1316
|
+
"total": total,
|
|
1317
|
+
"retries_used": max_retries,
|
|
1318
|
+
},
|
|
1319
|
+
print_event=True,
|
|
1320
|
+
)
|
|
1321
|
+
_dispatch_batch_completed_webhook(workspace, event_type, batch.id, final_completed)
|
|
1322
|
+
|
|
1323
|
+
# Print retry summary
|
|
1324
|
+
if final_failed == 0:
|
|
1325
|
+
print("\n✓ All tasks succeeded after retries")
|
|
1326
|
+
else:
|
|
1327
|
+
print(f"\n⚠ {final_failed} task(s) still failing after {max_retries} retries")
|
|
1328
|
+
|
|
1329
|
+
|
|
1330
|
+
def _run_batch_level_validation(workspace: Workspace, batch: BatchRun) -> tuple[bool, Optional[str]]:
|
|
1331
|
+
"""Run full gate sweep after all tasks complete to catch cross-task inconsistencies.
|
|
1332
|
+
|
|
1333
|
+
Args:
|
|
1334
|
+
workspace: The workspace
|
|
1335
|
+
batch: The batch run that just completed
|
|
1336
|
+
|
|
1337
|
+
Returns:
|
|
1338
|
+
Tuple of (passed: bool, failure_summary: Optional[str])
|
|
1339
|
+
- passed=True means all gates passed
|
|
1340
|
+
- passed=False means gates failed, with summary of failures
|
|
1341
|
+
"""
|
|
1342
|
+
from codeframe.core import gates
|
|
1343
|
+
|
|
1344
|
+
print("\n[Conductor] Running batch-level validation (full gate sweep)...")
|
|
1345
|
+
|
|
1346
|
+
# Run all auto-detected gates against the full workspace
|
|
1347
|
+
result = gates.run(workspace, gates=None, verbose=False, auto_install_deps=True)
|
|
1348
|
+
|
|
1349
|
+
if result.passed:
|
|
1350
|
+
print("[Conductor] ✓ Batch-level validation passed")
|
|
1351
|
+
return True, None
|
|
1352
|
+
else:
|
|
1353
|
+
# Extract failure summary
|
|
1354
|
+
failure_summary = result.get_error_summary()
|
|
1355
|
+
if not failure_summary:
|
|
1356
|
+
failure_summary = "Gates failed (see individual gate outputs)"
|
|
1357
|
+
|
|
1358
|
+
print(f"[Conductor] ✗ Batch-level validation failed:\n{failure_summary}")
|
|
1359
|
+
|
|
1360
|
+
# Emit batch validation failed event
|
|
1361
|
+
events.emit_for_workspace(
|
|
1362
|
+
workspace,
|
|
1363
|
+
events.EventType.BATCH_VALIDATION_FAILED,
|
|
1364
|
+
{
|
|
1365
|
+
"batch_id": batch.id,
|
|
1366
|
+
"failed_gates": [check.name for check in result.checks if check.status == gates.GateStatus.FAILED],
|
|
1367
|
+
"summary": failure_summary[:500], # Truncate for event payload
|
|
1368
|
+
},
|
|
1369
|
+
print_event=True,
|
|
1370
|
+
)
|
|
1371
|
+
|
|
1372
|
+
return False, failure_summary
|
|
1373
|
+
|
|
1374
|
+
|
|
1375
|
+
def _apply_pending_config_reload(
|
|
1376
|
+
batch: BatchRun,
|
|
1377
|
+
workspace: Workspace,
|
|
1378
|
+
reload_state: "ConfigReloadState",
|
|
1379
|
+
last_seen_reload: datetime,
|
|
1380
|
+
) -> datetime:
|
|
1381
|
+
"""Check for config reloads and record them in batch results.
|
|
1382
|
+
|
|
1383
|
+
Args:
|
|
1384
|
+
batch: Current batch run.
|
|
1385
|
+
workspace: Target workspace.
|
|
1386
|
+
reload_state: Shared reload state from ConfigFileWatcher.
|
|
1387
|
+
last_seen_reload: Timestamp of the last check.
|
|
1388
|
+
|
|
1389
|
+
Returns:
|
|
1390
|
+
Updated last_seen_reload timestamp.
|
|
1391
|
+
"""
|
|
1392
|
+
if reload_state.has_reloaded_since(last_seen_reload):
|
|
1393
|
+
now = datetime.now(timezone.utc)
|
|
1394
|
+
reloads = batch.results.setdefault("__config_reloads__", [])
|
|
1395
|
+
reloads.append(now.isoformat())
|
|
1396
|
+
_save_batch(workspace, batch)
|
|
1397
|
+
print(f" [config] Configuration reloaded at {now.strftime('%H:%M:%S')}")
|
|
1398
|
+
return now
|
|
1399
|
+
return last_seen_reload
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def _execute_serial(
|
|
1403
|
+
workspace: Workspace,
|
|
1404
|
+
batch: BatchRun,
|
|
1405
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1406
|
+
) -> None:
|
|
1407
|
+
"""Execute tasks serially (one at a time).
|
|
1408
|
+
|
|
1409
|
+
Updates batch.results and batch.status as tasks complete.
|
|
1410
|
+
"""
|
|
1411
|
+
# Start reconciliation thread for continuous state checking
|
|
1412
|
+
from codeframe.core.config import load_environment_config
|
|
1413
|
+
env_config = load_environment_config(workspace.repo_path)
|
|
1414
|
+
interval = env_config.reconciliation_interval_seconds if env_config else 30
|
|
1415
|
+
reconcile_stop = _start_reconciliation_thread(workspace, batch, interval_seconds=interval)
|
|
1416
|
+
|
|
1417
|
+
# Start config file watcher (optional — failure must not break batch)
|
|
1418
|
+
config_watcher = None
|
|
1419
|
+
reload_state = None
|
|
1420
|
+
_last_seen_reload = datetime.now(timezone.utc)
|
|
1421
|
+
try:
|
|
1422
|
+
from codeframe.core.config_watcher import ConfigFileWatcher
|
|
1423
|
+
from codeframe.core.agents_config import load_preferences
|
|
1424
|
+
|
|
1425
|
+
config_watcher = ConfigFileWatcher(Path(workspace.repo_path))
|
|
1426
|
+
reload_state = config_watcher.start(load_preferences(Path(workspace.repo_path)))
|
|
1427
|
+
except Exception:
|
|
1428
|
+
logger.debug("Config watcher failed to start in serial execution", exc_info=True)
|
|
1429
|
+
|
|
1430
|
+
completed_count = 0
|
|
1431
|
+
failed_count = 0
|
|
1432
|
+
blocked_count = 0
|
|
1433
|
+
|
|
1434
|
+
try:
|
|
1435
|
+
for i, task_id in enumerate(batch.task_ids):
|
|
1436
|
+
# Check if batch was cancelled
|
|
1437
|
+
current_batch = get_batch(workspace, batch.id)
|
|
1438
|
+
if current_batch and current_batch.status == BatchStatus.CANCELLED:
|
|
1439
|
+
break
|
|
1440
|
+
|
|
1441
|
+
# Check for config reloads between tasks
|
|
1442
|
+
if reload_state is not None:
|
|
1443
|
+
_last_seen_reload = _apply_pending_config_reload(
|
|
1444
|
+
batch, workspace, reload_state, _last_seen_reload
|
|
1445
|
+
)
|
|
1446
|
+
|
|
1447
|
+
# Emit task queued event
|
|
1448
|
+
events.emit_for_workspace(
|
|
1449
|
+
workspace,
|
|
1450
|
+
events.EventType.BATCH_TASK_QUEUED,
|
|
1451
|
+
{"batch_id": batch.id, "task_id": task_id, "position": i + 1},
|
|
1452
|
+
print_event=True,
|
|
1453
|
+
)
|
|
1454
|
+
|
|
1455
|
+
# Get task info for display
|
|
1456
|
+
task = tasks.get(workspace, task_id)
|
|
1457
|
+
task_title = task.title if task else task_id
|
|
1458
|
+
|
|
1459
|
+
print(f"\n[{i + 1}/{len(batch.task_ids)}] Starting task {task_id}: {task_title}")
|
|
1460
|
+
|
|
1461
|
+
# Emit task started event
|
|
1462
|
+
events.emit_for_workspace(
|
|
1463
|
+
workspace,
|
|
1464
|
+
events.EventType.BATCH_TASK_STARTED,
|
|
1465
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1466
|
+
print_event=True,
|
|
1467
|
+
)
|
|
1468
|
+
|
|
1469
|
+
if on_event:
|
|
1470
|
+
on_event("batch_task_started", {"task_id": task_id, "position": i + 1})
|
|
1471
|
+
|
|
1472
|
+
# Execute task via subprocess (with isolation context)
|
|
1473
|
+
from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
|
|
1474
|
+
exec_ctx = create_execution_context(task_id, IsolationLevel(batch.isolation), workspace.repo_path)
|
|
1475
|
+
try:
|
|
1476
|
+
result_status = _execute_task_subprocess(
|
|
1477
|
+
workspace, task_id, batch.id, engine=batch.engine,
|
|
1478
|
+
stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
|
|
1479
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1480
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1481
|
+
)
|
|
1482
|
+
|
|
1483
|
+
# If task is BLOCKED, try supervisor resolution
|
|
1484
|
+
if result_status == RunStatus.BLOCKED.value:
|
|
1485
|
+
supervisor = get_supervisor(workspace)
|
|
1486
|
+
if supervisor.try_resolve_blocked_task(task_id):
|
|
1487
|
+
# Supervisor resolved the blocker - retry the task
|
|
1488
|
+
print(" [Supervisor] Retrying task after auto-resolution...")
|
|
1489
|
+
result_status = _execute_task_subprocess(
|
|
1490
|
+
workspace, task_id, batch.id, engine=batch.engine,
|
|
1491
|
+
stall_timeout_s=batch.stall_timeout_s, stall_action=batch.stall_action,
|
|
1492
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1493
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1494
|
+
)
|
|
1495
|
+
finally:
|
|
1496
|
+
exec_ctx.cleanup()
|
|
1497
|
+
|
|
1498
|
+
# Record result
|
|
1499
|
+
batch.results[task_id] = result_status
|
|
1500
|
+
_save_batch(workspace, batch)
|
|
1501
|
+
|
|
1502
|
+
# Emit appropriate event based on result
|
|
1503
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
1504
|
+
completed_count += 1
|
|
1505
|
+
events.emit_for_workspace(
|
|
1506
|
+
workspace,
|
|
1507
|
+
events.EventType.BATCH_TASK_COMPLETED,
|
|
1508
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1509
|
+
print_event=True,
|
|
1510
|
+
)
|
|
1511
|
+
print(" ✓ Completed")
|
|
1512
|
+
elif result_status == RunStatus.BLOCKED.value:
|
|
1513
|
+
blocked_count += 1
|
|
1514
|
+
events.emit_for_workspace(
|
|
1515
|
+
workspace,
|
|
1516
|
+
events.EventType.BATCH_TASK_BLOCKED,
|
|
1517
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1518
|
+
print_event=True,
|
|
1519
|
+
)
|
|
1520
|
+
print(" ⊘ Blocked (requires human input)")
|
|
1521
|
+
else:
|
|
1522
|
+
failed_count += 1
|
|
1523
|
+
events.emit_for_workspace(
|
|
1524
|
+
workspace,
|
|
1525
|
+
events.EventType.BATCH_TASK_FAILED,
|
|
1526
|
+
{"batch_id": batch.id, "task_id": task_id, "status": result_status},
|
|
1527
|
+
print_event=True,
|
|
1528
|
+
)
|
|
1529
|
+
print(f" ✗ Failed: {result_status}")
|
|
1530
|
+
|
|
1531
|
+
# Check on_failure behavior
|
|
1532
|
+
if batch.on_failure == OnFailure.STOP:
|
|
1533
|
+
print("\nStopping batch due to --on-failure=stop")
|
|
1534
|
+
break
|
|
1535
|
+
|
|
1536
|
+
if on_event:
|
|
1537
|
+
on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
|
|
1538
|
+
|
|
1539
|
+
# Determine final batch status
|
|
1540
|
+
total = len(batch.task_ids)
|
|
1541
|
+
completed_count + failed_count + blocked_count
|
|
1542
|
+
|
|
1543
|
+
if completed_count == total:
|
|
1544
|
+
batch.status = BatchStatus.COMPLETED
|
|
1545
|
+
event_type = events.EventType.BATCH_COMPLETED
|
|
1546
|
+
|
|
1547
|
+
# Run batch-level validation (full gate sweep)
|
|
1548
|
+
validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
|
|
1549
|
+
|
|
1550
|
+
if not validation_passed:
|
|
1551
|
+
# Gates failed - change status to PARTIAL (tasks done, integration broken)
|
|
1552
|
+
batch.status = BatchStatus.PARTIAL
|
|
1553
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1554
|
+
print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
|
|
1555
|
+
print(f"Validation error: {validation_error}")
|
|
1556
|
+
|
|
1557
|
+
elif completed_count == 0 and (failed_count > 0 or blocked_count > 0):
|
|
1558
|
+
batch.status = BatchStatus.FAILED
|
|
1559
|
+
event_type = events.EventType.BATCH_FAILED
|
|
1560
|
+
elif completed_count > 0:
|
|
1561
|
+
batch.status = BatchStatus.PARTIAL
|
|
1562
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1563
|
+
else:
|
|
1564
|
+
# Nothing executed (e.g., cancelled before start)
|
|
1565
|
+
batch.status = BatchStatus.CANCELLED
|
|
1566
|
+
event_type = events.EventType.BATCH_CANCELLED
|
|
1567
|
+
|
|
1568
|
+
batch.completed_at = _utc_now()
|
|
1569
|
+
_save_batch(workspace, batch)
|
|
1570
|
+
|
|
1571
|
+
# Emit batch completion event
|
|
1572
|
+
events.emit_for_workspace(
|
|
1573
|
+
workspace,
|
|
1574
|
+
event_type,
|
|
1575
|
+
{
|
|
1576
|
+
"batch_id": batch.id,
|
|
1577
|
+
"completed": completed_count,
|
|
1578
|
+
"failed": failed_count,
|
|
1579
|
+
"blocked": blocked_count,
|
|
1580
|
+
"total": total,
|
|
1581
|
+
},
|
|
1582
|
+
print_event=True,
|
|
1583
|
+
)
|
|
1584
|
+
_dispatch_batch_completed_webhook(workspace, event_type, batch.id, completed_count)
|
|
1585
|
+
|
|
1586
|
+
# Stop reconciliation thread
|
|
1587
|
+
reconcile_stop.set()
|
|
1588
|
+
|
|
1589
|
+
# Print summary
|
|
1590
|
+
print(f"\nBatch {batch.status.value.lower()}: {completed_count}/{total} tasks completed")
|
|
1591
|
+
if failed_count > 0:
|
|
1592
|
+
print(f" Failed: {failed_count}")
|
|
1593
|
+
if blocked_count > 0:
|
|
1594
|
+
print(f" Blocked: {blocked_count}")
|
|
1595
|
+
finally:
|
|
1596
|
+
if config_watcher is not None:
|
|
1597
|
+
config_watcher.stop()
|
|
1598
|
+
|
|
1599
|
+
|
|
1600
|
+
def _execute_parallel(
|
|
1601
|
+
workspace: Workspace,
|
|
1602
|
+
batch: BatchRun,
|
|
1603
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1604
|
+
) -> None:
|
|
1605
|
+
"""Execute tasks in parallel using dependency-aware groups.
|
|
1606
|
+
|
|
1607
|
+
Creates an execution plan from task dependencies and executes each group
|
|
1608
|
+
in sequence. Tasks within a group run in parallel, up to max_parallel.
|
|
1609
|
+
|
|
1610
|
+
Args:
|
|
1611
|
+
workspace: Target workspace
|
|
1612
|
+
batch: BatchRun to execute
|
|
1613
|
+
on_event: Optional callback for batch events
|
|
1614
|
+
|
|
1615
|
+
Raises:
|
|
1616
|
+
CycleDetectedError: If circular dependencies are detected
|
|
1617
|
+
"""
|
|
1618
|
+
# Clean up orphaned worktrees from crashed workers on previous runs
|
|
1619
|
+
from codeframe.core.sandbox.context import IsolationLevel as _IL
|
|
1620
|
+
if batch.isolation == _IL.WORKTREE.value:
|
|
1621
|
+
from codeframe.core.worktrees import WorktreeRegistry
|
|
1622
|
+
WorktreeRegistry().cleanup_stale(workspace.repo_path)
|
|
1623
|
+
|
|
1624
|
+
# Create execution plan based on dependencies
|
|
1625
|
+
plan = create_execution_plan(workspace, batch.task_ids)
|
|
1626
|
+
|
|
1627
|
+
print(f"\nExecution plan: {plan.num_groups} groups, {plan.total_tasks} tasks")
|
|
1628
|
+
if plan.can_run_parallel():
|
|
1629
|
+
print(f"Parallelizable groups found - using max {batch.max_parallel} workers")
|
|
1630
|
+
else:
|
|
1631
|
+
print("All tasks are sequential (chain dependencies)")
|
|
1632
|
+
|
|
1633
|
+
# Start reconciliation thread for continuous state checking
|
|
1634
|
+
from codeframe.core.config import load_environment_config as _load_env_config
|
|
1635
|
+
_env_config_p = _load_env_config(workspace.repo_path)
|
|
1636
|
+
_interval_p = _env_config_p.reconciliation_interval_seconds if _env_config_p else 30
|
|
1637
|
+
_reconcile_stop_p = _start_reconciliation_thread(workspace, batch, interval_seconds=_interval_p)
|
|
1638
|
+
|
|
1639
|
+
# Start config file watcher (optional — failure must not break batch)
|
|
1640
|
+
config_watcher_p = None
|
|
1641
|
+
reload_state_p = None
|
|
1642
|
+
_last_seen_reload_p = datetime.now(timezone.utc)
|
|
1643
|
+
try:
|
|
1644
|
+
from codeframe.core.config_watcher import ConfigFileWatcher as _CFW
|
|
1645
|
+
from codeframe.core.agents_config import load_preferences as _load_prefs
|
|
1646
|
+
|
|
1647
|
+
config_watcher_p = _CFW(Path(workspace.repo_path))
|
|
1648
|
+
reload_state_p = config_watcher_p.start(_load_prefs(Path(workspace.repo_path)))
|
|
1649
|
+
except Exception:
|
|
1650
|
+
logger.debug("Config watcher failed to start in parallel execution", exc_info=True)
|
|
1651
|
+
|
|
1652
|
+
completed_count = 0
|
|
1653
|
+
failed_count = 0
|
|
1654
|
+
blocked_count = 0
|
|
1655
|
+
task_index = 0 # Global task counter for progress display
|
|
1656
|
+
|
|
1657
|
+
try:
|
|
1658
|
+
for group_idx, group in enumerate(plan.groups):
|
|
1659
|
+
# Check if batch was cancelled
|
|
1660
|
+
current_batch = get_batch(workspace, batch.id)
|
|
1661
|
+
if current_batch and current_batch.status == BatchStatus.CANCELLED:
|
|
1662
|
+
break
|
|
1663
|
+
|
|
1664
|
+
# Check for config reloads between groups
|
|
1665
|
+
if reload_state_p is not None:
|
|
1666
|
+
_last_seen_reload_p = _apply_pending_config_reload(
|
|
1667
|
+
batch, workspace, reload_state_p, _last_seen_reload_p
|
|
1668
|
+
)
|
|
1669
|
+
|
|
1670
|
+
# Check if any previous failure should stop execution
|
|
1671
|
+
if batch.on_failure == OnFailure.STOP and failed_count > 0:
|
|
1672
|
+
print("\nStopping batch due to --on-failure=stop")
|
|
1673
|
+
break
|
|
1674
|
+
|
|
1675
|
+
group_size = len(group)
|
|
1676
|
+
print(f"\n{'─'*60}")
|
|
1677
|
+
print(f"Group {group_idx + 1}/{plan.num_groups}: {group_size} task(s)")
|
|
1678
|
+
|
|
1679
|
+
if group_size == 1:
|
|
1680
|
+
# Single task - run directly
|
|
1681
|
+
task_id = group[0]
|
|
1682
|
+
task_index += 1
|
|
1683
|
+
result = _execute_single_task(
|
|
1684
|
+
workspace, batch, task_id, task_index, len(batch.task_ids), on_event
|
|
1685
|
+
)
|
|
1686
|
+
if result == RunStatus.COMPLETED.value:
|
|
1687
|
+
completed_count += 1
|
|
1688
|
+
elif result == RunStatus.BLOCKED.value:
|
|
1689
|
+
blocked_count += 1
|
|
1690
|
+
else:
|
|
1691
|
+
failed_count += 1
|
|
1692
|
+
else:
|
|
1693
|
+
# Multiple tasks - run in parallel (use per-status limits if configured)
|
|
1694
|
+
if batch.concurrency.by_status:
|
|
1695
|
+
group_statuses = []
|
|
1696
|
+
for tid in group:
|
|
1697
|
+
t = tasks.get(workspace, tid)
|
|
1698
|
+
if t:
|
|
1699
|
+
group_statuses.append(t.status.value)
|
|
1700
|
+
effective_workers = batch.concurrency.effective_workers(
|
|
1701
|
+
statuses=group_statuses, group_size=group_size, global_running=0,
|
|
1702
|
+
)
|
|
1703
|
+
else:
|
|
1704
|
+
effective_workers = min(group_size, batch.max_parallel)
|
|
1705
|
+
print(f"Running {group_size} tasks with {effective_workers} workers")
|
|
1706
|
+
|
|
1707
|
+
# Execute group in parallel
|
|
1708
|
+
results = _execute_group_parallel(
|
|
1709
|
+
workspace, batch, group, task_index, len(batch.task_ids),
|
|
1710
|
+
effective_workers, on_event
|
|
1711
|
+
)
|
|
1712
|
+
|
|
1713
|
+
# Process results
|
|
1714
|
+
for task_id, result_status in results.items():
|
|
1715
|
+
task_index += 1
|
|
1716
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
1717
|
+
completed_count += 1
|
|
1718
|
+
elif result_status == RunStatus.BLOCKED.value:
|
|
1719
|
+
blocked_count += 1
|
|
1720
|
+
else:
|
|
1721
|
+
failed_count += 1
|
|
1722
|
+
|
|
1723
|
+
# Check stop on failure within parallel group
|
|
1724
|
+
if batch.on_failure == OnFailure.STOP and failed_count > 0:
|
|
1725
|
+
# Can't stop mid-group, but will stop after group completes
|
|
1726
|
+
pass
|
|
1727
|
+
|
|
1728
|
+
# Determine final batch status
|
|
1729
|
+
total = len(batch.task_ids)
|
|
1730
|
+
completed_count + failed_count + blocked_count
|
|
1731
|
+
|
|
1732
|
+
if completed_count == total:
|
|
1733
|
+
batch.status = BatchStatus.COMPLETED
|
|
1734
|
+
event_type = events.EventType.BATCH_COMPLETED
|
|
1735
|
+
|
|
1736
|
+
# Run batch-level validation (full gate sweep)
|
|
1737
|
+
validation_passed, validation_error = _run_batch_level_validation(workspace, batch)
|
|
1738
|
+
|
|
1739
|
+
if not validation_passed:
|
|
1740
|
+
# Gates failed - change status to PARTIAL (tasks done, integration broken)
|
|
1741
|
+
batch.status = BatchStatus.PARTIAL
|
|
1742
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1743
|
+
print("\n⚠️ Batch marked PARTIAL due to failed batch-level gates")
|
|
1744
|
+
print(f"Validation error: {validation_error}")
|
|
1745
|
+
|
|
1746
|
+
elif completed_count == 0 and (failed_count > 0 or blocked_count > 0):
|
|
1747
|
+
batch.status = BatchStatus.FAILED
|
|
1748
|
+
event_type = events.EventType.BATCH_FAILED
|
|
1749
|
+
elif completed_count > 0:
|
|
1750
|
+
batch.status = BatchStatus.PARTIAL
|
|
1751
|
+
event_type = events.EventType.BATCH_PARTIAL
|
|
1752
|
+
else:
|
|
1753
|
+
batch.status = BatchStatus.CANCELLED
|
|
1754
|
+
event_type = events.EventType.BATCH_CANCELLED
|
|
1755
|
+
|
|
1756
|
+
batch.completed_at = _utc_now()
|
|
1757
|
+
_save_batch(workspace, batch)
|
|
1758
|
+
|
|
1759
|
+
# Emit batch completion event
|
|
1760
|
+
events.emit_for_workspace(
|
|
1761
|
+
workspace,
|
|
1762
|
+
event_type,
|
|
1763
|
+
{
|
|
1764
|
+
"batch_id": batch.id,
|
|
1765
|
+
"completed": completed_count,
|
|
1766
|
+
"failed": failed_count,
|
|
1767
|
+
"blocked": blocked_count,
|
|
1768
|
+
"total": total,
|
|
1769
|
+
"strategy": "parallel",
|
|
1770
|
+
"groups": plan.num_groups,
|
|
1771
|
+
},
|
|
1772
|
+
print_event=True,
|
|
1773
|
+
)
|
|
1774
|
+
_dispatch_batch_completed_webhook(workspace, event_type, batch.id, completed_count)
|
|
1775
|
+
|
|
1776
|
+
# Stop reconciliation thread
|
|
1777
|
+
_reconcile_stop_p.set()
|
|
1778
|
+
|
|
1779
|
+
# Print summary
|
|
1780
|
+
print(f"\nBatch {batch.status.value.lower()}: {completed_count}/{total} tasks completed")
|
|
1781
|
+
print(f" Execution: {plan.num_groups} groups (parallel strategy)")
|
|
1782
|
+
if failed_count > 0:
|
|
1783
|
+
print(f" Failed: {failed_count}")
|
|
1784
|
+
if blocked_count > 0:
|
|
1785
|
+
print(f" Blocked: {blocked_count}")
|
|
1786
|
+
finally:
|
|
1787
|
+
if config_watcher_p is not None:
|
|
1788
|
+
config_watcher_p.stop()
|
|
1789
|
+
|
|
1790
|
+
|
|
1791
|
+
def _start_reconciliation_thread(
|
|
1792
|
+
workspace: Workspace,
|
|
1793
|
+
batch: BatchRun,
|
|
1794
|
+
interval_seconds: int = 30,
|
|
1795
|
+
github_checker: Optional[Callable] = None,
|
|
1796
|
+
) -> threading.Event:
|
|
1797
|
+
"""Start a daemon thread that periodically reconciles batch state.
|
|
1798
|
+
|
|
1799
|
+
The thread checks all active tasks for external state changes every
|
|
1800
|
+
``interval_seconds`` and applies adjustments (skip completed tasks,
|
|
1801
|
+
re-queue unblocked tasks).
|
|
1802
|
+
|
|
1803
|
+
Returns a threading.Event that can be set to stop the thread.
|
|
1804
|
+
"""
|
|
1805
|
+
from codeframe.core.reconciliation import ReconciliationEngine
|
|
1806
|
+
|
|
1807
|
+
stop_event = threading.Event()
|
|
1808
|
+
engine = ReconciliationEngine(workspace, github_checker=github_checker)
|
|
1809
|
+
|
|
1810
|
+
def _loop() -> None:
|
|
1811
|
+
while not stop_event.wait(timeout=interval_seconds):
|
|
1812
|
+
try:
|
|
1813
|
+
# Get currently active task IDs from the batch
|
|
1814
|
+
active_ids = [
|
|
1815
|
+
tid for tid in batch.task_ids
|
|
1816
|
+
if batch.results.get(tid) is None
|
|
1817
|
+
or batch.results.get(tid) == "RUNNING"
|
|
1818
|
+
]
|
|
1819
|
+
if not active_ids:
|
|
1820
|
+
continue
|
|
1821
|
+
|
|
1822
|
+
result = engine.check_all_active(active_ids)
|
|
1823
|
+
if result.changes_detected:
|
|
1824
|
+
with _active_processes_lock:
|
|
1825
|
+
procs = _active_processes.get(batch.id, {})
|
|
1826
|
+
engine.apply_changes(result, batch, procs)
|
|
1827
|
+
|
|
1828
|
+
# Emit events for changes
|
|
1829
|
+
for tid in result.tasks_skipped:
|
|
1830
|
+
events.emit_for_workspace(
|
|
1831
|
+
workspace,
|
|
1832
|
+
events.EventType.RECONCILIATION_TASK_SKIPPED,
|
|
1833
|
+
{"batch_id": batch.id, "task_id": tid},
|
|
1834
|
+
)
|
|
1835
|
+
for tid in result.tasks_requeued:
|
|
1836
|
+
events.emit_for_workspace(
|
|
1837
|
+
workspace,
|
|
1838
|
+
events.EventType.RECONCILIATION_TASK_REQUEUED,
|
|
1839
|
+
{"batch_id": batch.id, "task_id": tid},
|
|
1840
|
+
)
|
|
1841
|
+
if result.errors:
|
|
1842
|
+
for err in result.errors:
|
|
1843
|
+
events.emit_for_workspace(
|
|
1844
|
+
workspace,
|
|
1845
|
+
events.EventType.RECONCILIATION_ERROR,
|
|
1846
|
+
{"batch_id": batch.id, "error": err},
|
|
1847
|
+
)
|
|
1848
|
+
except Exception as exc:
|
|
1849
|
+
logger.warning("Reconciliation loop error: %s", exc)
|
|
1850
|
+
|
|
1851
|
+
thread = threading.Thread(target=_loop, daemon=True, name=f"reconcile-{batch.id[:8]}")
|
|
1852
|
+
thread.start()
|
|
1853
|
+
return stop_event
|
|
1854
|
+
|
|
1855
|
+
|
|
1856
|
+
def _execute_single_task(
|
|
1857
|
+
workspace: Workspace,
|
|
1858
|
+
batch: BatchRun,
|
|
1859
|
+
task_id: str,
|
|
1860
|
+
position: int,
|
|
1861
|
+
total: int,
|
|
1862
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1863
|
+
) -> str:
|
|
1864
|
+
"""Execute a single task and update batch results.
|
|
1865
|
+
|
|
1866
|
+
Helper for parallel execution that handles one task.
|
|
1867
|
+
|
|
1868
|
+
Returns:
|
|
1869
|
+
RunStatus value string
|
|
1870
|
+
"""
|
|
1871
|
+
# Emit task queued event
|
|
1872
|
+
events.emit_for_workspace(
|
|
1873
|
+
workspace,
|
|
1874
|
+
events.EventType.BATCH_TASK_QUEUED,
|
|
1875
|
+
{"batch_id": batch.id, "task_id": task_id, "position": position},
|
|
1876
|
+
print_event=True,
|
|
1877
|
+
)
|
|
1878
|
+
|
|
1879
|
+
# Get task info for display
|
|
1880
|
+
task = tasks.get(workspace, task_id)
|
|
1881
|
+
task_title = task.title if task else task_id
|
|
1882
|
+
|
|
1883
|
+
print(f"\n[{position}/{total}] Starting task {task_id}: {task_title}")
|
|
1884
|
+
|
|
1885
|
+
# Emit task started event
|
|
1886
|
+
events.emit_for_workspace(
|
|
1887
|
+
workspace,
|
|
1888
|
+
events.EventType.BATCH_TASK_STARTED,
|
|
1889
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1890
|
+
print_event=True,
|
|
1891
|
+
)
|
|
1892
|
+
|
|
1893
|
+
if on_event:
|
|
1894
|
+
on_event("batch_task_started", {"task_id": task_id, "position": position})
|
|
1895
|
+
|
|
1896
|
+
# Create execution context (handles isolation level; NONE is a no-op)
|
|
1897
|
+
from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
|
|
1898
|
+
exec_ctx = create_execution_context(
|
|
1899
|
+
task_id, IsolationLevel(batch.isolation), workspace.repo_path
|
|
1900
|
+
)
|
|
1901
|
+
|
|
1902
|
+
try:
|
|
1903
|
+
# Execute task via subprocess
|
|
1904
|
+
result_status = _execute_task_subprocess(
|
|
1905
|
+
workspace, task_id, batch.id,
|
|
1906
|
+
engine=batch.engine,
|
|
1907
|
+
stall_timeout_s=batch.stall_timeout_s,
|
|
1908
|
+
stall_action=batch.stall_action,
|
|
1909
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1910
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1911
|
+
)
|
|
1912
|
+
|
|
1913
|
+
# If task is BLOCKED, try supervisor resolution
|
|
1914
|
+
if result_status == RunStatus.BLOCKED.value:
|
|
1915
|
+
supervisor = get_supervisor(workspace)
|
|
1916
|
+
if supervisor.try_resolve_blocked_task(task_id):
|
|
1917
|
+
# Supervisor resolved the blocker - retry the task
|
|
1918
|
+
print(" [Supervisor] Retrying task after auto-resolution...")
|
|
1919
|
+
result_status = _execute_task_subprocess(
|
|
1920
|
+
workspace, task_id, batch.id,
|
|
1921
|
+
engine=batch.engine,
|
|
1922
|
+
stall_timeout_s=batch.stall_timeout_s,
|
|
1923
|
+
stall_action=batch.stall_action,
|
|
1924
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
1925
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
1926
|
+
)
|
|
1927
|
+
finally:
|
|
1928
|
+
exec_ctx.cleanup()
|
|
1929
|
+
|
|
1930
|
+
# Record result
|
|
1931
|
+
batch.results[task_id] = result_status
|
|
1932
|
+
_save_batch(workspace, batch)
|
|
1933
|
+
|
|
1934
|
+
# Emit appropriate event based on result
|
|
1935
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
1936
|
+
events.emit_for_workspace(
|
|
1937
|
+
workspace,
|
|
1938
|
+
events.EventType.BATCH_TASK_COMPLETED,
|
|
1939
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1940
|
+
print_event=True,
|
|
1941
|
+
)
|
|
1942
|
+
print(" ✓ Completed")
|
|
1943
|
+
elif result_status == RunStatus.BLOCKED.value:
|
|
1944
|
+
events.emit_for_workspace(
|
|
1945
|
+
workspace,
|
|
1946
|
+
events.EventType.BATCH_TASK_BLOCKED,
|
|
1947
|
+
{"batch_id": batch.id, "task_id": task_id},
|
|
1948
|
+
print_event=True,
|
|
1949
|
+
)
|
|
1950
|
+
print(" ⊘ Blocked (requires human input)")
|
|
1951
|
+
else:
|
|
1952
|
+
events.emit_for_workspace(
|
|
1953
|
+
workspace,
|
|
1954
|
+
events.EventType.BATCH_TASK_FAILED,
|
|
1955
|
+
{"batch_id": batch.id, "task_id": task_id, "status": result_status},
|
|
1956
|
+
print_event=True,
|
|
1957
|
+
)
|
|
1958
|
+
print(f" ✗ Failed: {result_status}")
|
|
1959
|
+
|
|
1960
|
+
if on_event:
|
|
1961
|
+
on_event("batch_task_completed", {"task_id": task_id, "status": result_status})
|
|
1962
|
+
|
|
1963
|
+
return result_status
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def _execute_group_parallel(
|
|
1967
|
+
workspace: Workspace,
|
|
1968
|
+
batch: BatchRun,
|
|
1969
|
+
group: list[str],
|
|
1970
|
+
start_index: int,
|
|
1971
|
+
total: int,
|
|
1972
|
+
max_workers: int,
|
|
1973
|
+
on_event: Optional[Callable[[str, dict], None]] = None,
|
|
1974
|
+
) -> dict[str, str]:
|
|
1975
|
+
"""Execute a group of tasks in parallel.
|
|
1976
|
+
|
|
1977
|
+
Args:
|
|
1978
|
+
workspace: Target workspace
|
|
1979
|
+
batch: BatchRun being executed
|
|
1980
|
+
group: List of task IDs to execute concurrently
|
|
1981
|
+
start_index: Starting task index for progress display
|
|
1982
|
+
total: Total tasks in batch
|
|
1983
|
+
max_workers: Maximum concurrent workers
|
|
1984
|
+
on_event: Optional callback for events
|
|
1985
|
+
|
|
1986
|
+
Returns:
|
|
1987
|
+
Dict mapping task_id -> RunStatus value
|
|
1988
|
+
"""
|
|
1989
|
+
results: dict[str, str] = {}
|
|
1990
|
+
|
|
1991
|
+
# Show which tasks are starting
|
|
1992
|
+
for i, task_id in enumerate(group):
|
|
1993
|
+
task = tasks.get(workspace, task_id)
|
|
1994
|
+
task_title = task.title if task else task_id
|
|
1995
|
+
print(f" [{start_index + i + 1}/{total}] Queued: {task_id}: {task_title}")
|
|
1996
|
+
|
|
1997
|
+
# Emit task queued event
|
|
1998
|
+
events.emit_for_workspace(
|
|
1999
|
+
workspace,
|
|
2000
|
+
events.EventType.BATCH_TASK_QUEUED,
|
|
2001
|
+
{"batch_id": batch.id, "task_id": task_id, "position": start_index + i + 1, "parallel": True},
|
|
2002
|
+
print_event=True,
|
|
2003
|
+
)
|
|
2004
|
+
|
|
2005
|
+
def execute_task(task_id: str) -> tuple[str, str]:
|
|
2006
|
+
"""Execute a single task and return (task_id, status)."""
|
|
2007
|
+
# Emit task started event
|
|
2008
|
+
events.emit_for_workspace(
|
|
2009
|
+
workspace,
|
|
2010
|
+
events.EventType.BATCH_TASK_STARTED,
|
|
2011
|
+
{"batch_id": batch.id, "task_id": task_id, "parallel": True},
|
|
2012
|
+
print_event=True,
|
|
2013
|
+
)
|
|
2014
|
+
|
|
2015
|
+
if on_event:
|
|
2016
|
+
on_event("batch_task_started", {"task_id": task_id, "parallel": True})
|
|
2017
|
+
|
|
2018
|
+
# Create execution context for this task
|
|
2019
|
+
from codeframe.core.sandbox.context import IsolationLevel, create_execution_context
|
|
2020
|
+
exec_ctx = create_execution_context(
|
|
2021
|
+
task_id, IsolationLevel(batch.isolation), workspace.repo_path
|
|
2022
|
+
)
|
|
2023
|
+
|
|
2024
|
+
try:
|
|
2025
|
+
# Execute via subprocess
|
|
2026
|
+
result_status = _execute_task_subprocess(
|
|
2027
|
+
workspace, task_id, batch.id,
|
|
2028
|
+
engine=batch.engine,
|
|
2029
|
+
stall_timeout_s=batch.stall_timeout_s,
|
|
2030
|
+
stall_action=batch.stall_action,
|
|
2031
|
+
worktree_path=exec_ctx.workspace_path if exec_ctx.workspace_path != workspace.repo_path else None,
|
|
2032
|
+
llm_provider=batch.llm_provider, llm_model=batch.llm_model,
|
|
2033
|
+
)
|
|
2034
|
+
finally:
|
|
2035
|
+
exec_ctx.cleanup()
|
|
2036
|
+
|
|
2037
|
+
# Record result (thread-safe due to GIL for simple dict operations)
|
|
2038
|
+
batch.results[task_id] = result_status
|
|
2039
|
+
_save_batch(workspace, batch)
|
|
2040
|
+
|
|
2041
|
+
return task_id, result_status
|
|
2042
|
+
|
|
2043
|
+
# Execute tasks in parallel using thread pool
|
|
2044
|
+
with ThreadPoolExecutor(max_workers=max_workers) as executor:
|
|
2045
|
+
futures = {executor.submit(execute_task, tid): tid for tid in group}
|
|
2046
|
+
|
|
2047
|
+
for future in as_completed(futures):
|
|
2048
|
+
task_id, result_status = future.result()
|
|
2049
|
+
results[task_id] = result_status
|
|
2050
|
+
|
|
2051
|
+
# Get task info for display
|
|
2052
|
+
task = tasks.get(workspace, task_id)
|
|
2053
|
+
task_title = task.title if task else task_id
|
|
2054
|
+
|
|
2055
|
+
# Emit appropriate event and print result
|
|
2056
|
+
if result_status == RunStatus.COMPLETED.value:
|
|
2057
|
+
events.emit_for_workspace(
|
|
2058
|
+
workspace,
|
|
2059
|
+
events.EventType.BATCH_TASK_COMPLETED,
|
|
2060
|
+
{"batch_id": batch.id, "task_id": task_id, "parallel": True},
|
|
2061
|
+
print_event=True,
|
|
2062
|
+
)
|
|
2063
|
+
print(f" ✓ {task_id}: Completed")
|
|
2064
|
+
elif result_status == RunStatus.BLOCKED.value:
|
|
2065
|
+
events.emit_for_workspace(
|
|
2066
|
+
workspace,
|
|
2067
|
+
events.EventType.BATCH_TASK_BLOCKED,
|
|
2068
|
+
{"batch_id": batch.id, "task_id": task_id, "parallel": True},
|
|
2069
|
+
print_event=True,
|
|
2070
|
+
)
|
|
2071
|
+
print(f" ⊘ {task_id}: Blocked")
|
|
2072
|
+
else:
|
|
2073
|
+
events.emit_for_workspace(
|
|
2074
|
+
workspace,
|
|
2075
|
+
events.EventType.BATCH_TASK_FAILED,
|
|
2076
|
+
{"batch_id": batch.id, "task_id": task_id, "status": result_status, "parallel": True},
|
|
2077
|
+
print_event=True,
|
|
2078
|
+
)
|
|
2079
|
+
print(f" ✗ {task_id}: Failed ({result_status})")
|
|
2080
|
+
|
|
2081
|
+
if on_event:
|
|
2082
|
+
on_event("batch_task_completed", {"task_id": task_id, "status": result_status, "parallel": True})
|
|
2083
|
+
|
|
2084
|
+
return results
|
|
2085
|
+
|
|
2086
|
+
|
|
2087
|
+
def _execute_task_subprocess(
|
|
2088
|
+
workspace: Workspace,
|
|
2089
|
+
task_id: str,
|
|
2090
|
+
batch_id: Optional[str] = None,
|
|
2091
|
+
engine: str = "react",
|
|
2092
|
+
stall_timeout_s: int = 300,
|
|
2093
|
+
stall_action: str = "blocker",
|
|
2094
|
+
worktree_path: Optional[Path] = None,
|
|
2095
|
+
cloud_timeout_minutes: int = 30,
|
|
2096
|
+
llm_provider: Optional[str] = None,
|
|
2097
|
+
llm_model: Optional[str] = None,
|
|
2098
|
+
) -> str:
|
|
2099
|
+
"""Execute a single task via subprocess.
|
|
2100
|
+
|
|
2101
|
+
Runs `cf work start <task_id> --execute --engine <engine>` as a subprocess.
|
|
2102
|
+
|
|
2103
|
+
Args:
|
|
2104
|
+
workspace: Target workspace
|
|
2105
|
+
task_id: Task to execute
|
|
2106
|
+
batch_id: Optional batch ID for process tracking (enables force stop)
|
|
2107
|
+
engine: Agent engine to use ("plan", "react", "cloud", etc.)
|
|
2108
|
+
stall_timeout_s: Stall detection timeout in seconds (0 = disabled)
|
|
2109
|
+
stall_action: Recovery action on stall ("blocker", "retry", or "fail")
|
|
2110
|
+
cloud_timeout_minutes: Sandbox timeout for cloud engine (1-60)
|
|
2111
|
+
|
|
2112
|
+
Returns:
|
|
2113
|
+
RunStatus value string (COMPLETED, FAILED, BLOCKED)
|
|
2114
|
+
"""
|
|
2115
|
+
# Build command
|
|
2116
|
+
cmd = [
|
|
2117
|
+
sys.executable, "-m", "codeframe.cli.app",
|
|
2118
|
+
"work", "start", task_id, "--execute",
|
|
2119
|
+
"--engine", engine,
|
|
2120
|
+
"--stall-timeout", str(stall_timeout_s),
|
|
2121
|
+
"--stall-action", stall_action,
|
|
2122
|
+
]
|
|
2123
|
+
if engine == "cloud":
|
|
2124
|
+
cmd += ["--cloud-timeout", str(cloud_timeout_minutes)]
|
|
2125
|
+
if llm_provider:
|
|
2126
|
+
cmd += ["--llm-provider", llm_provider]
|
|
2127
|
+
if llm_model:
|
|
2128
|
+
cmd += ["--llm-model", llm_model]
|
|
2129
|
+
|
|
2130
|
+
process = None
|
|
2131
|
+
try:
|
|
2132
|
+
# Use Popen instead of run for process tracking
|
|
2133
|
+
process = subprocess.Popen(
|
|
2134
|
+
cmd,
|
|
2135
|
+
cwd=str(worktree_path) if worktree_path else workspace.repo_path,
|
|
2136
|
+
stdout=None, # Let output flow to terminal
|
|
2137
|
+
stderr=None,
|
|
2138
|
+
text=True,
|
|
2139
|
+
)
|
|
2140
|
+
|
|
2141
|
+
# Track process if batch_id provided (thread-safe)
|
|
2142
|
+
if batch_id:
|
|
2143
|
+
with _active_processes_lock:
|
|
2144
|
+
if batch_id not in _active_processes:
|
|
2145
|
+
_active_processes[batch_id] = {}
|
|
2146
|
+
_active_processes[batch_id][task_id] = process
|
|
2147
|
+
|
|
2148
|
+
# Wait for completion (outside lock to avoid blocking)
|
|
2149
|
+
returncode = process.wait()
|
|
2150
|
+
|
|
2151
|
+
# Untrack process (thread-safe)
|
|
2152
|
+
if batch_id:
|
|
2153
|
+
with _active_processes_lock:
|
|
2154
|
+
if batch_id in _active_processes:
|
|
2155
|
+
_active_processes[batch_id].pop(task_id, None)
|
|
2156
|
+
if not _active_processes[batch_id]:
|
|
2157
|
+
_active_processes.pop(batch_id, None)
|
|
2158
|
+
|
|
2159
|
+
# Check the run status from database
|
|
2160
|
+
# The subprocess should have updated the run record
|
|
2161
|
+
run = get_active_run(workspace, task_id)
|
|
2162
|
+
if run:
|
|
2163
|
+
return run.status.value
|
|
2164
|
+
|
|
2165
|
+
# If no active run, check if there's a recent completed/failed run
|
|
2166
|
+
# by listing runs for this task
|
|
2167
|
+
from codeframe.core.runtime import list_runs
|
|
2168
|
+
runs = list_runs(workspace, task_id=task_id, limit=1)
|
|
2169
|
+
if runs:
|
|
2170
|
+
return runs[0].status.value
|
|
2171
|
+
|
|
2172
|
+
# Fallback based on subprocess exit code
|
|
2173
|
+
if returncode == 0:
|
|
2174
|
+
return RunStatus.COMPLETED.value
|
|
2175
|
+
else:
|
|
2176
|
+
return RunStatus.FAILED.value
|
|
2177
|
+
|
|
2178
|
+
except Exception as e:
|
|
2179
|
+
print(f" Error executing task: {e}")
|
|
2180
|
+
# Cleanup on exception (thread-safe)
|
|
2181
|
+
if batch_id:
|
|
2182
|
+
with _active_processes_lock:
|
|
2183
|
+
if batch_id in _active_processes:
|
|
2184
|
+
_active_processes[batch_id].pop(task_id, None)
|
|
2185
|
+
if not _active_processes[batch_id]:
|
|
2186
|
+
_active_processes.pop(batch_id, None)
|
|
2187
|
+
return RunStatus.FAILED.value
|
|
2188
|
+
|
|
2189
|
+
|
|
2190
|
+
def _save_batch(workspace: Workspace, batch: BatchRun) -> None:
|
|
2191
|
+
"""Save or update a batch record in the database.
|
|
2192
|
+
|
|
2193
|
+
Uses a thread lock to prevent SQLite "database is locked" errors
|
|
2194
|
+
when multiple workers try to update batch status concurrently.
|
|
2195
|
+
"""
|
|
2196
|
+
task_ids_json = json.dumps(batch.task_ids)
|
|
2197
|
+
results_json = json.dumps(batch.results) if batch.results else None
|
|
2198
|
+
completed_at = batch.completed_at.isoformat() if batch.completed_at else None
|
|
2199
|
+
|
|
2200
|
+
with _batch_db_lock:
|
|
2201
|
+
conn = get_db_connection(workspace)
|
|
2202
|
+
try:
|
|
2203
|
+
cursor = conn.cursor()
|
|
2204
|
+
# Ensure isolation column exists (migration for existing databases)
|
|
2205
|
+
try:
|
|
2206
|
+
cursor.execute(
|
|
2207
|
+
"ALTER TABLE batch_runs ADD COLUMN isolation TEXT DEFAULT 'none'"
|
|
2208
|
+
)
|
|
2209
|
+
conn.commit()
|
|
2210
|
+
except Exception:
|
|
2211
|
+
pass # Column already exists
|
|
2212
|
+
|
|
2213
|
+
cursor.execute(
|
|
2214
|
+
"""
|
|
2215
|
+
INSERT OR REPLACE INTO batch_runs
|
|
2216
|
+
(id, workspace_id, task_ids, status, strategy, max_parallel, on_failure,
|
|
2217
|
+
started_at, completed_at, results, engine, isolation)
|
|
2218
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2219
|
+
""",
|
|
2220
|
+
(
|
|
2221
|
+
batch.id,
|
|
2222
|
+
batch.workspace_id,
|
|
2223
|
+
task_ids_json,
|
|
2224
|
+
batch.status.value,
|
|
2225
|
+
batch.strategy,
|
|
2226
|
+
batch.max_parallel,
|
|
2227
|
+
batch.on_failure.value,
|
|
2228
|
+
batch.started_at.isoformat(),
|
|
2229
|
+
completed_at,
|
|
2230
|
+
results_json,
|
|
2231
|
+
batch.engine,
|
|
2232
|
+
batch.isolation,
|
|
2233
|
+
),
|
|
2234
|
+
)
|
|
2235
|
+
conn.commit()
|
|
2236
|
+
finally:
|
|
2237
|
+
conn.close()
|
|
2238
|
+
|
|
2239
|
+
|
|
2240
|
+
def _row_to_batch(row: tuple) -> BatchRun:
|
|
2241
|
+
"""Convert a database row to a BatchRun object."""
|
|
2242
|
+
return BatchRun(
|
|
2243
|
+
id=row[0],
|
|
2244
|
+
workspace_id=row[1],
|
|
2245
|
+
task_ids=json.loads(row[2]),
|
|
2246
|
+
status=BatchStatus(row[3]),
|
|
2247
|
+
strategy=row[4],
|
|
2248
|
+
max_parallel=row[5],
|
|
2249
|
+
on_failure=OnFailure(row[6]),
|
|
2250
|
+
started_at=datetime.fromisoformat(row[7]),
|
|
2251
|
+
completed_at=datetime.fromisoformat(row[8]) if row[8] else None,
|
|
2252
|
+
results=json.loads(row[9]) if row[9] else {},
|
|
2253
|
+
engine=row[10] if len(row) > 10 and row[10] else "plan",
|
|
2254
|
+
isolation=row[11] if len(row) > 11 and row[11] else "none",
|
|
2255
|
+
)
|