engineering-platform 2.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- engineering_platform/ENGINEERING_PLATFORM_CONFIG.json +32 -0
- engineering_platform/ENGINEERING_PLATFORM_VERSION.json +15 -0
- engineering_platform/__init__.py +1 -0
- engineering_platform/__main__.py +7 -0
- engineering_platform/agent_state.py +530 -0
- engineering_platform/agent_trust.py +174 -0
- engineering_platform/assets/dashboard.css +1317 -0
- engineering_platform/assets/dashboard.js +8534 -0
- engineering_platform/assets/dashboard_locales.mjs +4049 -0
- engineering_platform/assets/dashboard_status_store.mjs +41 -0
- engineering_platform/assets/operations-console/apple-touch-icon-dark.png +0 -0
- engineering_platform/assets/operations-console/apple-touch-icon-light.png +0 -0
- engineering_platform/assets/operations-console/icon-dark.png +0 -0
- engineering_platform/assets/operations-console/icon-light.png +0 -0
- engineering_platform/assets/operations-console/icon-transparent.png +0 -0
- engineering_platform/assets/operations-console/manifest.webmanifest +11 -0
- engineering_platform/capability_preflight.py +285 -0
- engineering_platform/capability_review.py +261 -0
- engineering_platform/central_data_transfer.py +195 -0
- engineering_platform/central_database.py +245 -0
- engineering_platform/central_store_migration.py +1672 -0
- engineering_platform/codex_capacity.py +81 -0
- engineering_platform/codex_chat.py +226 -0
- engineering_platform/codex_observability.py +153 -0
- engineering_platform/component_lock.py +40 -0
- engineering_platform/component_logging.py +420 -0
- engineering_platform/console_presentation.py +14 -0
- engineering_platform/console_route_ownership.py +83 -0
- engineering_platform/contracts/__init__.py +38 -0
- engineering_platform/contracts/ep_consumer.py +391 -0
- engineering_platform/contracts/models.py +105 -0
- engineering_platform/contracts/projection.py +401 -0
- engineering_platform/dashboard_browser_validation.py +206 -0
- engineering_platform/dashboard_state.py +630 -0
- engineering_platform/dashboard_supervisor.swift +105 -0
- engineering_platform/dashboard_translation.py +129 -0
- engineering_platform/dependabot_producer.py +349 -0
- engineering_platform/drift_diagnostics.py +144 -0
- engineering_platform/emergency_recovery.py +268 -0
- engineering_platform/engineering_memory.py +139 -0
- engineering_platform/ep_consumer_credentials.py +473 -0
- engineering_platform/evidence_projection.py +213 -0
- engineering_platform/execution_activity.py +218 -0
- engineering_platform/execution_context.py +132 -0
- engineering_platform/execution_errors.py +42 -0
- engineering_platform/execution_evidence.py +24 -0
- engineering_platform/execution_executor.py +730 -0
- engineering_platform/execution_finalization.py +44 -0
- engineering_platform/execution_host.py +3306 -0
- engineering_platform/execution_lease.py +365 -0
- engineering_platform/execution_lifecycle.py +447 -0
- engineering_platform/execution_models.py +43 -0
- engineering_platform/execution_readiness.py +166 -0
- engineering_platform/execution_reporting.py +1607 -0
- engineering_platform/execution_repository.py +253 -0
- engineering_platform/execution_timeout_policy.py +56 -0
- engineering_platform/execution_timing.py +440 -0
- engineering_platform/execution_transaction.py +28 -0
- engineering_platform/external_producer_binding.py +235 -0
- engineering_platform/file_inbox.py +249 -0
- engineering_platform/forensic_attribution.py +338 -0
- engineering_platform/forensic_attribution_v2.py +134 -0
- engineering_platform/forensic_delta.py +299 -0
- engineering_platform/golden_scenario.py +63 -0
- engineering_platform/historical_dashboard_configuration.py +171 -0
- engineering_platform/host_admin.py +199 -0
- engineering_platform/host_preflight.py +231 -0
- engineering_platform/installation_relocation.py +122 -0
- engineering_platform/investigation_ledger.py +89 -0
- engineering_platform/legacy_inbox_migration.py +79 -0
- engineering_platform/lifecycle_worker.py +223 -0
- engineering_platform/live_status.py +267 -0
- engineering_platform/local_api.py +209 -0
- engineering_platform/local_api_keychain.py +51 -0
- engineering_platform/local_repository_binding.py +138 -0
- engineering_platform/managed_autonomy.py +509 -0
- engineering_platform/managed_codex_runtime.py +105 -0
- engineering_platform/parity_context.py +203 -0
- engineering_platform/parity_lifecycle_dispatcher.py +488 -0
- engineering_platform/platform_admin.py +13 -0
- engineering_platform/platform_api.py +428 -0
- engineering_platform/platform_bootstrap.py +385 -0
- engineering_platform/platform_components.py +65 -0
- engineering_platform/platform_version.py +171 -0
- engineering_platform/pr_check_repair.py +276 -0
- engineering_platform/pr_evidence_backfill.py +278 -0
- engineering_platform/producer.py +209 -0
- engineering_platform/project_agent.py +366 -0
- engineering_platform/project_agent_service.py +244 -0
- engineering_platform/project_topology.py +126 -0
- engineering_platform/prompt_history.py +591 -0
- engineering_platform/provider_context.py +136 -0
- engineering_platform/provider_context_benchmark.py +41 -0
- engineering_platform/provider_context_scope.py +90 -0
- engineering_platform/provider_interruption.py +168 -0
- engineering_platform/provider_process_identity.py +80 -0
- engineering_platform/provider_readiness.py +138 -0
- engineering_platform/provider_recovery.py +647 -0
- engineering_platform/provider_usage.py +497 -0
- engineering_platform/providers.py +471 -0
- engineering_platform/qualification.py +220 -0
- engineering_platform/recommendation_handoff.py +238 -0
- engineering_platform/report_analysis.py +193 -0
- engineering_platform/repository_attachment.py +171 -0
- engineering_platform/repository_handoff.py +95 -0
- engineering_platform/resources.py +38 -0
- engineering_platform/reviewer_evidence.py +70 -0
- engineering_platform/schemas/repository-attachment.schema.json +61 -0
- engineering_platform/server.py +3679 -0
- engineering_platform/server_console_services.py +2024 -0
- engineering_platform/server_relay.py +172 -0
- engineering_platform/server_service.py +122 -0
- engineering_platform/status_model.py +135 -0
- engineering_platform/status_reconciliation.py +34 -0
- engineering_platform/storage.py +2440 -0
- engineering_platform/submission_cli.py +77 -0
- engineering_platform/submission_intake.py +45 -0
- engineering_platform/submission_service.py +317 -0
- engineering_platform/telemetry.py +951 -0
- engineering_platform/templates/workspace-config.json +25 -0
- engineering_platform/validation_identity.py +50 -0
- engineering_platform/validation_profile.py +211 -0
- engineering_platform/workspace_preflight.py +263 -0
- engineering_platform/worktree_provenance.py +147 -0
- engineering_platform/worktree_tooling.py +18 -0
- engineering_platform-2.2.0.dist-info/METADATA +18 -0
- engineering_platform-2.2.0.dist-info/RECORD +130 -0
- engineering_platform-2.2.0.dist-info/WHEEL +5 -0
- engineering_platform-2.2.0.dist-info/entry_points.txt +6 -0
- engineering_platform-2.2.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,3306 @@
|
|
|
1
|
+
"""Thin foreground orchestrator for one bounded Engineering Platform prompt."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
from datetime import datetime, timezone
|
|
7
|
+
from dataclasses import replace
|
|
8
|
+
import json
|
|
9
|
+
import logging
|
|
10
|
+
import os
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
import subprocess # noqa: F401 - compatibility export for host tests
|
|
13
|
+
import time
|
|
14
|
+
from threading import Lock
|
|
15
|
+
from typing import Protocol
|
|
16
|
+
import uuid
|
|
17
|
+
import re
|
|
18
|
+
import sqlite3
|
|
19
|
+
|
|
20
|
+
from .validation_identity import is_canonical_dashboard_command
|
|
21
|
+
|
|
22
|
+
from .agent_state import MAX_COMMIT_EVIDENCE_RECORDS, StateError, StateStore, TransactionState, redact_diagnostic, verified_commit_evidence_record
|
|
23
|
+
from .capability_review import (
|
|
24
|
+
ReviewerResult,
|
|
25
|
+
ReviewerSelection,
|
|
26
|
+
records_for_storage,
|
|
27
|
+
run_reviews,
|
|
28
|
+
select_reviewers,
|
|
29
|
+
)
|
|
30
|
+
from .codex_observability import codex_final_message as _codex_final_message # noqa: F401
|
|
31
|
+
from .codex_observability import extract_codex_runtime_metadata # noqa: F401
|
|
32
|
+
from .codex_observability import extract_codex_usage # noqa: F401
|
|
33
|
+
from .codex_observability import write_codex_usage
|
|
34
|
+
from .engineering_memory import (
|
|
35
|
+
capture_engineering_memory,
|
|
36
|
+
load_engineering_memory,
|
|
37
|
+
retrieve_engineering_memory,
|
|
38
|
+
)
|
|
39
|
+
from .live_status import print_live_status, write_live_status, write_runner_process
|
|
40
|
+
from .platform_version import (
|
|
41
|
+
CURRENT_PLATFORM_VERSION,
|
|
42
|
+
EngineeringPlatformCompatibilityError,
|
|
43
|
+
EngineeringPlatformManifest,
|
|
44
|
+
RunnerCompatibility,
|
|
45
|
+
validate_compatibility,
|
|
46
|
+
)
|
|
47
|
+
from .qualification import dashboard, execute_qualification
|
|
48
|
+
from .report_analysis import analyze as analyze_terminal_report
|
|
49
|
+
from .prompt_history import record_terminal_report
|
|
50
|
+
from .status_model import build as build_canonical_status, publish as publish_canonical_status
|
|
51
|
+
from .platform_api import PlatformConfiguration, PlatformConfigurationError
|
|
52
|
+
from .resources import package_path
|
|
53
|
+
from .platform_bootstrap import runtime_workspace
|
|
54
|
+
from .providers import DeterministicValidationExecutor, GitProvider, CodexCliProvider
|
|
55
|
+
from .host_preflight import latest as latest_host_preflight
|
|
56
|
+
from .workspace_preflight import latest as latest_workspace_preflight
|
|
57
|
+
from .capability_preflight import latest as latest_capability_preflight
|
|
58
|
+
from .execution_lease import Lease, LeaseConflictError, LeaseHeartbeat, acquire as acquire_lease, heartbeat as heartbeat_lease, host_identity, host_instance_id, reconcile_stale, release as release_lease
|
|
59
|
+
from .execution_readiness import ReadinessFacts, decide as decide_readiness, evaluate as evaluate_readiness, selected_profile
|
|
60
|
+
from .execution_transaction import ExecutionTransaction
|
|
61
|
+
from .execution_context import (
|
|
62
|
+
additional_workspace_write_roots as context_workspace_write_roots,
|
|
63
|
+
execution_mode_for as context_execution_mode_for,
|
|
64
|
+
genesis_target_for as context_genesis_target_for,
|
|
65
|
+
genesis_workspace_preflight as context_genesis_workspace_preflight,
|
|
66
|
+
resolve_execution_context as context_resolve_execution_context,
|
|
67
|
+
target_repository_authorization as context_target_repository_authorization,
|
|
68
|
+
)
|
|
69
|
+
from .execution_models import AgentResult, PullRequestEvidence, RepositoryEvidence
|
|
70
|
+
from .validation_profile import (
|
|
71
|
+
VALIDATION_PROFILE_VERSION, ValidationControlLauncher, ValidationProfile,
|
|
72
|
+
ValidationProfileResolutionError, changed_paths, classify,
|
|
73
|
+
profile_control_bindings, resolve_producer_profile,
|
|
74
|
+
)
|
|
75
|
+
from .reviewer_evidence import ReviewerEvidence
|
|
76
|
+
from .investigation_ledger import InvocationInvestigationLedger
|
|
77
|
+
from .execution_errors import CodexHandoffTimeout, CodexInvocationError, RunnerError
|
|
78
|
+
from .execution_errors import ProviderReadinessBlocked
|
|
79
|
+
from .execution_timeout_policy import FINALIZATION, REPAIR, agent_timeout
|
|
80
|
+
from .provider_readiness import failures as provider_readiness_failures
|
|
81
|
+
from .execution_repository import GitHubClient as ProviderGitHubClient, RepositoryClient as ProviderRepositoryClient
|
|
82
|
+
from .execution_repository import GhCliClient as ProviderGhCliClient, SubprocessRepositoryClient as ProviderRepositoryClientImpl
|
|
83
|
+
from .execution_executor import format_cli_failure as executor_format_cli_failure
|
|
84
|
+
from .execution_executor import (
|
|
85
|
+
project_codex_activity as executor_project_codex_activity,
|
|
86
|
+
project_codex_live_action_name as executor_project_codex_live_action_name,
|
|
87
|
+
)
|
|
88
|
+
from .execution_executor import redacted_cli_tail as executor_redacted_cli_tail
|
|
89
|
+
from .execution_executor import record_redacted_codex_cli_diagnostic
|
|
90
|
+
from .execution_executor import persist_validation_failure_diagnostic
|
|
91
|
+
from .execution_executor import CodexCliClient
|
|
92
|
+
from .execution_finalization import FinalizationCoordinator
|
|
93
|
+
from .storage import EngineeringStorageError, load_admission_decision, load_submission_for_run, load_validation_context, open_storage, record_artifact, record_readiness_evaluation, record_validation_command_invocation, record_validation_command_terminal, record_validation_control_result, record_validation_profile
|
|
94
|
+
from .dashboard_browser_validation import dashboard_evidence_path, load_dashboard_evidence
|
|
95
|
+
from .storage import dismissal_for_run
|
|
96
|
+
from .provider_usage import AUTHORITATIVE, ProviderInvocation, normalize_codex_model, persist_provider_invocation
|
|
97
|
+
from .provider_recovery import (
|
|
98
|
+
claim_replacement_launch, create_recovery_available, load_recovery_state, record_provider_started,
|
|
99
|
+
load_recovery_agent_result, mark_precheck_failed, persist_recovery_agent_result,
|
|
100
|
+
record_pre_execution_launch_failure, record_replacement_terminal, reconcile_recovery,
|
|
101
|
+
transition_recovery_state, consume_controlled_interruption_hook,
|
|
102
|
+
)
|
|
103
|
+
from .worktree_provenance import capture as capture_worktree_provenance, verify_recovery as verify_worktree_recovery
|
|
104
|
+
from .provider_context import ProviderRole, project_context, provider_need_for_phase, role_for_phase
|
|
105
|
+
from .provider_context_scope import ContextScope, POLICY_ID, initial_context_scope, provider_instruction
|
|
106
|
+
from .execution_timing import ActivePhase
|
|
107
|
+
from .execution_timing import complete_active_phase as _complete_active_phase
|
|
108
|
+
from .execution_timing import complete_phase as _complete_phase
|
|
109
|
+
from .execution_timing import start_or_resume_phase as _start_or_resume_phase
|
|
110
|
+
from .execution_timing import start_phase as _start_phase
|
|
111
|
+
from .managed_autonomy import (
|
|
112
|
+
append_action as record_managed_action,
|
|
113
|
+
append_pr_check_observation as record_managed_pr_check,
|
|
114
|
+
append_validation_observation as record_managed_validation,
|
|
115
|
+
record_gate as record_managed_gate,
|
|
116
|
+
)
|
|
117
|
+
from .component_logging import component_logger, shutdown_signal_logging
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
LOGGER = logging.getLogger(__name__)
|
|
121
|
+
# Compatibility exports for integrations that already import these names.
|
|
122
|
+
FINALIZATION_PR_HANDOFF_MAX_SECONDS = FINALIZATION.seconds
|
|
123
|
+
REPAIR_AGENT_MAX_SECONDS = REPAIR.seconds
|
|
124
|
+
|
|
125
|
+
# A repair remains scoped to its original PR, but it must also have a finite
|
|
126
|
+
# attempt budget. This prevents a persistently failing required check from
|
|
127
|
+
# repeatedly invoking the provider without an operator decision.
|
|
128
|
+
MAX_PR_CHECK_REPAIR_ATTEMPTS = 3
|
|
129
|
+
MAX_LOCAL_REPOSITORY_VALIDATION_ATTEMPTS = 3
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _timing_unavailable(error: EngineeringStorageError) -> None:
|
|
133
|
+
"""Keep optional phase telemetry from changing the run outcome."""
|
|
134
|
+
LOGGER.warning("Execution phase telemetry is unavailable: %s", error)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def start_phase(root: Path, run_id: str, phase_name: str, **kwargs: object) -> ActivePhase | None:
|
|
138
|
+
try:
|
|
139
|
+
return _start_phase(root, run_id, phase_name, **kwargs)
|
|
140
|
+
except EngineeringStorageError as error:
|
|
141
|
+
_timing_unavailable(error)
|
|
142
|
+
return None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def start_or_resume_phase(root: Path, run_id: str, phase_name: str, **kwargs: object) -> ActivePhase | None:
|
|
146
|
+
try:
|
|
147
|
+
return _start_or_resume_phase(root, run_id, phase_name, **kwargs)
|
|
148
|
+
except EngineeringStorageError as error:
|
|
149
|
+
_timing_unavailable(error)
|
|
150
|
+
return None
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def complete_phase(root: Path, active: ActivePhase | None, **kwargs: object) -> None:
|
|
154
|
+
if active is None:
|
|
155
|
+
return
|
|
156
|
+
try:
|
|
157
|
+
_complete_phase(root, active, **kwargs)
|
|
158
|
+
except EngineeringStorageError as error:
|
|
159
|
+
_timing_unavailable(error)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def complete_active_phase(root: Path, run_id: str, phase_name: str, **kwargs: object) -> bool:
|
|
163
|
+
try:
|
|
164
|
+
return _complete_active_phase(root, run_id, phase_name, **kwargs)
|
|
165
|
+
except EngineeringStorageError as error:
|
|
166
|
+
_timing_unavailable(error)
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
# Compatibility exports remain at this façade while implementation resides in
|
|
170
|
+
# the dedicated context, repository and executor modules.
|
|
171
|
+
RepositoryClient = ProviderRepositoryClient
|
|
172
|
+
GitHubClient = ProviderGitHubClient
|
|
173
|
+
SubprocessRepositoryClient = ProviderRepositoryClientImpl
|
|
174
|
+
GhCliClient = ProviderGhCliClient
|
|
175
|
+
additional_workspace_write_roots = context_workspace_write_roots
|
|
176
|
+
target_repository_authorization = context_target_repository_authorization
|
|
177
|
+
resolve_execution_context = context_resolve_execution_context
|
|
178
|
+
execution_mode_for = context_execution_mode_for
|
|
179
|
+
genesis_target_for = context_genesis_target_for
|
|
180
|
+
genesis_workspace_preflight = context_genesis_workspace_preflight
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
class AgentClient(Protocol):
|
|
185
|
+
def available(self) -> bool: ...
|
|
186
|
+
|
|
187
|
+
def version(self) -> str: ...
|
|
188
|
+
|
|
189
|
+
def invoke(self, root: Path, prompt: str) -> AgentResult: ...
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
project_codex_activity = executor_project_codex_activity
|
|
193
|
+
project_codex_live_action_name = executor_project_codex_live_action_name
|
|
194
|
+
_redacted_cli_tail = executor_redacted_cli_tail
|
|
195
|
+
_format_cli_failure = executor_format_cli_failure
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def assemble_prompt(
|
|
199
|
+
prompt_path: Path,
|
|
200
|
+
state: TransactionState | None,
|
|
201
|
+
*,
|
|
202
|
+
managed_target: Path | None = None,
|
|
203
|
+
reviewer_evidence: ReviewerEvidence | None = None,
|
|
204
|
+
role: ProviderRole | None = None,
|
|
205
|
+
) -> str:
|
|
206
|
+
objective = prompt_path.read_text(encoding="utf-8")
|
|
207
|
+
provider_role = role or role_for_phase(state.phase if state else "EXECUTE_AGENT")
|
|
208
|
+
projection = project_context(provider_role, objective)
|
|
209
|
+
scope = initial_context_scope(
|
|
210
|
+
phase=state.phase if state else "EXECUTE_AGENT",
|
|
211
|
+
repair_iterations=state.repair_iterations if state else 0,
|
|
212
|
+
objective=objective,
|
|
213
|
+
)
|
|
214
|
+
resume = (
|
|
215
|
+
"No prior transaction checkpoint exists."
|
|
216
|
+
if state is None
|
|
217
|
+
else json.dumps(state.to_dict(), sort_keys=True)
|
|
218
|
+
)
|
|
219
|
+
authority = (
|
|
220
|
+
"""This is the sole automatic post-Finalization reconciliation. You may only update
|
|
221
|
+
the four canonical rolling records, commit them directly to the already synchronized `main`,
|
|
222
|
+
and push that one commit. Do not create a branch or pull request. Do not change runtime code,
|
|
223
|
+
authority, lifecycle, retry, validation, provider, Forge, queue, or delivery semantics. Return
|
|
224
|
+
`COMPLETE`, `repository_reconciled`, and the pushed main commit SHA only after verifying a clean
|
|
225
|
+
workspace and that `main` contains that exact commit."""
|
|
226
|
+
if state and state.transaction_kind == "RECONCILIATION"
|
|
227
|
+
else
|
|
228
|
+
"""The runner holds explicit owner authorization for this exact bounded transaction. You may create, commit and push one bounded branch and draft pull request, or repair that same pull request. The runner may mark that pull request ready for review, but only the human operator may merge it. Do not merge, release, deploy, tag, publish, upload, change repository settings, bypass protection, or expand the objective."""
|
|
229
|
+
if state and state.owner_authorized
|
|
230
|
+
else "Do not create a merge, release, deployment, daemon, remote-control, or architecture authority beyond the supplied objective."
|
|
231
|
+
)
|
|
232
|
+
genesis = "" if not state or state.execution_mode != "GENESIS" else """
|
|
233
|
+
This is an explicit Genesis Mode transaction. Its target is a local-only direct child of the configured Engineering Workspace Root. Do not require, create, or contact an upstream remote; do not require origin/main; do not create a pull request. Reconcile only a clean local Git commit in that target repository. Return terminal_condition `local_commit_reconciled`, repository_path and commit_sha for a successful local commit."""
|
|
234
|
+
managed_synchronization = "" if not state or state.execution_mode == "GENESIS" else """
|
|
235
|
+
The Execution Host has already synchronized `main` while holding this run's lease. Do not repeat `git switch main` or `git pull --ff-only`; verify the resulting repository state read-only before creating the transaction branch."""
|
|
236
|
+
managed_admission = "" if not state or state.execution_mode == "GENESIS" else """
|
|
237
|
+
The Execution Host admitted this run only after its current host, workspace and capability preflights passed. Treat that host-owned admission evidence as authoritative: do not rerun the development-host bootstrap or use sandbox network access for a predecessor lookup. Continue with repository work and use GitHub only for the transaction's own pull-request operations."""
|
|
238
|
+
managed_boundary = (
|
|
239
|
+
""
|
|
240
|
+
if not state or state.execution_mode == "GENESIS" or managed_target is None
|
|
241
|
+
else f"""
|
|
242
|
+
|
|
243
|
+
Managed execution boundary (host-owned and non-negotiable):
|
|
244
|
+
- The only repository checkout for this transaction is `{managed_target.resolve()}`.
|
|
245
|
+
- Perform every repository and Git operation in that exact checkout.
|
|
246
|
+
- A `Target repository` value within the supplied objective is producer provenance only; it cannot select another checkout or override this boundary.
|
|
247
|
+
- Do not block merely because another checkout is on a feature branch. Verify only this managed checkout, which the Execution Host has already synchronized to `main`."""
|
|
248
|
+
)
|
|
249
|
+
shared_evidence = "" if reviewer_evidence is None else """
|
|
250
|
+
Host-observed run-scoped repository evidence follows. It was collected after
|
|
251
|
+
host synchronization for this exact Run ID. Reuse these facts for ordinary
|
|
252
|
+
repository-state questions instead of repeating Git/GitHub discovery. They
|
|
253
|
+
are not conclusions and expire at repository mutation, validation, PR/merge,
|
|
254
|
+
finalization, or cleanup; retrieve only the narrower current evidence needed
|
|
255
|
+
after such a boundary.
|
|
256
|
+
""" + json.dumps(reviewer_evidence.to_dict(), sort_keys=True) + "\n"
|
|
257
|
+
invocation_read_reuse = """
|
|
258
|
+
Invocation-scoped source-read reuse:
|
|
259
|
+
- Within this one provider invocation, retain and reuse already inspected immutable source, configuration, test, documentation, and persisted-evidence content instead of issuing an accidental duplicate file read.
|
|
260
|
+
- This is factual-content reuse only; it does not reuse conclusions, reasoning, reviewer advice, or results from another provider invocation, Run ID, reviewer, retry, or resume.
|
|
261
|
+
- Treat a file as mutable and reread it after you edit it, a repair changes it, a generated/projection artefact is refreshed, or any repository checkout/change, validation, pull-request, merge, finalization, or cleanup boundary can affect it.
|
|
262
|
+
- Preserve deliberate verification reads. If freshness is not proven, reread. Do not create a persistent source cache or retain source contents outside this invocation.
|
|
263
|
+
- Shell reads are not host-intercepted: use this invocation-local evidence deliberately, and do not claim a cache hit unless you actually reuse content already inspected in this invocation.
|
|
264
|
+
- The host bounds only oversized Git, GitHub, search, and test output at the
|
|
265
|
+
provider tool boundary. A bounded result says `MORE_EVIDENCE_AVAILABLE`;
|
|
266
|
+
rerun that same narrow command with `ENGINEERING_PLATFORM_EVIDENCE_EXPAND=1` only when
|
|
267
|
+
its exact raw output is required. Source reads remain exact by default.
|
|
268
|
+
- A successful test result may be compact, but a failed test keeps its failing
|
|
269
|
+
identity, assertion and diagnostic context. Never treat a bounded result as
|
|
270
|
+
proof when it is ambiguous: expand it or fail closed.
|
|
271
|
+
"""
|
|
272
|
+
context_scope_instruction = "\n" + provider_instruction(scope) + "\n"
|
|
273
|
+
investigation_ledger = InvocationInvestigationLedger().record(
|
|
274
|
+
"repository_identity", "repository_status", "git_ancestry"
|
|
275
|
+
) if reviewer_evidence is not None else InvocationInvestigationLedger()
|
|
276
|
+
primary_tool_loop = """
|
|
277
|
+
Primary Invocation Investigation Ledger (ephemeral and primary-only):
|
|
278
|
+
Use this identifier-only ledger to avoid rediscovering a fact already established
|
|
279
|
+
in this invocation. Record a fact only after its narrow real check; never record
|
|
280
|
+
source text, paths, commands, tool output, prompts, conclusions, or reviewer
|
|
281
|
+
reasoning. Before a tool call, it must establish one missing fact, refresh an
|
|
282
|
+
invalidated fact, perform a mutation, or execute required validation.
|
|
283
|
+
|
|
284
|
+
For unchanged state, reuse an established source inspection, test surface,
|
|
285
|
+
repository status, or ancestry fact. Prefer exact branch/HEAD/status, named
|
|
286
|
+
diff/stat, and targeted ancestry queries over broad logs or full diffs. Do not
|
|
287
|
+
rerun a passing validation unless relevant code/test inputs changed or a
|
|
288
|
+
canonical boundary requires it. At every listed boundary, invalidate all
|
|
289
|
+
non-RUN-STABLE facts and obtain narrow fresh evidence; uncertainty is itself a
|
|
290
|
+
freshness boundary. Reviewer advice and primary conclusions are never ledger
|
|
291
|
+
facts and must never cross the primary/reviewer boundary.
|
|
292
|
+
|
|
293
|
+
Ledger bootstrap:
|
|
294
|
+
""" + json.dumps(investigation_ledger.to_prompt_dict(), sort_keys=True) + "\n"
|
|
295
|
+
pr_handoff = "" if not state or state.execution_mode == "GENESIS" else """
|
|
296
|
+
PR hand-off boundary (host-owned and non-negotiable):
|
|
297
|
+
- Your work ends when the bounded branch and its pull request have been
|
|
298
|
+
created or repaired, pushed, and locally validated.
|
|
299
|
+
- If this is a repair, preserve the exact checkpointed pull-request number
|
|
300
|
+
and branch. Never create a replacement pull request.
|
|
301
|
+
- Return the required JSON object immediately after that hand-off. Do not
|
|
302
|
+
poll or wait for GitHub checks, review, merge, Finalization, reconciliation,
|
|
303
|
+
or any other external terminal evidence. The Execution Host alone records
|
|
304
|
+
the pull request, polls checks, and schedules at most three bounded repairs.
|
|
305
|
+
- Write pull-request Markdown with real line breaks. Never serialize a line
|
|
306
|
+
break as the literal characters `\\n`.
|
|
307
|
+
"""
|
|
308
|
+
local_gate = "" if not state or not (
|
|
309
|
+
state.execution_mode == "MANAGED" and state.transaction_kind == "IMPLEMENTATION" and state.phase == "EXECUTE_AGENT"
|
|
310
|
+
) else """
|
|
311
|
+
Local validation hand-off boundary:
|
|
312
|
+
- Create, commit and push the bounded implementation branch, but do not create a pull request yet.
|
|
313
|
+
- Return that branch with `pull_request: null` after relevant focused validation. The host owns the next local repository validation gate and only that gate may create the implementation PR after the canonical suite passes.
|
|
314
|
+
"""
|
|
315
|
+
return f"""You are executing one bounded Engineering Platform transaction.
|
|
316
|
+
Provider role: {provider_role.value}. Context projection: {projection.budget_version}; source items: {projection.source_item_count}; omitted lower-priority items: {projection.omitted_low_priority_count}.{context_scope_instruction}
|
|
317
|
+
Read BOOTSTRAP.md, ENGINEERING_METHOD.md, PROMPT_INITIALIZATION.md and AGENTS.md from the actual repository before acting. Repository and GitHub evidence override this checkpoint: {resume}
|
|
318
|
+
{authority}{genesis}{managed_synchronization}{managed_admission}{shared_evidence}{invocation_read_reuse}{primary_tool_loop}{local_gate}{pr_handoff}
|
|
319
|
+
Supplied bounded objective follows:\n\n{projection.text}\n{managed_boundary}\n\nReturn only one JSON object with terminal_state (COMPLETE, WAITING, BLOCKED, or FAILED), branch, pull_request, terminal_condition (repository_reconciled, open_pr_checks_terminal, external_blocked, or local_commit_reconciled), diagnostic, repository_path, commit_sha, validation_evidence, quality_evidence and validation_disposition. validation_evidence is a bounded list of executed validation {{command, result}} summaries; use [] when none ran. quality_evidence is [] except for the autonomous quality-control stage, where it contains only bounded, executed {{activity, result}} records. validation_disposition is product_failure unless the required suite failed for an environmental instability that you demonstrated with bounded evidence, such as an isolated rerun of the same failing check passing without a code change. It never makes a failed suite pass. Never include secrets, tokens, headers, environment values, prompts, repository file contents, stack traces, or raw command output. Use null for other fields that do not apply. The diagnostic must be a short human-readable reason without secrets, tokens, headers, environment values, prompt content, repository file content, stack traces, or raw command output."""
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
class EngineeringRunner:
|
|
323
|
+
def __init__(
|
|
324
|
+
self,
|
|
325
|
+
root: Path,
|
|
326
|
+
store: StateStore,
|
|
327
|
+
repository: RepositoryClient,
|
|
328
|
+
github: GitHubClient,
|
|
329
|
+
agent: AgentClient,
|
|
330
|
+
sleep=time.sleep,
|
|
331
|
+
compatibility: RunnerCompatibility = RunnerCompatibility(),
|
|
332
|
+
) -> None:
|
|
333
|
+
self.root, self.store, self.repository, self.github, self.agent, self.sleep = (
|
|
334
|
+
root,
|
|
335
|
+
store,
|
|
336
|
+
repository,
|
|
337
|
+
github,
|
|
338
|
+
agent,
|
|
339
|
+
sleep,
|
|
340
|
+
)
|
|
341
|
+
self.compatibility = compatibility
|
|
342
|
+
self.platform_manifest: EngineeringPlatformManifest | None = None
|
|
343
|
+
self.detected_codex_cli: str | None = None
|
|
344
|
+
self.reviewer_records: tuple[dict[str, object], ...] = ()
|
|
345
|
+
self.reviewer_runtime: list[dict[str, object]] = []
|
|
346
|
+
self._reviewer_runtime_lock = Lock()
|
|
347
|
+
self.console_detail: str | None = None
|
|
348
|
+
self.host_identity = host_identity()
|
|
349
|
+
self.host_instance_id = host_instance_id()
|
|
350
|
+
self.active_lease: Lease | None = None
|
|
351
|
+
self.transaction: ExecutionTransaction | None = None
|
|
352
|
+
self.lease_heartbeat: LeaseHeartbeat | None = None
|
|
353
|
+
self.finalization = FinalizationCoordinator()
|
|
354
|
+
self.validation_executor = DeterministicValidationExecutor()
|
|
355
|
+
self._total_phase: ActivePhase | None = None
|
|
356
|
+
self._provider_context_telemetry: dict[str, int] = {}
|
|
357
|
+
self._provider_dispatch_telemetry: dict[str, int] = {}
|
|
358
|
+
self._dispatch_guard_enforced = False
|
|
359
|
+
self._last_provider_invocation_id: str | None = None
|
|
360
|
+
|
|
361
|
+
def _start_phase(self, run_id: str, phase_name: str, **kwargs: object) -> ActivePhase | None:
|
|
362
|
+
return start_phase(self.root, run_id, phase_name, central_database=self.store.central_database, **kwargs)
|
|
363
|
+
|
|
364
|
+
def _resume_phase(self, run_id: str, phase_name: str, **kwargs: object) -> ActivePhase | None:
|
|
365
|
+
return start_or_resume_phase(self.root, run_id, phase_name, central_database=self.store.central_database, **kwargs)
|
|
366
|
+
|
|
367
|
+
def _provider_recovery_preflight(self, state: TransactionState) -> str | None:
|
|
368
|
+
"""Return a fail-closed reason before the sole same-run restart.
|
|
369
|
+
|
|
370
|
+
This deliberately relies on the host's already-held run lease and on
|
|
371
|
+
the process receipt that the client removes in its ``finally`` block;
|
|
372
|
+
a delay is never evidence that an old provider has stopped.
|
|
373
|
+
"""
|
|
374
|
+
if state.terminal or dismissal_for_run(self.root, state.run_id):
|
|
375
|
+
return "CANCELLED"
|
|
376
|
+
if len(state.provider_recovery_attempts) >= 1 and state.provider_recovery_attempts[0].get("result") != "ACTIVE":
|
|
377
|
+
return "PRECHECK_FAILED"
|
|
378
|
+
if self.active_lease is None or self.active_lease.run_id != state.run_id:
|
|
379
|
+
return "PRECHECK_FAILED"
|
|
380
|
+
try:
|
|
381
|
+
evidence = self.repository.inspect(self.root)
|
|
382
|
+
except Exception:
|
|
383
|
+
return "PRECHECK_FAILED"
|
|
384
|
+
if state.branch and evidence.branch != state.branch:
|
|
385
|
+
return "PRECHECK_FAILED"
|
|
386
|
+
provenance = verify_worktree_recovery(
|
|
387
|
+
self.root, run_id=state.run_id, branch=state.branch,
|
|
388
|
+
transaction_baseline_sha=state.last_verified_sha,
|
|
389
|
+
)
|
|
390
|
+
if provenance is False:
|
|
391
|
+
return "PRECHECK_FAILED"
|
|
392
|
+
process_path = self.root / ".engineering" / "status" / "runner_process.json"
|
|
393
|
+
try:
|
|
394
|
+
process = json.loads(process_path.read_text(encoding="utf-8"))
|
|
395
|
+
except (OSError, json.JSONDecodeError):
|
|
396
|
+
process = None
|
|
397
|
+
if isinstance(process, dict) and process.get("run_id") == state.run_id:
|
|
398
|
+
pid = process.get("pid")
|
|
399
|
+
if isinstance(pid, int) and pid > 0:
|
|
400
|
+
try:
|
|
401
|
+
os.kill(pid, 0)
|
|
402
|
+
except OSError:
|
|
403
|
+
process_path.unlink(missing_ok=True)
|
|
404
|
+
else:
|
|
405
|
+
return "PRECHECK_FAILED"
|
|
406
|
+
return None
|
|
407
|
+
|
|
408
|
+
def _recovery_record(
|
|
409
|
+
self, state: TransactionState, *, original: str, replacement: str,
|
|
410
|
+
eligibility: str, result: str, requested_at: str, started_at: str,
|
|
411
|
+
completed_at: str,
|
|
412
|
+
) -> TransactionState:
|
|
413
|
+
return replace(state, provider_recovery_attempts=({
|
|
414
|
+
"original_invocation_id": original, "replacement_invocation_id": replacement,
|
|
415
|
+
"phase": state.phase, "classification": "provider_turn_interrupted",
|
|
416
|
+
"eligibility": eligibility, "result": result,
|
|
417
|
+
"requested_at": requested_at, "started_at": started_at, "completed_at": completed_at,
|
|
418
|
+
},))
|
|
419
|
+
|
|
420
|
+
def _controlled_interruption_requested(self, state: TransactionState) -> bool:
|
|
421
|
+
"""Read the opt-in, run-bound qualification hook; prompt text cannot set it.
|
|
422
|
+
|
|
423
|
+
The value is deliberately exact and process-local. It is useful only
|
|
424
|
+
to qualification fixtures and remains inert for normal executions.
|
|
425
|
+
Durable consumption is represented by the recovery ledger.
|
|
426
|
+
"""
|
|
427
|
+
return consume_controlled_interruption_hook(
|
|
428
|
+
self.root, run_id=state.run_id, phase=state.phase,
|
|
429
|
+
)
|
|
430
|
+
|
|
431
|
+
def _recovery_state(self, run_id: str) -> dict[str, object] | None:
|
|
432
|
+
"""Read recovery truth from this runner's explicit lifecycle authority."""
|
|
433
|
+
return load_recovery_state(
|
|
434
|
+
self.root, run_id, central_database=self.store.central_database,
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
def _provider_process_boundary(self, state: TransactionState, process: object) -> None:
|
|
438
|
+
"""Record the real provider start boundary for a claimed recovery."""
|
|
439
|
+
write_runner_process(self.root, state.run_id, process)
|
|
440
|
+
if not isinstance(process, dict):
|
|
441
|
+
return
|
|
442
|
+
recovery = self._recovery_state(state.run_id)
|
|
443
|
+
receipt_id = recovery.get("process_receipt_id") if isinstance(recovery, dict) else None
|
|
444
|
+
if recovery and recovery.get("state") == "RECOVERY_STARTING" and isinstance(receipt_id, str):
|
|
445
|
+
record_provider_started(
|
|
446
|
+
self.root, run_id=state.run_id, receipt_id=receipt_id,
|
|
447
|
+
pid=int(process["pid"]), process_group=int(process["process_group"]),
|
|
448
|
+
central_database=self.store.central_database,
|
|
449
|
+
)
|
|
450
|
+
|
|
451
|
+
def _project_durable_recovery(self, state: TransactionState, recovery: dict[str, object]) -> None:
|
|
452
|
+
"""Keep the legacy checkpoint field read-only-compatible with SQLite."""
|
|
453
|
+
result = str(recovery.get("state") or "")
|
|
454
|
+
projected_result = {
|
|
455
|
+
"RECOVERED": "RECOVERED",
|
|
456
|
+
"EXHAUSTED": "INTERRUPTED_AGAIN",
|
|
457
|
+
"PRECHECK_FAILED": "PRECHECK_FAILED",
|
|
458
|
+
"AMBIGUOUS": "PROVIDER_FAILED",
|
|
459
|
+
}.get(result)
|
|
460
|
+
if projected_result is None:
|
|
461
|
+
return
|
|
462
|
+
self.store.save(self._recovery_record(
|
|
463
|
+
state,
|
|
464
|
+
original=str(recovery.get("triggering_invocation_id") or "unavailable"),
|
|
465
|
+
replacement=str(recovery.get("replacement_invocation_id") or "unavailable"),
|
|
466
|
+
eligibility="ELIGIBLE" if result in {"RECOVERED", "EXHAUSTED"} else result,
|
|
467
|
+
result=projected_result,
|
|
468
|
+
requested_at=str(recovery.get("requested_at") or "unknown"),
|
|
469
|
+
started_at=str(recovery.get("provider_confirmed_active_at") or "not_started"),
|
|
470
|
+
completed_at=str(recovery.get("completed_at") or "not_completed"),
|
|
471
|
+
))
|
|
472
|
+
|
|
473
|
+
def _confirm_deterministic_admission(self, state: TransactionState) -> tuple[TransactionState, str | None]:
|
|
474
|
+
"""Confirm the persisted provider-free decision at the dispatch boundary.
|
|
475
|
+
|
|
476
|
+
Managed runs spawned by the Inbox watcher carry the storage schema it
|
|
477
|
+
admitted. Those runs must have an immutable watcher decision of PASS.
|
|
478
|
+
Direct callers retain their provider-free host readiness path, but the
|
|
479
|
+
resulting PASS is still checkpointed before any provider can run.
|
|
480
|
+
"""
|
|
481
|
+
if state.admission_decision == "PASS" and state.admission_completed_at:
|
|
482
|
+
return state, None
|
|
483
|
+
source = "RUNNER"
|
|
484
|
+
if state.execution_mode == "MANAGED" and os.environ.get("ENGINEERING_PLATFORM_ADMITTED_STORAGE_SCHEMA"):
|
|
485
|
+
try:
|
|
486
|
+
admission = load_admission_decision(self.root, state.run_id, central_database=self.store.central_database)
|
|
487
|
+
except EngineeringStorageError:
|
|
488
|
+
admission = None
|
|
489
|
+
# A recovered row remains durable historical evidence after its
|
|
490
|
+
# originating phase; only that phase may consume the result.
|
|
491
|
+
if (
|
|
492
|
+
admission is None
|
|
493
|
+
or admission.get("run_id") != state.run_id
|
|
494
|
+
or not isinstance(admission.get("submission_id"), str)
|
|
495
|
+
or not admission["submission_id"]
|
|
496
|
+
or admission.get("decision") != "PASS"
|
|
497
|
+
or admission.get("execution_mode") != "MANAGED"
|
|
498
|
+
):
|
|
499
|
+
blocked = replace(
|
|
500
|
+
state,
|
|
501
|
+
admission_decision="BLOCKED",
|
|
502
|
+
admission_completed_at=datetime.now(timezone.utc).isoformat(),
|
|
503
|
+
admission_evidence_source="WATCHER",
|
|
504
|
+
)
|
|
505
|
+
self.store.save(blocked)
|
|
506
|
+
return blocked, "Provider dispatch refused: deterministic admission is not a persisted PASS."
|
|
507
|
+
source = "WATCHER"
|
|
508
|
+
admitted = replace(
|
|
509
|
+
state,
|
|
510
|
+
admission_decision="PASS",
|
|
511
|
+
admission_completed_at=datetime.now(timezone.utc).isoformat(),
|
|
512
|
+
admission_evidence_source=source,
|
|
513
|
+
)
|
|
514
|
+
self.store.save(admitted)
|
|
515
|
+
return admitted, None
|
|
516
|
+
|
|
517
|
+
def _require_provider_dispatch_admission(self, state: TransactionState) -> None:
|
|
518
|
+
"""Fail closed for every Codex-backed path, including reviewers."""
|
|
519
|
+
if self._dispatch_guard_enforced and (
|
|
520
|
+
state.admission_decision != "PASS" or not state.admission_completed_at
|
|
521
|
+
):
|
|
522
|
+
raise RunnerError("provider invocation refused: deterministic admission is not completed with PASS")
|
|
523
|
+
|
|
524
|
+
def _heartbeat(self) -> None:
|
|
525
|
+
if self.lease_heartbeat is not None and self.lease_heartbeat.error is not None:
|
|
526
|
+
raise RunnerError("active-run lease heartbeat was lost") from self.lease_heartbeat.error
|
|
527
|
+
if self.active_lease is not None:
|
|
528
|
+
self.active_lease = heartbeat_lease(self.root, self.active_lease, central_database=self.store.central_database)
|
|
529
|
+
if self.lease_heartbeat is not None:
|
|
530
|
+
self.lease_heartbeat.lease = self.active_lease
|
|
531
|
+
|
|
532
|
+
def _provider_readiness_gate(
|
|
533
|
+
self, state: TransactionState, *, require_codex: bool, require_github: bool
|
|
534
|
+
) -> TransactionState:
|
|
535
|
+
"""Persist a fail-closed provider block without losing the original phase.
|
|
536
|
+
|
|
537
|
+
A waiting pull request is passive GitHub observation, whereas every
|
|
538
|
+
agent action requires Codex too. The saved original action lets a
|
|
539
|
+
verified resume continue exactly where it stopped.
|
|
540
|
+
"""
|
|
541
|
+
missing = provider_readiness_failures(self.root, require_github=require_github)
|
|
542
|
+
if not require_codex:
|
|
543
|
+
missing = tuple(provider for provider in missing if provider != "CODEX")
|
|
544
|
+
if missing:
|
|
545
|
+
blocked = replace(
|
|
546
|
+
state,
|
|
547
|
+
next_action="provider_auth_repair_required",
|
|
548
|
+
diagnostic=redact_diagnostic(
|
|
549
|
+
"Provider readiness required before "
|
|
550
|
+
f"{state.auth_recovery_phase or state.phase}: {', '.join(missing)}."
|
|
551
|
+
),
|
|
552
|
+
auth_recovery_phase=state.auth_recovery_phase or state.phase,
|
|
553
|
+
auth_recovery_next_action=state.auth_recovery_next_action or state.next_action,
|
|
554
|
+
auth_recovery_providers=tuple(missing),
|
|
555
|
+
)
|
|
556
|
+
self.store.save(blocked)
|
|
557
|
+
write_live_status(self.root, blocked, blocked.next_action)
|
|
558
|
+
return blocked
|
|
559
|
+
if state.auth_recovery_phase is not None:
|
|
560
|
+
restored = replace(
|
|
561
|
+
state,
|
|
562
|
+
next_action=state.auth_recovery_next_action or state.next_action,
|
|
563
|
+
diagnostic=None,
|
|
564
|
+
auth_recovery_phase=None,
|
|
565
|
+
auth_recovery_next_action=None,
|
|
566
|
+
auth_recovery_providers=(),
|
|
567
|
+
)
|
|
568
|
+
self.store.save(restored)
|
|
569
|
+
write_live_status(self.root, restored, restored.next_action)
|
|
570
|
+
return restored
|
|
571
|
+
return state
|
|
572
|
+
|
|
573
|
+
def _require_agent_readiness(self, state: TransactionState) -> None:
|
|
574
|
+
checked = self._provider_readiness_gate(
|
|
575
|
+
state, require_codex=True, require_github=state.execution_mode == "MANAGED"
|
|
576
|
+
)
|
|
577
|
+
if checked.next_action == "provider_auth_repair_required":
|
|
578
|
+
raise ProviderReadinessBlocked(checked)
|
|
579
|
+
|
|
580
|
+
def _publish_reviewer_progress(
|
|
581
|
+
self,
|
|
582
|
+
state: TransactionState,
|
|
583
|
+
selection: ReviewerSelection,
|
|
584
|
+
event: str,
|
|
585
|
+
result: ReviewerResult | None = None,
|
|
586
|
+
) -> None:
|
|
587
|
+
"""Publish bounded reviewer lifecycle status without granting reviewer authority."""
|
|
588
|
+
self._heartbeat()
|
|
589
|
+
status_by_event = {"started": "running", "completed": "completed", "failed": "failed"}
|
|
590
|
+
status = status_by_event.get(event)
|
|
591
|
+
if status is None:
|
|
592
|
+
return
|
|
593
|
+
with self._reviewer_runtime_lock:
|
|
594
|
+
for reviewer in self.reviewer_runtime:
|
|
595
|
+
if reviewer.get("reviewer") != selection.reviewer:
|
|
596
|
+
continue
|
|
597
|
+
reviewer["status"] = status
|
|
598
|
+
if event == "started":
|
|
599
|
+
reviewer["started_at"] = datetime.now(timezone.utc).isoformat()
|
|
600
|
+
else:
|
|
601
|
+
reviewer["finished_at"] = datetime.now(timezone.utc).isoformat()
|
|
602
|
+
reviewer["failed"] = bool(result and result.failed)
|
|
603
|
+
churn = result.churn if result is not None and isinstance(result.churn, dict) else {}
|
|
604
|
+
command_count = churn.get("tool_loop_operations", 0)
|
|
605
|
+
reviewer["codex_commands_executed"] = (
|
|
606
|
+
command_count
|
|
607
|
+
if isinstance(command_count, int) and not isinstance(command_count, bool)
|
|
608
|
+
else 0
|
|
609
|
+
)
|
|
610
|
+
break
|
|
611
|
+
write_live_status(self.root, state, "Capability review: " + selection.reviewer, self.reviewer_runtime)
|
|
612
|
+
|
|
613
|
+
def _persist_agent_usage(self, run_id: str) -> None:
|
|
614
|
+
usage = getattr(self.agent, "last_usage", None)
|
|
615
|
+
if isinstance(usage, dict):
|
|
616
|
+
write_codex_usage(self.root, run_id, usage)
|
|
617
|
+
|
|
618
|
+
def _persist_provider_invocation(self, state: TransactionState, *, phase: str, role: str = "agent", started_at: str | None = None, observed_usage: dict[str, object] | None = None, observed_metadata: dict[str, object] | None = None, observed_churn: dict[str, object] | None = None, observed_duration: float | None = None, observed_snapshots: tuple[dict[str, int], ...] | None = None, interruption_reason: str | None = None, invocation_id: str | None = None) -> str | None:
|
|
619
|
+
"""Append safe per-invocation evidence without affecting execution outcome."""
|
|
620
|
+
usage = observed_usage if observed_usage is not None else getattr(self.agent, "last_usage", None)
|
|
621
|
+
snapshots = observed_snapshots if observed_snapshots is not None else getattr(self.agent, "last_usage_snapshots", ())
|
|
622
|
+
if not isinstance(usage, dict):
|
|
623
|
+
usage = {}
|
|
624
|
+
metadata = observed_metadata if observed_metadata is not None else getattr(self.agent, "last_runtime_metadata", None)
|
|
625
|
+
churn = observed_churn if observed_churn is not None else getattr(self.agent, "last_churn", None)
|
|
626
|
+
churn = {
|
|
627
|
+
**(churn if isinstance(churn, dict) else {}),
|
|
628
|
+
**self._provider_context_telemetry,
|
|
629
|
+
**self._provider_dispatch_telemetry,
|
|
630
|
+
}
|
|
631
|
+
escalations = getattr(self.agent, "last_context_escalations", ())
|
|
632
|
+
if isinstance(escalations, tuple):
|
|
633
|
+
safe_escalations = [item for item in escalations if isinstance(item, dict)]
|
|
634
|
+
if safe_escalations:
|
|
635
|
+
churn.update({
|
|
636
|
+
"context_scope_effective": ContextScope.INVESTIGATION.value,
|
|
637
|
+
"context_escalation_count": len(safe_escalations),
|
|
638
|
+
"context_escalation_reasons": ",".join(str(item.get("reason", "")) for item in safe_escalations),
|
|
639
|
+
"context_escalation_boundaries": ",".join(str(item.get("boundary_kind", "")) for item in safe_escalations),
|
|
640
|
+
"context_escalation_diagnostic": str(safe_escalations[-1].get("diagnostic", "")),
|
|
641
|
+
})
|
|
642
|
+
if interruption_reason:
|
|
643
|
+
# Existing bounded invocation telemetry is the durable diagnostic
|
|
644
|
+
# channel; an interrupted turn has no AgentResult or final usage.
|
|
645
|
+
churn["interruption_classification"] = "provider_turn_interrupted"
|
|
646
|
+
churn["interruption_reason"] = redact_diagnostic(interruption_reason, limit=120)
|
|
647
|
+
duration = observed_duration if observed_duration is not None else getattr(self.agent, "last_execution_seconds", None)
|
|
648
|
+
raw_model = metadata.get("raw_provider_model") if isinstance(metadata, dict) else None
|
|
649
|
+
normalized_model = normalize_codex_model(raw_model)
|
|
650
|
+
try:
|
|
651
|
+
connection = (
|
|
652
|
+
sqlite3.connect(self.store.central_database, isolation_level=None)
|
|
653
|
+
if self.store.central_database is not None
|
|
654
|
+
else open_storage(self.root)
|
|
655
|
+
)
|
|
656
|
+
try:
|
|
657
|
+
ordinal = int(connection.execute(
|
|
658
|
+
"SELECT COALESCE(MAX(ordinal), 0) + 1 FROM provider_invocations WHERE run_id=?", (state.run_id,)
|
|
659
|
+
).fetchone()[0])
|
|
660
|
+
finally:
|
|
661
|
+
connection.close()
|
|
662
|
+
now = datetime.now(timezone.utc).isoformat()
|
|
663
|
+
identifier = persist_provider_invocation(self.root, ProviderInvocation(
|
|
664
|
+
run_id=state.run_id, ordinal=ordinal, provider="codex_cli",
|
|
665
|
+
model=normalized_model,
|
|
666
|
+
model_authority=AUTHORITATIVE if isinstance(raw_model, str) else "UNAVAILABLE",
|
|
667
|
+
raw_provider_model=raw_model if isinstance(raw_model, str) else None,
|
|
668
|
+
phase=phase, role=role, started_at=started_at or now, completed_at=now,
|
|
669
|
+
duration_ms=round(duration * 1000) if isinstance(duration, (int, float)) and duration >= 0 else None,
|
|
670
|
+
usage=usage, runtime_metadata=metadata if isinstance(metadata, dict) else None,
|
|
671
|
+
retry_ordinal=state.repair_iterations, churn=churn if isinstance(churn, dict) else None,
|
|
672
|
+
usage_snapshots=snapshots if isinstance(snapshots, tuple) else (), invocation_id=invocation_id,
|
|
673
|
+
), central_database=self.store.central_database)
|
|
674
|
+
self._last_provider_invocation_id = identifier
|
|
675
|
+
return identifier
|
|
676
|
+
except (EngineeringStorageError, OSError, sqlite3.DatabaseError):
|
|
677
|
+
LOGGER.warning("Provider invocation telemetry is unavailable for run %s", state.run_id)
|
|
678
|
+
return None
|
|
679
|
+
|
|
680
|
+
def _record_agent_execution_time(self, state: TransactionState) -> TransactionState:
|
|
681
|
+
"""Accumulate only measured Codex CLI invocation time for this run."""
|
|
682
|
+
# A nested recovery invocation checkpoints its immutable ledger before
|
|
683
|
+
# returning to the ordinary lifecycle caller. Merge that one durable
|
|
684
|
+
# field here so a stale in-memory phase state cannot erase it.
|
|
685
|
+
try:
|
|
686
|
+
persisted = self.store.load(state.run_id)
|
|
687
|
+
if persisted.provider_recovery_attempts != state.provider_recovery_attempts:
|
|
688
|
+
state = replace(state, provider_recovery_attempts=persisted.provider_recovery_attempts)
|
|
689
|
+
except StateError:
|
|
690
|
+
pass
|
|
691
|
+
measured = getattr(self.agent, "last_execution_seconds", None)
|
|
692
|
+
if isinstance(measured, bool) or not isinstance(measured, (int, float)):
|
|
693
|
+
return state
|
|
694
|
+
if not 0 <= measured <= 86_400:
|
|
695
|
+
return state
|
|
696
|
+
return replace(
|
|
697
|
+
state,
|
|
698
|
+
agent_execution_seconds=round((state.agent_execution_seconds or 0) + measured, 3),
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
def _record_validation_evidence(self, state: TransactionState, result: AgentResult) -> TransactionState:
|
|
702
|
+
"""Persist only bounded report evidence; it has no lifecycle authority."""
|
|
703
|
+
if not result.validation_evidence:
|
|
704
|
+
return state
|
|
705
|
+
try:
|
|
706
|
+
profile_context = load_validation_context(self.root, state.run_id, central_database=self.store.central_database)
|
|
707
|
+
except EngineeringStorageError:
|
|
708
|
+
profile_context = None
|
|
709
|
+
required_controls = set(profile_context["required_validation_controls"]) if profile_context else set()
|
|
710
|
+
tier = profile_context.get("selected_validation_tier") if profile_context else None
|
|
711
|
+
for item in result.validation_evidence:
|
|
712
|
+
command, summary = item.get("command", ""), item.get("result", "")
|
|
713
|
+
kind = self._validation_kind(command)
|
|
714
|
+
if kind is None:
|
|
715
|
+
continue
|
|
716
|
+
status = self._validation_summary_status(summary)
|
|
717
|
+
try:
|
|
718
|
+
record_managed_validation(
|
|
719
|
+
self.root, run_id=state.run_id, control=f"validation_{kind}", state=status,
|
|
720
|
+
required=True, currentness=state.repair_iterations,
|
|
721
|
+
central_database=self.store.central_database,
|
|
722
|
+
)
|
|
723
|
+
validation_id = (
|
|
724
|
+
"git_diff_check" if kind == "format_or_diff" else
|
|
725
|
+
"documentation_contract" if kind == "documentation_contract" else
|
|
726
|
+
self._validation_id(command, kind) if kind == "browser_e2e" else
|
|
727
|
+
"repository_suite" if kind == "tests" and tier == "FULL" else
|
|
728
|
+
"engineering_python" if kind == "tests" else f"validation_{kind}"
|
|
729
|
+
)
|
|
730
|
+
record_validation_control_result(
|
|
731
|
+
self.root, run_id=state.run_id, validation_id=validation_id, category="agent",
|
|
732
|
+
control_identity=command[:160], required_for_profile=validation_id in required_controls, execution_status="EXECUTED",
|
|
733
|
+
result=status, evidence_ref="agent_result", observed_at=datetime.now(timezone.utc).isoformat(),
|
|
734
|
+
currentness=state.repair_iterations,
|
|
735
|
+
central_database=self.store.central_database,
|
|
736
|
+
)
|
|
737
|
+
except EngineeringStorageError:
|
|
738
|
+
LOGGER.warning("Managed validation evidence is unavailable for run %s", state.run_id)
|
|
739
|
+
return replace(state, validation_evidence=result.validation_evidence)
|
|
740
|
+
|
|
741
|
+
@staticmethod
|
|
742
|
+
def _validation_summary_status(summary: str) -> str:
|
|
743
|
+
"""Classify bounded validation prose without treating ``no errors`` as a failure."""
|
|
744
|
+
normalized = summary.casefold()
|
|
745
|
+
if "not applicable" in normalized:
|
|
746
|
+
return "NOT_APPLICABLE"
|
|
747
|
+
if any(token in normalized for token in ("unavailable", "not recorded")):
|
|
748
|
+
return "UNAVAILABLE"
|
|
749
|
+
# Remove explicitly negated error terms before looking for a failure.
|
|
750
|
+
# This keeps a real ``ERROR:`` or ``error detected`` fail-closed while
|
|
751
|
+
# allowing ordinary success summaries such as ``no whitespace errors``.
|
|
752
|
+
failure_subject = re.sub(
|
|
753
|
+
r"\b(?:no|without)(?:\s+[a-z0-9_-]+){0,4}\s+errors?\b", "", normalized,
|
|
754
|
+
)
|
|
755
|
+
if any(token in failure_subject for token in ("fail", "error", "blocked", "timeout", "timed out")):
|
|
756
|
+
return "FAIL"
|
|
757
|
+
if any(token in normalized for token in ("pass", "passed", "succeed", "no errors", "without errors")):
|
|
758
|
+
return "PASS"
|
|
759
|
+
return "UNAVAILABLE"
|
|
760
|
+
|
|
761
|
+
def _bind_validation_only_profile(
|
|
762
|
+
self, state: TransactionState, producer_context: object,
|
|
763
|
+
) -> TransactionState:
|
|
764
|
+
"""Resolve and snapshot a producer-selected registry profile.
|
|
765
|
+
|
|
766
|
+
This runs after deterministic admission and before any provider or
|
|
767
|
+
deterministic-control work. Its single immutable storage record is
|
|
768
|
+
the source of truth for the rest of the run.
|
|
769
|
+
"""
|
|
770
|
+
profile_payload = producer_context.get("validation_profile") if isinstance(producer_context, dict) else None
|
|
771
|
+
try:
|
|
772
|
+
profile, reference = resolve_producer_profile(profile_payload)
|
|
773
|
+
bindings = profile_control_bindings(profile)
|
|
774
|
+
record_validation_profile(
|
|
775
|
+
self.root, run_id=state.run_id, selected_validation_tier=profile.tier,
|
|
776
|
+
validation_profile_version=VALIDATION_PROFILE_VERSION,
|
|
777
|
+
required_validation_controls=profile.required_controls,
|
|
778
|
+
profile_reference=reference,
|
|
779
|
+
profile_selection_source="producer_execution_context",
|
|
780
|
+
control_bindings=bindings,
|
|
781
|
+
recorded_at=datetime.now(timezone.utc).isoformat(),
|
|
782
|
+
central_database=self.store.central_database,
|
|
783
|
+
)
|
|
784
|
+
except (EngineeringStorageError, ValidationProfileResolutionError):
|
|
785
|
+
return self._save_terminal(
|
|
786
|
+
state, "BLOCKED", "validation_profile_resolution",
|
|
787
|
+
"Selected validation profile is unresolved or unavailable.",
|
|
788
|
+
)
|
|
789
|
+
return state
|
|
790
|
+
|
|
791
|
+
def _execute_required_validation_controls(self, state: TransactionState) -> TransactionState:
|
|
792
|
+
"""Execute already-persisted required controls before qualification."""
|
|
793
|
+
try:
|
|
794
|
+
validation_context = load_validation_context(self.root, state.run_id, central_database=self.store.central_database)
|
|
795
|
+
except EngineeringStorageError:
|
|
796
|
+
validation_context = None
|
|
797
|
+
if not isinstance(validation_context, dict):
|
|
798
|
+
return self._save_terminal(state, "BLOCKED", "validation_profile_persistence", "Required validation profile evidence could not be loaded.")
|
|
799
|
+
required = validation_context.get("required_validation_controls")
|
|
800
|
+
if not isinstance(required, tuple) or not required:
|
|
801
|
+
return self._save_terminal(state, "BLOCKED", "validation_profile_persistence", "Required validation profile evidence does not contain required controls.")
|
|
802
|
+
bindings = validation_context.get("control_bindings")
|
|
803
|
+
if not isinstance(bindings, tuple) or len(bindings) != len(required):
|
|
804
|
+
return self._save_terminal(state, "BLOCKED", "validation_profile_persistence", "Required validation profile control bindings could not be loaded.")
|
|
805
|
+
binding_by_id = {binding.get("validation_id"): binding for binding in bindings if isinstance(binding, dict)}
|
|
806
|
+
if tuple(binding_by_id) != required:
|
|
807
|
+
return self._save_terminal(state, "BLOCKED", "validation_profile_persistence", "Required validation profile control bindings are invalid.")
|
|
808
|
+
validation = replace(state, phase="LOCAL_REPOSITORY_VALIDATION", next_action="execute_required_validation_controls")
|
|
809
|
+
self.store.save(validation)
|
|
810
|
+
write_live_status(self.root, validation, validation.next_action)
|
|
811
|
+
self._managed_action(validation, "VALIDATION_EXECUTION")
|
|
812
|
+
for ordinal, validation_id in enumerate(required, start=1):
|
|
813
|
+
binding = binding_by_id[validation_id]
|
|
814
|
+
command = binding.get("command")
|
|
815
|
+
if not isinstance(command, list) or not all(isinstance(item, str) and item for item in command):
|
|
816
|
+
return self._save_terminal(validation, "BLOCKED", "validation_profile_persistence", "Required validation profile control binding is invalid.")
|
|
817
|
+
if not command:
|
|
818
|
+
try:
|
|
819
|
+
record_validation_control_result(
|
|
820
|
+
self.root, run_id=validation.run_id, validation_id=validation_id,
|
|
821
|
+
category=str(binding.get("category") or "unavailable"),
|
|
822
|
+
control_identity=str(binding.get("control_identity") or validation_id),
|
|
823
|
+
required_for_profile=True, execution_status="NOT_EXECUTED",
|
|
824
|
+
result="UNAVAILABLE", evidence_ref="control_launcher_unavailable",
|
|
825
|
+
observed_at=datetime.now(timezone.utc).isoformat(), currentness=validation.repair_iterations,
|
|
826
|
+
central_database=self.store.central_database,
|
|
827
|
+
)
|
|
828
|
+
except EngineeringStorageError:
|
|
829
|
+
return self._save_terminal(validation, "BLOCKED", "validation_evidence_persistence", "Required validation control evidence could not be persisted.")
|
|
830
|
+
continue
|
|
831
|
+
launcher = ValidationControlLauncher(
|
|
832
|
+
validation_id=validation_id,
|
|
833
|
+
category=str(binding.get("category") or "unavailable"),
|
|
834
|
+
control_identity=str(binding.get("control_identity") or validation_id),
|
|
835
|
+
command=tuple(command),
|
|
836
|
+
)
|
|
837
|
+
observed_at = datetime.now(timezone.utc).isoformat()
|
|
838
|
+
command_id = f"required-control-{ordinal}-{uuid.uuid4().hex[:12]}"
|
|
839
|
+
span = self._start_phase(
|
|
840
|
+
validation.run_id, "VALIDATION", category="DETERMINISTIC_CONTROL",
|
|
841
|
+
attempt=max(1, validation.repair_iterations + 1),
|
|
842
|
+
metadata={"validation_id": launcher.validation_id, "command_id": command_id},
|
|
843
|
+
)
|
|
844
|
+
try:
|
|
845
|
+
record_validation_command_invocation(
|
|
846
|
+
self.root, run_id=validation.run_id, validation_id=launcher.validation_id,
|
|
847
|
+
command_id=command_id, category=launcher.category,
|
|
848
|
+
control_identity=launcher.control_identity, required_for_profile=True,
|
|
849
|
+
started_at=observed_at, currentness=validation.repair_iterations,
|
|
850
|
+
central_database=self.store.central_database,
|
|
851
|
+
)
|
|
852
|
+
except EngineeringStorageError:
|
|
853
|
+
complete_phase(self.root, span, outcome="FAILED")
|
|
854
|
+
return self._save_terminal(validation, "BLOCKED", "validation_evidence_persistence", "Required validation control invocation evidence could not be persisted.")
|
|
855
|
+
exit_code: int | None
|
|
856
|
+
previous_run_id = os.environ.get("ENGINEERING_PLATFORM_VALIDATION_RUN_ID")
|
|
857
|
+
os.environ["ENGINEERING_PLATFORM_VALIDATION_RUN_ID"] = validation.run_id
|
|
858
|
+
try:
|
|
859
|
+
command_outcome = self._run_required_validation_command(launcher.command)
|
|
860
|
+
finally:
|
|
861
|
+
if previous_run_id is None:
|
|
862
|
+
os.environ.pop("ENGINEERING_PLATFORM_VALIDATION_RUN_ID", None)
|
|
863
|
+
else:
|
|
864
|
+
os.environ["ENGINEERING_PLATFORM_VALIDATION_RUN_ID"] = previous_run_id
|
|
865
|
+
# Compatibility with direct host tests that intentionally stub the
|
|
866
|
+
# old scalar boundary; production always supplies a structured
|
|
867
|
+
# deterministic outcome with captured subprocess output.
|
|
868
|
+
if isinstance(command_outcome, int) or command_outcome is None:
|
|
869
|
+
exit_code = command_outcome
|
|
870
|
+
diagnostic_stdout = diagnostic_stderr = None
|
|
871
|
+
diagnostic_capture_available = False
|
|
872
|
+
else:
|
|
873
|
+
exit_code = command_outcome.exit_code
|
|
874
|
+
diagnostic_stdout = command_outcome.stdout
|
|
875
|
+
diagnostic_stderr = command_outcome.stderr
|
|
876
|
+
diagnostic_capture_available = command_outcome.diagnostic_capture_available
|
|
877
|
+
completed_at = datetime.now(timezone.utc).isoformat()
|
|
878
|
+
try:
|
|
879
|
+
record_validation_command_terminal(
|
|
880
|
+
self.root, run_id=validation.run_id, command_id=command_id,
|
|
881
|
+
completed_at=completed_at, exit_code=exit_code,
|
|
882
|
+
central_database=self.store.central_database,
|
|
883
|
+
)
|
|
884
|
+
result = "PASS" if exit_code == 0 else "FAIL" if exit_code is not None else "UNAVAILABLE"
|
|
885
|
+
record_validation_control_result(
|
|
886
|
+
self.root, run_id=validation.run_id, validation_id=launcher.validation_id,
|
|
887
|
+
category=launcher.category, control_identity=launcher.control_identity,
|
|
888
|
+
required_for_profile=True, execution_status="EXECUTED", result=result,
|
|
889
|
+
evidence_ref="command_terminal", observed_at=completed_at,
|
|
890
|
+
currentness=validation.repair_iterations,
|
|
891
|
+
central_database=self.store.central_database,
|
|
892
|
+
)
|
|
893
|
+
except EngineeringStorageError:
|
|
894
|
+
complete_phase(self.root, span, outcome="FAILED")
|
|
895
|
+
return self._save_terminal(validation, "BLOCKED", "validation_evidence_persistence", "Required validation control terminal evidence could not be persisted.")
|
|
896
|
+
if exit_code != 0:
|
|
897
|
+
try:
|
|
898
|
+
persist_validation_failure_diagnostic(
|
|
899
|
+
self.root, run_id=validation.run_id, command_id=command_id,
|
|
900
|
+
validation_id=launcher.validation_id,
|
|
901
|
+
control_identity=launcher.control_identity, exit_code=exit_code,
|
|
902
|
+
stdout=diagnostic_stdout, stderr=diagnostic_stderr,
|
|
903
|
+
capture_available=diagnostic_capture_available,
|
|
904
|
+
captured_at=completed_at,
|
|
905
|
+
central_database=self.store.central_database,
|
|
906
|
+
artifact_root=(self.store.central_database.parent / "artifacts") if self.store.central_database else None,
|
|
907
|
+
)
|
|
908
|
+
except (EngineeringStorageError, OSError):
|
|
909
|
+
# Diagnostics are supplementary. A capture failure must
|
|
910
|
+
# never erase or weaken the authoritative terminal exit.
|
|
911
|
+
LOGGER.warning("Required validation diagnostic capture unavailable for %s", command_id)
|
|
912
|
+
complete_phase(self.root, span, outcome="COMPLETE" if exit_code == 0 else "FAILED")
|
|
913
|
+
return validation
|
|
914
|
+
|
|
915
|
+
def _run_required_validation_command(self, command: tuple[str, ...]):
|
|
916
|
+
"""Run one deterministic control, preserving unavailable terminals."""
|
|
917
|
+
return self.validation_executor.run(self.root, command)
|
|
918
|
+
|
|
919
|
+
def _managed_action(self, state: TransactionState, action: str, authority: str = "AUTONOMOUS_EP_ACTION", *, actor: str = "execution_host", evidence_ref: str = "runtime") -> None:
|
|
920
|
+
"""Best-effort evidence instrumentation; it never changes lifecycle outcome."""
|
|
921
|
+
try:
|
|
922
|
+
record_managed_action(
|
|
923
|
+
self.root, run_id=state.run_id, action=action, authority=authority,
|
|
924
|
+
actor=actor, evidence_ref=evidence_ref,
|
|
925
|
+
central_database=self.store.central_database,
|
|
926
|
+
)
|
|
927
|
+
except EngineeringStorageError:
|
|
928
|
+
LOGGER.warning("Managed-autonomy evidence is unavailable for run %s", state.run_id)
|
|
929
|
+
|
|
930
|
+
def _managed_gate(self, state: TransactionState, gate_type: str, status: str, pr: int, *, resolved: bool = False) -> None:
|
|
931
|
+
try:
|
|
932
|
+
record_managed_gate(
|
|
933
|
+
self.root, run_id=state.run_id, gate_type=gate_type, status=status,
|
|
934
|
+
related_pr=pr, phase=state.phase,
|
|
935
|
+
resolution_actor="operator" if resolved else None,
|
|
936
|
+
resolved_at=datetime.now(timezone.utc).isoformat() if resolved else None,
|
|
937
|
+
central_database=self.store.central_database,
|
|
938
|
+
)
|
|
939
|
+
except EngineeringStorageError:
|
|
940
|
+
LOGGER.warning("Managed governance-gate evidence is unavailable for run %s", state.run_id)
|
|
941
|
+
|
|
942
|
+
def _managed_pr_check(self, state: TransactionState, pr: PullRequestEvidence) -> None:
|
|
943
|
+
"""Persist current GitHub required-check evidence separately from historical waits."""
|
|
944
|
+
role = state.transaction_kind
|
|
945
|
+
if role not in {"IMPLEMENTATION", "FINALIZATION"}:
|
|
946
|
+
return
|
|
947
|
+
check_state = "PASS" if pr.checks_terminal and pr.checks_passed else "FAIL" if pr.checks_terminal else "WAITING"
|
|
948
|
+
try:
|
|
949
|
+
record_managed_pr_check(
|
|
950
|
+
self.root, run_id=state.run_id, pr_number=pr.number, pr_role=role,
|
|
951
|
+
pr_state=pr.state, merge_commit=pr.merge_commit,
|
|
952
|
+
required_checks_state=check_state, evidence_ref="github_pr_status_check_rollup",
|
|
953
|
+
currentness=state.repair_iterations,
|
|
954
|
+
central_database=self.store.central_database,
|
|
955
|
+
)
|
|
956
|
+
except EngineeringStorageError:
|
|
957
|
+
LOGGER.warning("Managed PR check evidence is unavailable for run %s", state.run_id)
|
|
958
|
+
|
|
959
|
+
@staticmethod
|
|
960
|
+
def _audit_record(
|
|
961
|
+
*, iteration: int, failed_checks: str, proposed_action: str,
|
|
962
|
+
result: AgentResult | None, outcome: str, empty_summary: str,
|
|
963
|
+
) -> dict[str, str]:
|
|
964
|
+
"""Build one safe, schema-compatible bounded-attempt record."""
|
|
965
|
+
summary = (result.diagnostic if result else None) or empty_summary
|
|
966
|
+
return {
|
|
967
|
+
"iteration": str(iteration),
|
|
968
|
+
"observed_at": datetime.now(timezone.utc).isoformat(),
|
|
969
|
+
"failed_checks": redact_diagnostic(failed_checks),
|
|
970
|
+
"proposed_action": redact_diagnostic(proposed_action),
|
|
971
|
+
"agent_summary": redact_diagnostic(summary),
|
|
972
|
+
"commit_sha": result.commit_sha if result and result.commit_sha else "not_recorded",
|
|
973
|
+
"outcome": outcome,
|
|
974
|
+
}
|
|
975
|
+
|
|
976
|
+
def _record_repair_audit(self, state: TransactionState, *, failed_checks: str, objective: str, result: AgentResult | None, outcome: str) -> TransactionState:
|
|
977
|
+
"""Persist one bounded repair attempt, replacing its prior durable plan."""
|
|
978
|
+
record = self._audit_record(
|
|
979
|
+
iteration=state.repair_iterations,
|
|
980
|
+
failed_checks=failed_checks,
|
|
981
|
+
proposed_action=objective,
|
|
982
|
+
result=result,
|
|
983
|
+
outcome=outcome,
|
|
984
|
+
empty_summary="Agent invocation did not return a repair summary.",
|
|
985
|
+
)
|
|
986
|
+
if state.repair_audit and state.repair_audit[-1].get("iteration") == str(state.repair_iterations) and state.repair_audit[-1].get("outcome") == "planned":
|
|
987
|
+
return replace(state, repair_audit=state.repair_audit[:-1] + (record,))
|
|
988
|
+
return replace(state, repair_audit=state.repair_audit + (record,))
|
|
989
|
+
|
|
990
|
+
def _repair_plan(self, state: TransactionState) -> dict[str, str] | None:
|
|
991
|
+
"""Reload the exact pre-invocation plan; never reconstruct it from a prompt."""
|
|
992
|
+
if not state.repair_audit:
|
|
993
|
+
return None
|
|
994
|
+
plan = state.repair_audit[-1]
|
|
995
|
+
if plan.get("iteration") != str(state.repair_iterations) or plan.get("outcome") != "planned":
|
|
996
|
+
return None
|
|
997
|
+
return plan
|
|
998
|
+
|
|
999
|
+
def _advance_after_repair_agent_result(self, repair: TransactionState, result: AgentResult) -> TransactionState:
|
|
1000
|
+
"""Apply live or recovered Repair success using its persisted plan."""
|
|
1001
|
+
plan = self._repair_plan(repair)
|
|
1002
|
+
if plan is None:
|
|
1003
|
+
return self._save_terminal(repair, "BLOCKED", "repair_plan_missing", "Repair result cannot be resumed without its persisted repair plan.")
|
|
1004
|
+
failed_checks, objective = plan["failed_checks"], plan["proposed_action"]
|
|
1005
|
+
repair = self._record_repair_audit(
|
|
1006
|
+
repair, failed_checks=failed_checks, objective=objective, result=result,
|
|
1007
|
+
outcome="agent_failed" if result.terminal_state in {"BLOCKED", "FAILED"} else "submitted_for_recheck",
|
|
1008
|
+
)
|
|
1009
|
+
self.store.save(repair)
|
|
1010
|
+
if result.terminal_state in {"BLOCKED", "FAILED"}:
|
|
1011
|
+
return self._save_terminal(repair, result.terminal_state, "external_action_required", result.diagnostic)
|
|
1012
|
+
if result.pull_request != repair.pull_request:
|
|
1013
|
+
return self._save_terminal(repair, "BLOCKED", "bounded_scope_conflict", "Repair did not preserve the bounded pull request.")
|
|
1014
|
+
return self._poll(replace(repair, phase="WAIT_FOR_TERMINAL_EVIDENCE", next_action="poll_required_checks"), result)
|
|
1015
|
+
|
|
1016
|
+
def _record_local_validation_audit(self, state: TransactionState, *, result: AgentResult | None, outcome: str, profile: ValidationProfile) -> TransactionState:
|
|
1017
|
+
"""Append one bounded local-validation iteration without sharing PR repair budget."""
|
|
1018
|
+
return replace(state, local_validation_audit=state.local_validation_audit + (self._audit_record(
|
|
1019
|
+
iteration=state.local_validation_iterations,
|
|
1020
|
+
failed_checks=(result.diagnostic if result else None) or "Local repository validation did not return a passing result.",
|
|
1021
|
+
proposed_action=f"{profile.tier}: {'; '.join(profile.commands)}",
|
|
1022
|
+
result=result,
|
|
1023
|
+
outcome=outcome,
|
|
1024
|
+
empty_summary="Agent invocation did not return a validation summary.",
|
|
1025
|
+
),))
|
|
1026
|
+
|
|
1027
|
+
@staticmethod
|
|
1028
|
+
def _is_environmental_validation_instability(result: AgentResult) -> bool:
|
|
1029
|
+
"""Require explicit classification and contradictory bounded test evidence.
|
|
1030
|
+
|
|
1031
|
+
This does not pass the implementation run. It only stops repeated
|
|
1032
|
+
implementation retries when the same validation environment has been
|
|
1033
|
+
demonstrated as unstable.
|
|
1034
|
+
"""
|
|
1035
|
+
if result.validation_disposition != "environmental_instability" or not result.validation_evidence:
|
|
1036
|
+
return False
|
|
1037
|
+
summaries = " ".join(item.get("result", "").casefold() for item in result.validation_evidence)
|
|
1038
|
+
passed = any(token in summaries for token in ("pass", "passed", "succeeded"))
|
|
1039
|
+
failed = any(token in summaries for token in ("fail", "failed", "timeout", "timed out", "error"))
|
|
1040
|
+
return passed and failed
|
|
1041
|
+
|
|
1042
|
+
@staticmethod
|
|
1043
|
+
def _has_failed_validation_evidence(result: AgentResult) -> bool:
|
|
1044
|
+
"""Return whether an agent supplied bounded evidence of a failed local check."""
|
|
1045
|
+
if not result.validation_evidence:
|
|
1046
|
+
return False
|
|
1047
|
+
summaries = " ".join(
|
|
1048
|
+
item.get("result", "").casefold()
|
|
1049
|
+
for item in result.validation_evidence
|
|
1050
|
+
if isinstance(item, dict)
|
|
1051
|
+
)
|
|
1052
|
+
return any(token in summaries for token in ("fail", "failed", "timeout", "timed out", "error"))
|
|
1053
|
+
|
|
1054
|
+
@staticmethod
|
|
1055
|
+
def _is_external_agent_block(result: AgentResult) -> bool:
|
|
1056
|
+
"""Keep explicit external blocks out of the mutable validation route."""
|
|
1057
|
+
return result.terminal_condition == "external_blocked"
|
|
1058
|
+
|
|
1059
|
+
def _is_recoverable_implementation_validation_failure(
|
|
1060
|
+
self, state: TransactionState, result: AgentResult
|
|
1061
|
+
) -> bool:
|
|
1062
|
+
"""Admit only a verified implementation commit into local validation repair.
|
|
1063
|
+
|
|
1064
|
+
The implementation provider may faithfully return FAILED after it has
|
|
1065
|
+
committed a bounded change and discovered that the broader local suite
|
|
1066
|
+
still fails. That is a product-validation result, not an external
|
|
1067
|
+
dependency. It is safe to enter the existing three-attempt local gate
|
|
1068
|
+
only after the host recorded the exact branch/HEAD evidence.
|
|
1069
|
+
"""
|
|
1070
|
+
if not (
|
|
1071
|
+
result.terminal_state == "FAILED"
|
|
1072
|
+
and result.branch
|
|
1073
|
+
and result.branch != "main"
|
|
1074
|
+
and result.commit_sha
|
|
1075
|
+
and not result.pull_request
|
|
1076
|
+
and not self._is_external_agent_block(result)
|
|
1077
|
+
and self._has_failed_validation_evidence(result)
|
|
1078
|
+
):
|
|
1079
|
+
return False
|
|
1080
|
+
return any(
|
|
1081
|
+
item.get("phase") == "EXECUTE_AGENT"
|
|
1082
|
+
and item.get("commit_sha") == result.commit_sha
|
|
1083
|
+
for item in state.commit_evidence
|
|
1084
|
+
)
|
|
1085
|
+
|
|
1086
|
+
@staticmethod
|
|
1087
|
+
def _append_verified_commit_evidence(
|
|
1088
|
+
state: TransactionState, *, phase: str, commit_sha: str, description: str
|
|
1089
|
+
) -> TransactionState:
|
|
1090
|
+
"""Append immutable, compact commit evidence after its caller verified it.
|
|
1091
|
+
|
|
1092
|
+
The checkpoint save is the transaction boundary. Deduplication by
|
|
1093
|
+
phase and SHA makes retries idempotent without rewriting earlier
|
|
1094
|
+
operational evidence.
|
|
1095
|
+
"""
|
|
1096
|
+
if not re.fullmatch(r"[0-9a-f]{40}", commit_sha):
|
|
1097
|
+
return state
|
|
1098
|
+
if len(state.commit_evidence) >= MAX_COMMIT_EVIDENCE_RECORDS:
|
|
1099
|
+
return state
|
|
1100
|
+
if any(item["phase"] == phase and item["commit_sha"] == commit_sha for item in state.commit_evidence):
|
|
1101
|
+
return state
|
|
1102
|
+
try:
|
|
1103
|
+
record = verified_commit_evidence_record(
|
|
1104
|
+
phase=phase,
|
|
1105
|
+
observed_at=datetime.now(timezone.utc).isoformat(),
|
|
1106
|
+
commit_sha=commit_sha,
|
|
1107
|
+
description=redact_diagnostic(description),
|
|
1108
|
+
)
|
|
1109
|
+
except StateError:
|
|
1110
|
+
return state
|
|
1111
|
+
return replace(state, commit_evidence=state.commit_evidence + (record,))
|
|
1112
|
+
|
|
1113
|
+
def _record_verified_result_commit(
|
|
1114
|
+
self, state: TransactionState, result: AgentResult, *, phase: str, description: str
|
|
1115
|
+
) -> TransactionState:
|
|
1116
|
+
"""Record an agent commit only when the checked-out repository proves it.
|
|
1117
|
+
|
|
1118
|
+
A reported SHA is intentionally insufficient: the live repository must
|
|
1119
|
+
be clean, on the reported transaction branch, and at that exact SHA.
|
|
1120
|
+
Any unavailable or mismatched evidence is simply not recorded.
|
|
1121
|
+
"""
|
|
1122
|
+
if not result.commit_sha or not re.fullmatch(r"[0-9a-f]{40}", result.commit_sha):
|
|
1123
|
+
return state
|
|
1124
|
+
expected_branch = result.branch or state.branch
|
|
1125
|
+
if not expected_branch or expected_branch == "main":
|
|
1126
|
+
return state
|
|
1127
|
+
try:
|
|
1128
|
+
evidence = self.repository.inspect(self.root)
|
|
1129
|
+
except RunnerError:
|
|
1130
|
+
return state
|
|
1131
|
+
if not (evidence.clean and evidence.branch == expected_branch and evidence.head_sha == result.commit_sha):
|
|
1132
|
+
return state
|
|
1133
|
+
return self._append_verified_commit_evidence(
|
|
1134
|
+
state, phase=phase, commit_sha=result.commit_sha, description=description,
|
|
1135
|
+
)
|
|
1136
|
+
|
|
1137
|
+
@staticmethod
|
|
1138
|
+
def _validation_kind(command: str) -> str | None:
|
|
1139
|
+
"""Classify only known validation commands at their live boundary.
|
|
1140
|
+
|
|
1141
|
+
The command itself is transient observability input: timing metadata
|
|
1142
|
+
retains the category, never command text or output.
|
|
1143
|
+
"""
|
|
1144
|
+
normalized = command.casefold()
|
|
1145
|
+
if any(tool in normalized for tool in ("markdown", "link", "documentation", "document-contract")):
|
|
1146
|
+
return "documentation_contract"
|
|
1147
|
+
if any(tool in normalized for tool in ("ruff", "flake8", "mypy", "pyright")):
|
|
1148
|
+
return "static_analysis"
|
|
1149
|
+
if any(tool in normalized for tool in ("bandit", "semgrep", "codeql", "pip-audit", "safety")):
|
|
1150
|
+
return "security"
|
|
1151
|
+
if "git diff --check" in normalized or "prettier" in normalized or "black --check" in normalized:
|
|
1152
|
+
return "format_or_diff"
|
|
1153
|
+
if any(tool in normalized for tool in ("npm run test:engineering-dashboard", "playwright", "selenium", "cypress", "e2e")):
|
|
1154
|
+
return "browser_e2e"
|
|
1155
|
+
if any(tool in normalized for tool in ("pytest", "unittest", "tox", "nox")):
|
|
1156
|
+
return "tests"
|
|
1157
|
+
return None
|
|
1158
|
+
|
|
1159
|
+
@staticmethod
|
|
1160
|
+
def _validation_id(command: str, kind: str) -> str:
|
|
1161
|
+
"""Reserve dashboard_browser for the canonical dashboard suite only."""
|
|
1162
|
+
if kind == "browser_e2e" and is_canonical_dashboard_command(command):
|
|
1163
|
+
return "dashboard_browser"
|
|
1164
|
+
return f"validation_{kind}"
|
|
1165
|
+
|
|
1166
|
+
def _invoke_provider_attempt_with_timing(self, state: TransactionState, prompt: str, *, repair: bool = False, quality: bool = False, local_validation: bool = False, attempt: int | None = None) -> AgentResult:
|
|
1167
|
+
"""Run one provider attempt; recovery launch authority lives in storage."""
|
|
1168
|
+
# Baseline capture is content-free and idempotent. Recovery progress
|
|
1169
|
+
# is captured separately at the interruption boundary below.
|
|
1170
|
+
capture_worktree_provenance(
|
|
1171
|
+
self.root, run_id=state.run_id, phase=state.phase, stage="baseline",
|
|
1172
|
+
transaction_baseline_sha=state.last_verified_sha,
|
|
1173
|
+
)
|
|
1174
|
+
role = role_for_phase(state.phase, repair=repair, quality=quality)
|
|
1175
|
+
initial_scope = initial_context_scope(
|
|
1176
|
+
phase=state.phase, repair_iterations=state.repair_iterations,
|
|
1177
|
+
objective=Path(state.prompt_path).read_text(encoding="utf-8"),
|
|
1178
|
+
)
|
|
1179
|
+
decision = provider_need_for_phase(state.phase)
|
|
1180
|
+
if not decision.required:
|
|
1181
|
+
raise RunnerError(f"provider invocation refused: {decision.reason}")
|
|
1182
|
+
self._require_provider_dispatch_admission(state)
|
|
1183
|
+
self._provider_context_telemetry = {
|
|
1184
|
+
"context_projected_bytes": len(prompt.encode("utf-8")),
|
|
1185
|
+
"context_budget_version": 1,
|
|
1186
|
+
"context_scope_policy": POLICY_ID,
|
|
1187
|
+
"context_scope_initial": initial_scope.value,
|
|
1188
|
+
"context_scope_effective": initial_scope.value,
|
|
1189
|
+
"context_escalation_count": 0,
|
|
1190
|
+
}
|
|
1191
|
+
self._require_agent_readiness(state)
|
|
1192
|
+
process_callback = getattr(self.agent, "set_process_callback", None)
|
|
1193
|
+
if callable(process_callback):
|
|
1194
|
+
process_callback(lambda process: self._provider_process_boundary(state, process))
|
|
1195
|
+
parent = (
|
|
1196
|
+
self._start_phase(state.run_id, "REPAIR", attempt=state.repair_iterations, metadata={"iteration": state.repair_iterations})
|
|
1197
|
+
if repair else self._start_phase(state.run_id, "QUALITY_CONTROL", metadata={"kind": "autonomous_refactor_quality"}) if quality else None
|
|
1198
|
+
)
|
|
1199
|
+
provider_attempt = attempt if attempt is not None else max(1, state.repair_iterations + 1)
|
|
1200
|
+
provider_metadata: dict[str, object] = {"provider": "codex_cli"}
|
|
1201
|
+
if local_validation:
|
|
1202
|
+
# This invocation is distinct from the implementation agent even
|
|
1203
|
+
# though both emit PROVIDER_EXECUTION spans. The read-only
|
|
1204
|
+
# lifecycle projection uses this bounded marker to avoid showing
|
|
1205
|
+
# their combined duration on both visible steps.
|
|
1206
|
+
provider_metadata["lifecycle_step"] = "LOCAL_REPOSITORY_VALIDATION"
|
|
1207
|
+
provider = self._start_phase(
|
|
1208
|
+
state.run_id, "PROVIDER_EXECUTION", parent_phase_id=parent.phase_id if parent else None,
|
|
1209
|
+
attempt=provider_attempt, metadata=provider_metadata,
|
|
1210
|
+
)
|
|
1211
|
+
validation_spans: dict[str, ActivePhase | None] = {}
|
|
1212
|
+
validation_commands: dict[str, tuple[str, str]] = {}
|
|
1213
|
+
|
|
1214
|
+
def command_boundary(event: str, command_id: str, command: str, exit_code: int | None = None) -> None:
|
|
1215
|
+
if event == "started":
|
|
1216
|
+
kind = self._validation_kind(command)
|
|
1217
|
+
if kind is not None:
|
|
1218
|
+
validation_id = self._validation_id(command, kind)
|
|
1219
|
+
try:
|
|
1220
|
+
profile = load_validation_context(self.root, state.run_id, central_database=self.store.central_database)
|
|
1221
|
+
required = validation_id in set(profile["required_validation_controls"]) if profile else False
|
|
1222
|
+
started_at = datetime.now(timezone.utc).isoformat()
|
|
1223
|
+
record_validation_command_invocation(
|
|
1224
|
+
self.root, run_id=state.run_id, validation_id=validation_id, command_id=command_id,
|
|
1225
|
+
category="agent", control_identity=command[:160], required_for_profile=required,
|
|
1226
|
+
started_at=started_at, currentness=state.repair_iterations,
|
|
1227
|
+
central_database=self.store.central_database,
|
|
1228
|
+
)
|
|
1229
|
+
validation_commands[command_id] = (validation_id, started_at)
|
|
1230
|
+
except EngineeringStorageError:
|
|
1231
|
+
LOGGER.warning("Validation command start evidence is unavailable for run %s", state.run_id)
|
|
1232
|
+
validation_spans[command_id] = self._start_phase(
|
|
1233
|
+
state.run_id,
|
|
1234
|
+
"VALIDATION",
|
|
1235
|
+
parent_phase_id=provider.phase_id if provider else None,
|
|
1236
|
+
attempt=provider_attempt,
|
|
1237
|
+
# This persisted span is the invocation receipt. The
|
|
1238
|
+
# result remains separate and is recorded only after
|
|
1239
|
+
# the agent returns explicit validation evidence.
|
|
1240
|
+
metadata={
|
|
1241
|
+
"validation_kind": kind,
|
|
1242
|
+
"validation_id": validation_id,
|
|
1243
|
+
"command_id": command_id,
|
|
1244
|
+
},
|
|
1245
|
+
)
|
|
1246
|
+
elif event == "completed":
|
|
1247
|
+
if command_id in validation_commands:
|
|
1248
|
+
try:
|
|
1249
|
+
validation_id, _ = validation_commands[command_id]
|
|
1250
|
+
evidence_ref = "command_terminal"
|
|
1251
|
+
if validation_id == "dashboard_browser":
|
|
1252
|
+
evidence = load_dashboard_evidence(self.root, state.run_id)
|
|
1253
|
+
evidence_path = dashboard_evidence_path(self.root, state.run_id)
|
|
1254
|
+
if evidence is not None and evidence_path.is_file():
|
|
1255
|
+
artifact_id = f"dashboard-browser:{state.run_id}:{command_id}"
|
|
1256
|
+
record_artifact(
|
|
1257
|
+
self.root, evidence_path, artifact_id=artifact_id,
|
|
1258
|
+
artifact_type="DASHBOARD_BROWSER_SHARD_RESULTS", content_type="application/json",
|
|
1259
|
+
created_at=datetime.now(timezone.utc).isoformat(), run_id=state.run_id,
|
|
1260
|
+
)
|
|
1261
|
+
evidence_ref = f"artifact:{artifact_id}"
|
|
1262
|
+
record_validation_command_terminal(
|
|
1263
|
+
self.root, run_id=state.run_id, command_id=command_id,
|
|
1264
|
+
completed_at=datetime.now(timezone.utc).isoformat(), exit_code=exit_code,
|
|
1265
|
+
evidence_ref=evidence_ref,
|
|
1266
|
+
central_database=self.store.central_database,
|
|
1267
|
+
)
|
|
1268
|
+
except EngineeringStorageError:
|
|
1269
|
+
LOGGER.warning("Validation command terminal evidence is unavailable for run %s", state.run_id)
|
|
1270
|
+
active = validation_spans.pop(command_id, None)
|
|
1271
|
+
complete_phase(self.root, active)
|
|
1272
|
+
|
|
1273
|
+
command_callback = getattr(self.agent, "set_command_callback", None)
|
|
1274
|
+
if callable(command_callback):
|
|
1275
|
+
command_callback(command_boundary)
|
|
1276
|
+
invocation_started = datetime.now(timezone.utc).isoformat()
|
|
1277
|
+
set_handoff_deadline = getattr(self.agent, "set_handoff_deadline_callback", None)
|
|
1278
|
+
deadline_started = time.monotonic()
|
|
1279
|
+
timeout = agent_timeout(
|
|
1280
|
+
phase=state.phase, repair=repair, quality=quality,
|
|
1281
|
+
local_validation=local_validation,
|
|
1282
|
+
)
|
|
1283
|
+
if callable(set_handoff_deadline):
|
|
1284
|
+
# Every managed provider action has a host-owned maximum. The
|
|
1285
|
+
# client terminates the whole invocation process group when it
|
|
1286
|
+
# expires, so an inherited stdout pipe cannot strand the worker.
|
|
1287
|
+
set_handoff_deadline(
|
|
1288
|
+
lambda: time.monotonic() - deadline_started >= timeout.seconds
|
|
1289
|
+
)
|
|
1290
|
+
prior_validation_run_id = os.environ.get("ENGINEERING_PLATFORM_VALIDATION_RUN_ID")
|
|
1291
|
+
os.environ["ENGINEERING_PLATFORM_VALIDATION_RUN_ID"] = state.run_id
|
|
1292
|
+
try:
|
|
1293
|
+
if self._controlled_interruption_requested(state):
|
|
1294
|
+
raise CodexInvocationError(
|
|
1295
|
+
"Provider turn interrupted before returning the required structured AgentResult.",
|
|
1296
|
+
"Controlled qualification interruption.", next_action="NONE",
|
|
1297
|
+
terminal_condition="provider_turn_interrupted",
|
|
1298
|
+
interruption_reason="controlled_qualification_interruption",
|
|
1299
|
+
)
|
|
1300
|
+
result = self.agent.invoke(self.root, prompt)
|
|
1301
|
+
except KeyboardInterrupt as error:
|
|
1302
|
+
# A managed SIGINT/SIGTERM while the provider is active means no
|
|
1303
|
+
# valid AgentResult exists. Persist the canonical interruption
|
|
1304
|
+
# before returning control to the normal terminal path.
|
|
1305
|
+
interruption_reason = "host_shutdown_during_provider_turn"
|
|
1306
|
+
self._persist_provider_invocation(
|
|
1307
|
+
state, phase="REPAIR" if repair else "QUALITY_CONTROL" if quality else "PROVIDER_EXECUTION",
|
|
1308
|
+
role=role.value, started_at=invocation_started, interruption_reason=interruption_reason,
|
|
1309
|
+
)
|
|
1310
|
+
for active in validation_spans.values():
|
|
1311
|
+
complete_phase(self.root, active, outcome="INTERRUPTED")
|
|
1312
|
+
complete_phase(self.root, provider, outcome="INTERRUPTED")
|
|
1313
|
+
if parent:
|
|
1314
|
+
complete_phase(self.root, parent, outcome="INTERRUPTED")
|
|
1315
|
+
raise CodexInvocationError(
|
|
1316
|
+
"Provider turn interrupted before returning the required structured AgentResult.",
|
|
1317
|
+
"Execution Host received a shutdown signal while a provider turn was active.",
|
|
1318
|
+
next_action="NONE",
|
|
1319
|
+
terminal_condition="provider_turn_interrupted",
|
|
1320
|
+
interruption_reason=interruption_reason,
|
|
1321
|
+
) from error
|
|
1322
|
+
except CodexHandoffTimeout as error:
|
|
1323
|
+
# Finalization has a dedicated reconciliation path below. Every
|
|
1324
|
+
# other timed-out action becomes an ordinary, durable provider
|
|
1325
|
+
# failure rather than leaving the project lease active forever.
|
|
1326
|
+
if repair or state.transaction_kind == "FINALIZATION" or state.phase.upper() == "FINALIZE_AGENT":
|
|
1327
|
+
raise
|
|
1328
|
+
raise CodexInvocationError(
|
|
1329
|
+
f"Provider action exceeded the {timeout.seconds // 60}-minute host-owned deadline.",
|
|
1330
|
+
"The provider invocation was stopped after its configured workflow deadline.",
|
|
1331
|
+
next_action="inspect_codex_cli",
|
|
1332
|
+
terminal_condition="provider_invocation_timeout",
|
|
1333
|
+
) from error
|
|
1334
|
+
except Exception as error:
|
|
1335
|
+
interruption_reason = error.interruption_reason if isinstance(error, CodexInvocationError) else None
|
|
1336
|
+
recovery = self._recovery_state(state.run_id)
|
|
1337
|
+
replacement_id = (
|
|
1338
|
+
recovery.get("replacement_invocation_id")
|
|
1339
|
+
if isinstance(recovery, dict) and recovery.get("state") == "RECOVERY_IN_PROGRESS"
|
|
1340
|
+
else None
|
|
1341
|
+
)
|
|
1342
|
+
original_invocation = self._persist_provider_invocation(
|
|
1343
|
+
state, phase="REPAIR" if repair else "QUALITY_CONTROL" if quality else "PROVIDER_EXECUTION",
|
|
1344
|
+
role=role.value, started_at=invocation_started, interruption_reason=interruption_reason,
|
|
1345
|
+
invocation_id=replacement_id if isinstance(replacement_id, str) else None,
|
|
1346
|
+
)
|
|
1347
|
+
for active in validation_spans.values():
|
|
1348
|
+
complete_phase(self.root, active, outcome="INTERRUPTED")
|
|
1349
|
+
outcome = "INTERRUPTED" if interruption_reason else "FAILED"
|
|
1350
|
+
complete_phase(self.root, provider, outcome=outcome)
|
|
1351
|
+
if parent:
|
|
1352
|
+
complete_phase(self.root, parent, outcome=outcome)
|
|
1353
|
+
# A single provider-proven interruption may create durable
|
|
1354
|
+
# recovery availability. This attempt never launches a retry:
|
|
1355
|
+
# the outer state-driven controller consumes that evidence.
|
|
1356
|
+
if (
|
|
1357
|
+
interruption_reason and isinstance(error, CodexInvocationError)
|
|
1358
|
+
and original_invocation and not isinstance(recovery, dict)
|
|
1359
|
+
):
|
|
1360
|
+
try:
|
|
1361
|
+
create_recovery_available(
|
|
1362
|
+
self.root, run_id=state.run_id, triggering_invocation_id=original_invocation,
|
|
1363
|
+
lifecycle_phase=state.phase, branch=state.branch,
|
|
1364
|
+
worktree_identity=str(self.root.resolve()),
|
|
1365
|
+
lease_id=self.active_lease.lease_id if self.active_lease else None,
|
|
1366
|
+
central_database=self.store.central_database,
|
|
1367
|
+
)
|
|
1368
|
+
capture_worktree_provenance(
|
|
1369
|
+
self.root, run_id=state.run_id, phase=state.phase, stage="interrupted",
|
|
1370
|
+
transaction_baseline_sha=state.last_verified_sha,
|
|
1371
|
+
)
|
|
1372
|
+
except EngineeringStorageError:
|
|
1373
|
+
raise CodexInvocationError(
|
|
1374
|
+
"Provider interruption recovery evidence could not be persisted.",
|
|
1375
|
+
"Recovery storage is unavailable.", next_action="NONE",
|
|
1376
|
+
terminal_condition="provider_turn_interrupted",
|
|
1377
|
+
interruption_reason=interruption_reason,
|
|
1378
|
+
) from error
|
|
1379
|
+
elif isinstance(recovery, dict) and recovery.get("state") == "RECOVERY_IN_PROGRESS":
|
|
1380
|
+
# Attempt two is terminal evidence, never another launch
|
|
1381
|
+
# opportunity. Unknown non-interruption failures are marked
|
|
1382
|
+
# ambiguous by the controller.
|
|
1383
|
+
record_replacement_terminal(
|
|
1384
|
+
self.root, run_id=state.run_id,
|
|
1385
|
+
outcome="INTERRUPTED" if interruption_reason else "FAILED",
|
|
1386
|
+
central_database=self.store.central_database,
|
|
1387
|
+
)
|
|
1388
|
+
elif isinstance(recovery, dict) and recovery.get("state") == "RECOVERY_STARTING":
|
|
1389
|
+
# No process callback was observed, so the provider adapter
|
|
1390
|
+
# authoritatively failed before entering provider execution.
|
|
1391
|
+
record_pre_execution_launch_failure(
|
|
1392
|
+
self.root, run_id=state.run_id, diagnostic_code=type(error).__name__,
|
|
1393
|
+
central_database=self.store.central_database,
|
|
1394
|
+
)
|
|
1395
|
+
raise
|
|
1396
|
+
finally:
|
|
1397
|
+
if prior_validation_run_id is None:
|
|
1398
|
+
os.environ.pop("ENGINEERING_PLATFORM_VALIDATION_RUN_ID", None)
|
|
1399
|
+
else:
|
|
1400
|
+
os.environ["ENGINEERING_PLATFORM_VALIDATION_RUN_ID"] = prior_validation_run_id
|
|
1401
|
+
if callable(command_callback):
|
|
1402
|
+
command_callback(None)
|
|
1403
|
+
if callable(process_callback):
|
|
1404
|
+
process_callback(None)
|
|
1405
|
+
if callable(set_handoff_deadline):
|
|
1406
|
+
set_handoff_deadline(None)
|
|
1407
|
+
durable_recovery = self._recovery_state(state.run_id)
|
|
1408
|
+
replacement_id = (
|
|
1409
|
+
durable_recovery.get("replacement_invocation_id")
|
|
1410
|
+
if isinstance(durable_recovery, dict) and durable_recovery.get("state") == "RECOVERY_IN_PROGRESS"
|
|
1411
|
+
else None
|
|
1412
|
+
)
|
|
1413
|
+
replacement = self._persist_provider_invocation(state, phase="REPAIR" if repair else "QUALITY_CONTROL" if quality else "PROVIDER_EXECUTION", role=role.value, started_at=invocation_started, invocation_id=replacement_id if isinstance(replacement_id, str) else None)
|
|
1414
|
+
if isinstance(durable_recovery, dict) and durable_recovery.get("state") == "RECOVERY_IN_PROGRESS":
|
|
1415
|
+
result_reference = persist_recovery_agent_result(
|
|
1416
|
+
self.root, run_id=state.run_id, invocation_id=str(replacement_id), result=result,
|
|
1417
|
+
central_database=self.store.central_database,
|
|
1418
|
+
artifact_root=(self.store.central_database.parent / "artifacts") if self.store.central_database else None,
|
|
1419
|
+
)
|
|
1420
|
+
record_replacement_terminal(
|
|
1421
|
+
self.root, run_id=state.run_id, outcome="SUCCESS", result_evidence_ref=result_reference,
|
|
1422
|
+
central_database=self.store.central_database,
|
|
1423
|
+
)
|
|
1424
|
+
if state.provider_recovery_attempts and state.provider_recovery_attempts[0].get("result") in {"RECOVERY_AVAILABLE", "ACTIVE"}:
|
|
1425
|
+
prior = state.provider_recovery_attempts[0]
|
|
1426
|
+
recovered = self._recovery_record(
|
|
1427
|
+
state, original=prior["original_invocation_id"], replacement=replacement or "unavailable",
|
|
1428
|
+
eligibility="ELIGIBLE", result="RECOVERED", requested_at=prior["requested_at"],
|
|
1429
|
+
started_at=prior["started_at"], completed_at=datetime.now(timezone.utc).isoformat(),
|
|
1430
|
+
)
|
|
1431
|
+
self.store.save(recovered)
|
|
1432
|
+
complete_phase(self.root, provider)
|
|
1433
|
+
if parent:
|
|
1434
|
+
complete_phase(self.root, parent)
|
|
1435
|
+
return result
|
|
1436
|
+
|
|
1437
|
+
def _invoke_agent_with_timing(self, state: TransactionState, prompt: str, *, repair: bool = False, quality: bool = False, local_validation: bool = False, attempt: int | None = None) -> AgentResult:
|
|
1438
|
+
"""Consume the durable recovery controller around individual attempts.
|
|
1439
|
+
|
|
1440
|
+
The controller is the sole authority for replacement identity and
|
|
1441
|
+
launch. This deliberately uses an iterative control flow: a provider
|
|
1442
|
+
exception never recursively re-enters the lifecycle method.
|
|
1443
|
+
"""
|
|
1444
|
+
current_attempt = attempt
|
|
1445
|
+
while True:
|
|
1446
|
+
recovery = self._recovery_state(state.run_id)
|
|
1447
|
+
if isinstance(recovery, dict) and recovery.get("state") == "RECOVERY_AVAILABLE":
|
|
1448
|
+
precheck = self._provider_recovery_preflight(state)
|
|
1449
|
+
if precheck is not None:
|
|
1450
|
+
mark_precheck_failed(
|
|
1451
|
+
self.root, run_id=state.run_id, diagnostic_code=precheck,
|
|
1452
|
+
central_database=self.store.central_database,
|
|
1453
|
+
)
|
|
1454
|
+
self._project_durable_recovery(state, self._recovery_state(state.run_id) or recovery)
|
|
1455
|
+
raise CodexInvocationError(
|
|
1456
|
+
"Provider interruption recovery cannot continue.", "Recovery continuation preflight failed.",
|
|
1457
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1458
|
+
)
|
|
1459
|
+
if not transition_recovery_state(
|
|
1460
|
+
self.root, run_id=state.run_id, expected="RECOVERY_AVAILABLE", target="RECOVERY_STARTING",
|
|
1461
|
+
central_database=self.store.central_database,
|
|
1462
|
+
):
|
|
1463
|
+
continue
|
|
1464
|
+
if claim_replacement_launch(
|
|
1465
|
+
self.root, run_id=state.run_id, central_database=self.store.central_database,
|
|
1466
|
+
) is None:
|
|
1467
|
+
raise CodexInvocationError(
|
|
1468
|
+
"Provider interruption recovery launch is ambiguous.", "Replacement launch is already claimed.",
|
|
1469
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1470
|
+
)
|
|
1471
|
+
self.sleep(0.25)
|
|
1472
|
+
current_attempt = (current_attempt or max(1, state.repair_iterations + 1)) + 1
|
|
1473
|
+
# The next iteration dispatches the exact persisted
|
|
1474
|
+
# replacement ID; it does not create a fresh invocation.
|
|
1475
|
+
try:
|
|
1476
|
+
result = self._invoke_provider_attempt_with_timing(
|
|
1477
|
+
state, prompt, repair=repair, quality=quality,
|
|
1478
|
+
local_validation=local_validation, attempt=current_attempt,
|
|
1479
|
+
)
|
|
1480
|
+
except CodexInvocationError:
|
|
1481
|
+
completed_recovery = self._recovery_state(state.run_id)
|
|
1482
|
+
if isinstance(completed_recovery, dict):
|
|
1483
|
+
self._project_durable_recovery(state, completed_recovery)
|
|
1484
|
+
raise
|
|
1485
|
+
completed_recovery = self._recovery_state(state.run_id)
|
|
1486
|
+
if isinstance(completed_recovery, dict):
|
|
1487
|
+
self._project_durable_recovery(state, completed_recovery)
|
|
1488
|
+
return result
|
|
1489
|
+
if (
|
|
1490
|
+
isinstance(recovery, dict)
|
|
1491
|
+
and recovery.get("state") == "RECOVERED"
|
|
1492
|
+
and recovery.get("lifecycle_phase") == state.phase
|
|
1493
|
+
):
|
|
1494
|
+
replacement_id = recovery.get("replacement_invocation_id")
|
|
1495
|
+
if (
|
|
1496
|
+
recovery.get("lifecycle_phase") != state.phase
|
|
1497
|
+
or not isinstance(replacement_id, str)
|
|
1498
|
+
):
|
|
1499
|
+
raise CodexInvocationError(
|
|
1500
|
+
"Recovered provider result is invalid.", "Recovery phase or invocation lineage is inconsistent.",
|
|
1501
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1502
|
+
)
|
|
1503
|
+
payload = load_recovery_agent_result(
|
|
1504
|
+
self.root, str(recovery.get("result_evidence_ref") or ""),
|
|
1505
|
+
run_id=state.run_id, invocation_id=replacement_id,
|
|
1506
|
+
central_database=self.store.central_database,
|
|
1507
|
+
artifact_root=(self.store.central_database.parent / "artifacts") if self.store.central_database else None,
|
|
1508
|
+
)
|
|
1509
|
+
if payload is None:
|
|
1510
|
+
raise CodexInvocationError(
|
|
1511
|
+
"Recovered provider result is unavailable.", "Recovery result evidence failed integrity verification.",
|
|
1512
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1513
|
+
)
|
|
1514
|
+
try:
|
|
1515
|
+
return AgentResult(
|
|
1516
|
+
terminal_state=str(payload["terminal_state"]), branch=payload.get("branch"),
|
|
1517
|
+
pull_request=payload.get("pull_request"), terminal_condition=str(payload.get("terminal_condition") or "repository_reconciled"),
|
|
1518
|
+
diagnostic=payload.get("diagnostic"), repository_path=payload.get("repository_path"),
|
|
1519
|
+
commit_sha=payload.get("commit_sha"),
|
|
1520
|
+
validation_evidence=tuple(payload.get("validation_evidence") or ()),
|
|
1521
|
+
quality_evidence=tuple(payload.get("quality_evidence") or ()),
|
|
1522
|
+
validation_disposition=str(payload.get("validation_disposition") or "product_failure"),
|
|
1523
|
+
)
|
|
1524
|
+
except (KeyError, TypeError, ValueError) as error:
|
|
1525
|
+
raise CodexInvocationError(
|
|
1526
|
+
"Recovered provider result is invalid.", "Recovery result evidence cannot be consumed safely.",
|
|
1527
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1528
|
+
) from error
|
|
1529
|
+
if isinstance(recovery, dict) and recovery.get("state") in {"EXHAUSTED", "PRECHECK_FAILED", "AMBIGUOUS"}:
|
|
1530
|
+
raise CodexInvocationError(
|
|
1531
|
+
"Provider interruption recovery cannot continue.", "Recovery budget is exhausted or recovery evidence is unsafe.",
|
|
1532
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1533
|
+
)
|
|
1534
|
+
if isinstance(recovery, dict) and recovery.get("state") in {"RECOVERY_STARTING", "RECOVERY_IN_PROGRESS"}:
|
|
1535
|
+
reconciliation = reconcile_recovery(
|
|
1536
|
+
self.root, run_id=state.run_id, central_database=self.store.central_database,
|
|
1537
|
+
)
|
|
1538
|
+
if reconciliation == "LAUNCH_UNCLAIMED":
|
|
1539
|
+
claim = claim_replacement_launch(
|
|
1540
|
+
self.root, run_id=state.run_id, central_database=self.store.central_database,
|
|
1541
|
+
)
|
|
1542
|
+
if claim is None:
|
|
1543
|
+
raise CodexInvocationError(
|
|
1544
|
+
"Provider recovery launch is ambiguous.", "Replacement launch claim could not be acquired.",
|
|
1545
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1546
|
+
)
|
|
1547
|
+
current_attempt = (current_attempt or max(1, state.repair_iterations + 1)) + 1
|
|
1548
|
+
return self._invoke_provider_attempt_with_timing(
|
|
1549
|
+
state, prompt, repair=repair, quality=quality,
|
|
1550
|
+
local_validation=local_validation, attempt=current_attempt,
|
|
1551
|
+
)
|
|
1552
|
+
if reconciliation == "LAUNCH_CLAIMED_PREEXEC_FAILURE":
|
|
1553
|
+
current_attempt = (current_attempt or max(1, state.repair_iterations + 1)) + 1
|
|
1554
|
+
return self._invoke_provider_attempt_with_timing(
|
|
1555
|
+
state, prompt, repair=repair, quality=quality,
|
|
1556
|
+
local_validation=local_validation, attempt=current_attempt,
|
|
1557
|
+
)
|
|
1558
|
+
if reconciliation == "RECOVERED":
|
|
1559
|
+
continue
|
|
1560
|
+
if reconciliation == "SAME_PROVIDER_STILL_ACTIVE":
|
|
1561
|
+
# This is an active run, not a provider failure. The
|
|
1562
|
+
# caller returns the durable checkpoint without creating
|
|
1563
|
+
# a second provider attempt or terminalizing it.
|
|
1564
|
+
raise ProviderReadinessBlocked(state)
|
|
1565
|
+
# No restart path exists after a claimed/provider-started
|
|
1566
|
+
# attempt unless a terminal result can be proven.
|
|
1567
|
+
raise CodexInvocationError(
|
|
1568
|
+
"Provider recovery is awaiting process reconciliation.", f"Recovery reconciliation: {reconciliation}.",
|
|
1569
|
+
next_action="NONE", terminal_condition="provider_turn_interrupted",
|
|
1570
|
+
)
|
|
1571
|
+
try:
|
|
1572
|
+
result = self._invoke_provider_attempt_with_timing(
|
|
1573
|
+
state, prompt, repair=repair, quality=quality,
|
|
1574
|
+
local_validation=local_validation, attempt=current_attempt,
|
|
1575
|
+
)
|
|
1576
|
+
recovery = self._recovery_state(state.run_id)
|
|
1577
|
+
if isinstance(recovery, dict):
|
|
1578
|
+
self._project_durable_recovery(state, recovery)
|
|
1579
|
+
return result
|
|
1580
|
+
except CodexInvocationError as error:
|
|
1581
|
+
recovery = self._recovery_state(state.run_id)
|
|
1582
|
+
if isinstance(recovery, dict) and recovery.get("state") in {"EXHAUSTED", "PRECHECK_FAILED", "AMBIGUOUS"}:
|
|
1583
|
+
self._project_durable_recovery(state, recovery)
|
|
1584
|
+
if not error.provider_turn_interrupted or not isinstance(recovery, dict):
|
|
1585
|
+
raise
|
|
1586
|
+
if recovery.get("state") != "RECOVERY_AVAILABLE":
|
|
1587
|
+
raise
|
|
1588
|
+
# The top of the loop now claims the durable launch intent.
|
|
1589
|
+
# This handler never allocates or authorizes a replacement.
|
|
1590
|
+
continue
|
|
1591
|
+
|
|
1592
|
+
def _run_local_repository_validation(
|
|
1593
|
+
self, state: TransactionState, implementation: AgentResult
|
|
1594
|
+
) -> tuple[TransactionState, AgentResult]:
|
|
1595
|
+
"""Run the bounded, mutable local gate before an implementation PR exists."""
|
|
1596
|
+
if state.action_intent == "VALIDATION_ONLY":
|
|
1597
|
+
# This gate is a delivery-only boundary. A producer-authorized
|
|
1598
|
+
# qualification run may supply validation evidence but must never
|
|
1599
|
+
# be forced to invent a branch, commit, or pull request.
|
|
1600
|
+
return state, implementation
|
|
1601
|
+
branch = implementation.branch or state.branch
|
|
1602
|
+
# Legacy/resumed checkpoints can already carry their implementation PR.
|
|
1603
|
+
# Never rewrite that evidence; new managed prompts are instructed to
|
|
1604
|
+
# stop before PR creation and therefore enter this gate normally.
|
|
1605
|
+
if implementation.pull_request:
|
|
1606
|
+
return state, implementation
|
|
1607
|
+
if not branch:
|
|
1608
|
+
return self._save_terminal(
|
|
1609
|
+
state, "BLOCKED", "local_validation_scope", "Implementation must return one branch and no pull request before local validation."
|
|
1610
|
+
), implementation
|
|
1611
|
+
validation = replace(
|
|
1612
|
+
state, phase="LOCAL_REPOSITORY_VALIDATION", branch=branch, pull_request=None,
|
|
1613
|
+
next_action="run_local_repository_validation", local_validation_iterations=0,
|
|
1614
|
+
local_validation_audit=(),
|
|
1615
|
+
)
|
|
1616
|
+
for iteration in range(1, MAX_LOCAL_REPOSITORY_VALIDATION_ATTEMPTS + 1):
|
|
1617
|
+
try:
|
|
1618
|
+
profile = classify(changed_paths(self.root, "main"))
|
|
1619
|
+
except OSError:
|
|
1620
|
+
profile = classify(())
|
|
1621
|
+
try:
|
|
1622
|
+
record_validation_profile(
|
|
1623
|
+
self.root, run_id=validation.run_id, selected_validation_tier=profile.tier,
|
|
1624
|
+
validation_profile_version=VALIDATION_PROFILE_VERSION,
|
|
1625
|
+
required_validation_controls=profile.required_controls,
|
|
1626
|
+
profile_reference=f"validation-profile-registry:{profile.tier}@{VALIDATION_PROFILE_VERSION}",
|
|
1627
|
+
profile_selection_source="diff_classification",
|
|
1628
|
+
control_bindings=profile_control_bindings(profile),
|
|
1629
|
+
recorded_at=datetime.now(timezone.utc).isoformat(),
|
|
1630
|
+
central_database=self.store.central_database,
|
|
1631
|
+
)
|
|
1632
|
+
except (EngineeringStorageError, ValidationProfileResolutionError):
|
|
1633
|
+
return self._save_terminal(
|
|
1634
|
+
validation, "BLOCKED", "validation_profile_persistence",
|
|
1635
|
+
"Required validation profile evidence could not be persisted."
|
|
1636
|
+
), implementation
|
|
1637
|
+
validation = replace(validation, local_validation_iterations=iteration)
|
|
1638
|
+
self.store.save(validation)
|
|
1639
|
+
write_live_status(self.root, validation, validation.next_action)
|
|
1640
|
+
instruction = f"""
|
|
1641
|
+
|
|
1642
|
+
Local repository validation gate — iteration {iteration} of {MAX_LOCAL_REPOSITORY_VALIDATION_ATTEMPTS}:
|
|
1643
|
+
- Stay on exactly `{branch}`. Do not merge or change scope.
|
|
1644
|
+
- Diff-derived validation profile: `{profile.tier}`. Required evidence: {"; ".join(profile.commands)}. If the diff is unavailable or scope becomes mixed, use the full required suite.
|
|
1645
|
+
- You may correct only the bounded production code and its tests, commit and push those corrections, then rerun the required validation.
|
|
1646
|
+
- If validation still fails, return `WAITING` with a concise safe diagnostic; the host may allow the next bounded iteration.
|
|
1647
|
+
- Create one draft implementation pull request only after the required local validation passes. Return that same branch and PR number. Never poll remote checks.
|
|
1648
|
+
"""
|
|
1649
|
+
try:
|
|
1650
|
+
result = self._invoke_agent_with_timing(
|
|
1651
|
+
validation,
|
|
1652
|
+
assemble_prompt(Path(validation.prompt_path), validation, managed_target=self.root) + instruction,
|
|
1653
|
+
local_validation=True,
|
|
1654
|
+
attempt=iteration,
|
|
1655
|
+
)
|
|
1656
|
+
validation = self._record_agent_execution_time(validation)
|
|
1657
|
+
validation = self._record_validation_evidence(validation, result)
|
|
1658
|
+
validation = self._record_verified_result_commit(
|
|
1659
|
+
validation,
|
|
1660
|
+
result,
|
|
1661
|
+
phase="LOCAL_REPOSITORY_VALIDATION",
|
|
1662
|
+
description="local_repository_validation_commit_verified",
|
|
1663
|
+
)
|
|
1664
|
+
self._persist_agent_usage(validation.run_id)
|
|
1665
|
+
except ProviderReadinessBlocked as blocked:
|
|
1666
|
+
return blocked.state, implementation
|
|
1667
|
+
except CodexInvocationError as error:
|
|
1668
|
+
validation = self._record_agent_execution_time(validation)
|
|
1669
|
+
self.console_detail = error.console_detail
|
|
1670
|
+
validation = self._record_local_validation_audit(validation, result=None, outcome="agent_failed", profile=profile)
|
|
1671
|
+
return self._terminalize_provider_invocation_error(validation, error), implementation
|
|
1672
|
+
if result.terminal_state in {"BLOCKED", "FAILED"}:
|
|
1673
|
+
if (
|
|
1674
|
+
result.terminal_state == "FAILED"
|
|
1675
|
+
and not self._is_external_agent_block(result)
|
|
1676
|
+
and self._has_failed_validation_evidence(result)
|
|
1677
|
+
):
|
|
1678
|
+
validation = self._record_local_validation_audit(
|
|
1679
|
+
validation, result=result, outcome="validation_failed", profile=profile
|
|
1680
|
+
)
|
|
1681
|
+
if self._is_environmental_validation_instability(result):
|
|
1682
|
+
return self._save_terminal(
|
|
1683
|
+
validation,
|
|
1684
|
+
"BLOCKED",
|
|
1685
|
+
"validation_infrastructure_recovery_required",
|
|
1686
|
+
"Required local validation is unstable: a failed required suite and a passing isolated rerun were recorded without an implementation correction. Preserve this run and create a separate validation-infrastructure recovery item.",
|
|
1687
|
+
), implementation
|
|
1688
|
+
continue
|
|
1689
|
+
validation = self._record_local_validation_audit(validation, result=result, outcome="agent_failed", profile=profile)
|
|
1690
|
+
return self._save_terminal(validation, result.terminal_state, "local_repository_validation_failed", result.diagnostic or "Local repository validation failed."), implementation
|
|
1691
|
+
if result.branch and result.branch != branch:
|
|
1692
|
+
validation = self._record_local_validation_audit(validation, result=result, outcome="agent_failed", profile=profile)
|
|
1693
|
+
return self._save_terminal(validation, "BLOCKED", "local_validation_scope", "Local validation changed the bounded implementation branch."), implementation
|
|
1694
|
+
if result.pull_request:
|
|
1695
|
+
validation = self._record_local_validation_audit(validation, result=result, outcome="validated", profile=profile)
|
|
1696
|
+
return validation, replace(
|
|
1697
|
+
result,
|
|
1698
|
+
branch=branch,
|
|
1699
|
+
validation_evidence=implementation.validation_evidence + result.validation_evidence,
|
|
1700
|
+
)
|
|
1701
|
+
validation = self._record_local_validation_audit(validation, result=result, outcome="validation_failed", profile=profile)
|
|
1702
|
+
if self._is_environmental_validation_instability(result):
|
|
1703
|
+
return self._save_terminal(
|
|
1704
|
+
validation,
|
|
1705
|
+
"BLOCKED",
|
|
1706
|
+
"validation_infrastructure_recovery_required",
|
|
1707
|
+
"Required local validation is unstable: a failed required suite and a passing isolated rerun were recorded without an implementation correction. Preserve this run and create a separate validation-infrastructure recovery item.",
|
|
1708
|
+
), implementation
|
|
1709
|
+
return self._save_terminal(validation, "BLOCKED", "local_validation_attempt_limit_reached", "Required local repository validation did not pass after 3 bounded iterations."), implementation
|
|
1710
|
+
|
|
1711
|
+
def _run_autonomous_quality_control(
|
|
1712
|
+
self, state: TransactionState, implementation: AgentResult
|
|
1713
|
+
) -> tuple[TransactionState, AgentResult]:
|
|
1714
|
+
"""Run the required post-implementation refactor and quality boundary.
|
|
1715
|
+
|
|
1716
|
+
The controller is autonomous but cannot widen delivery scope: it may
|
|
1717
|
+
amend only the current transaction branch and its existing PR.
|
|
1718
|
+
"""
|
|
1719
|
+
quality = replace(
|
|
1720
|
+
state,
|
|
1721
|
+
phase="QUALITY_CONTROL_AGENT",
|
|
1722
|
+
branch=implementation.branch or state.branch,
|
|
1723
|
+
pull_request=implementation.pull_request or state.pull_request,
|
|
1724
|
+
next_action="autonomous_refactor_and_quality_control",
|
|
1725
|
+
)
|
|
1726
|
+
self.store.save(quality)
|
|
1727
|
+
write_live_status(self.root, quality, quality.next_action)
|
|
1728
|
+
prompt = assemble_prompt(
|
|
1729
|
+
Path(quality.prompt_path), quality,
|
|
1730
|
+
managed_target=self.root if quality.execution_mode == "MANAGED" else None,
|
|
1731
|
+
) + """
|
|
1732
|
+
|
|
1733
|
+
Mandatory autonomous refactor and quality-control stage:
|
|
1734
|
+
- Inspect the implementation now present on this transaction branch.
|
|
1735
|
+
- Autonomously make only demonstrable maintainability, clarity, safety, or
|
|
1736
|
+
test-coverage improvements within the original bounded objective.
|
|
1737
|
+
- Assess test coverage for every changed behavior. Add or strengthen focused
|
|
1738
|
+
regression tests whenever existing coverage does not prove that behavior.
|
|
1739
|
+
- Assess the applicable operator, contract, and implementation documentation.
|
|
1740
|
+
Update it whenever the bounded change affects documented behavior; only
|
|
1741
|
+
leave documentation unchanged when the inspection proves it is unaffected.
|
|
1742
|
+
- Run the relevant focused validation, including the added or affected tests,
|
|
1743
|
+
and `git diff --check`.
|
|
1744
|
+
- Preserve the existing transaction branch and pull request. If changes are
|
|
1745
|
+
needed, commit and push them to that same branch; do not create another PR,
|
|
1746
|
+
merge, alter authority, or expand scope.
|
|
1747
|
+
- Return the same pull-request number and branch after the quality boundary.
|
|
1748
|
+
- In quality_evidence, record only work actually performed in this stage. Use
|
|
1749
|
+
activity values REFACTOR, TEST_COVERAGE, DOCUMENTATION, VALIDATION, or
|
|
1750
|
+
NO_CHANGE_REQUIRED and a short safe result for each. Do not include raw
|
|
1751
|
+
commands, output, prompts, source content, paths, secrets, or reasoning.
|
|
1752
|
+
"""
|
|
1753
|
+
try:
|
|
1754
|
+
result = self._invoke_agent_with_timing(quality, prompt, quality=True)
|
|
1755
|
+
quality = self._record_agent_execution_time(quality)
|
|
1756
|
+
quality = self._record_validation_evidence(quality, result)
|
|
1757
|
+
quality = replace(quality, quality_evidence=result.quality_evidence)
|
|
1758
|
+
quality = self._record_verified_result_commit(
|
|
1759
|
+
quality,
|
|
1760
|
+
result,
|
|
1761
|
+
phase="QUALITY_CONTROL_AGENT",
|
|
1762
|
+
description="quality_control_commit_verified",
|
|
1763
|
+
)
|
|
1764
|
+
self._persist_agent_usage(quality.run_id)
|
|
1765
|
+
except ProviderReadinessBlocked as blocked:
|
|
1766
|
+
return blocked.state, implementation
|
|
1767
|
+
except CodexInvocationError as error:
|
|
1768
|
+
quality = self._record_agent_execution_time(quality)
|
|
1769
|
+
self.console_detail = error.console_detail
|
|
1770
|
+
return self._terminalize_provider_invocation_error(quality, error), implementation
|
|
1771
|
+
return self._advance_after_quality_control_agent_result(quality, implementation, result)
|
|
1772
|
+
|
|
1773
|
+
def _advance_after_quality_control_agent_result(
|
|
1774
|
+
self, quality: TransactionState, implementation: AgentResult, result: AgentResult,
|
|
1775
|
+
) -> tuple[TransactionState, AgentResult]:
|
|
1776
|
+
"""Apply live or recovered QC success without provider-session state."""
|
|
1777
|
+
if result.terminal_state in {"BLOCKED", "FAILED"}:
|
|
1778
|
+
return self._save_terminal(quality, result.terminal_state, "autonomous_quality_control_failed", result.diagnostic or "Autonomous quality control did not complete."), implementation
|
|
1779
|
+
if implementation.pull_request and result.pull_request and result.pull_request != implementation.pull_request:
|
|
1780
|
+
return self._save_terminal(quality, "BLOCKED", "autonomous_quality_control_scope", "Autonomous quality control returned a different pull request."), implementation
|
|
1781
|
+
if implementation.branch and result.branch and result.branch != implementation.branch:
|
|
1782
|
+
return self._save_terminal(quality, "BLOCKED", "autonomous_quality_control_scope", "Autonomous quality control returned a different branch."), implementation
|
|
1783
|
+
return quality, replace(
|
|
1784
|
+
implementation,
|
|
1785
|
+
branch=result.branch or implementation.branch,
|
|
1786
|
+
pull_request=result.pull_request or implementation.pull_request,
|
|
1787
|
+
validation_evidence=implementation.validation_evidence + result.validation_evidence,
|
|
1788
|
+
)
|
|
1789
|
+
|
|
1790
|
+
def _reject_historical_agent_pull_request(
|
|
1791
|
+
self, state: TransactionState
|
|
1792
|
+
) -> TransactionState | None:
|
|
1793
|
+
"""Keep a newly invoked agent from reusing a merged PR as its evidence.
|
|
1794
|
+
|
|
1795
|
+
A run has no transaction evidence until its first agent invocation has
|
|
1796
|
+
returned. A merged PR at that point belongs to earlier work and must
|
|
1797
|
+
not be marked ready or silently adopted into this new transaction.
|
|
1798
|
+
"""
|
|
1799
|
+
if not state.pull_request:
|
|
1800
|
+
return None
|
|
1801
|
+
try:
|
|
1802
|
+
pull_request = self.github.pull_request(state.pull_request)
|
|
1803
|
+
except RunnerError:
|
|
1804
|
+
# _poll owns bounded retry behaviour for transient GitHub reads.
|
|
1805
|
+
return None
|
|
1806
|
+
if pull_request.state != "MERGED":
|
|
1807
|
+
return None
|
|
1808
|
+
try:
|
|
1809
|
+
objective = Path(state.prompt_path).read_text(encoding="utf-8")
|
|
1810
|
+
except OSError:
|
|
1811
|
+
objective = ""
|
|
1812
|
+
has_retry_lineage = bool(re.search(r"(?mi)^Retry-Of:\s*[-a-z0-9]+\s*$", objective))
|
|
1813
|
+
is_reconcilable_lineage_merge = (
|
|
1814
|
+
has_retry_lineage
|
|
1815
|
+
and state.branch is not None
|
|
1816
|
+
and state.branch == pull_request.head_branch
|
|
1817
|
+
and pull_request.base_branch == "main"
|
|
1818
|
+
and pull_request.merge_commit is not None
|
|
1819
|
+
and self.repository.main_contains(self.root, pull_request.merge_commit)
|
|
1820
|
+
)
|
|
1821
|
+
if is_reconcilable_lineage_merge:
|
|
1822
|
+
evidence = self.repository.inspect(self.root)
|
|
1823
|
+
reconciled = self._record_merged_evidence(state, pull_request, evidence)
|
|
1824
|
+
if reconciled.owner_authorized and reconciled.transaction_kind == "IMPLEMENTATION":
|
|
1825
|
+
return self._start_finalization(reconciled, pull_request.number)
|
|
1826
|
+
return self._cleanup(reconciled)
|
|
1827
|
+
return self._save_terminal(
|
|
1828
|
+
state,
|
|
1829
|
+
"BLOCKED",
|
|
1830
|
+
"historical_pull_request_evidence",
|
|
1831
|
+
f"Agent result referenced already merged PR #{pull_request.number}; a new transaction must return its own open pull request.",
|
|
1832
|
+
)
|
|
1833
|
+
|
|
1834
|
+
def _advance_after_primary_agent_result(
|
|
1835
|
+
self, state: TransactionState, result: AgentResult, evidence: RepositoryEvidence,
|
|
1836
|
+
) -> TransactionState:
|
|
1837
|
+
"""Shared post-provider transition for live and recovered results."""
|
|
1838
|
+
if state.execution_mode == "GENESIS":
|
|
1839
|
+
return self._reconcile_genesis_result(state, result)
|
|
1840
|
+
recoverable_local_failure = self._is_recoverable_implementation_validation_failure(state, result)
|
|
1841
|
+
if state.transaction_kind == "IMPLEMENTATION" and state.action_intent == "MUTATING_DELIVERY" and (
|
|
1842
|
+
result.terminal_state not in {"BLOCKED", "FAILED"} or recoverable_local_failure
|
|
1843
|
+
):
|
|
1844
|
+
if state.owner_authorized:
|
|
1845
|
+
state, result = self._run_local_repository_validation(state, result)
|
|
1846
|
+
if state.terminal:
|
|
1847
|
+
return state
|
|
1848
|
+
state, result = self._run_autonomous_quality_control(state, result)
|
|
1849
|
+
if state.terminal:
|
|
1850
|
+
return state
|
|
1851
|
+
return self._continue_after_quality_control(state, result, evidence)
|
|
1852
|
+
|
|
1853
|
+
def _continue_after_quality_control(
|
|
1854
|
+
self, state: TransactionState, result: AgentResult, evidence: RepositoryEvidence,
|
|
1855
|
+
) -> TransactionState:
|
|
1856
|
+
"""Advance delivery only after QC is already complete."""
|
|
1857
|
+
state = replace(
|
|
1858
|
+
state, phase="WAIT_FOR_TERMINAL_EVIDENCE", branch=result.branch or evidence.branch,
|
|
1859
|
+
pull_request=result.pull_request, next_action="poll_required_checks",
|
|
1860
|
+
terminal_condition=result.terminal_condition,
|
|
1861
|
+
finalization_branch=(result.branch or evidence.branch)
|
|
1862
|
+
if state.transaction_kind == "FINALIZATION" else state.finalization_branch,
|
|
1863
|
+
finalization_pull_request=result.pull_request
|
|
1864
|
+
if state.transaction_kind == "FINALIZATION" else state.finalization_pull_request,
|
|
1865
|
+
reconciliation_pull_request=result.pull_request
|
|
1866
|
+
if state.transaction_kind == "RECONCILIATION" else state.reconciliation_pull_request,
|
|
1867
|
+
)
|
|
1868
|
+
self.store.save(state)
|
|
1869
|
+
write_live_status(self.root, state, state.next_action)
|
|
1870
|
+
if state.owner_authorized and state.pull_request:
|
|
1871
|
+
historical = self._reject_historical_agent_pull_request(state)
|
|
1872
|
+
if historical is not None:
|
|
1873
|
+
return historical
|
|
1874
|
+
self.github.normalize_markdown_body(state.pull_request)
|
|
1875
|
+
self.github.ready(state.pull_request)
|
|
1876
|
+
return self._poll(state, result)
|
|
1877
|
+
|
|
1878
|
+
def _advance_after_recovered_provider_result(
|
|
1879
|
+
self, state: TransactionState, result: AgentResult, evidence: RepositoryEvidence,
|
|
1880
|
+
lifecycle_phase: str,
|
|
1881
|
+
) -> TransactionState:
|
|
1882
|
+
"""Route a validated durable result to its existing phase handler."""
|
|
1883
|
+
if lifecycle_phase == "EXECUTE_AGENT":
|
|
1884
|
+
return self._advance_after_primary_agent_result(state, result, evidence)
|
|
1885
|
+
if lifecycle_phase == "QUALITY_CONTROL_AGENT":
|
|
1886
|
+
implementation = AgentResult("COMPLETE", branch=state.branch, pull_request=state.pull_request)
|
|
1887
|
+
quality, implementation = self._advance_after_quality_control_agent_result(state, implementation, result)
|
|
1888
|
+
return quality if quality.terminal else self._continue_after_quality_control(quality, implementation, evidence)
|
|
1889
|
+
if lifecycle_phase == "REPAIR_AGENT":
|
|
1890
|
+
return self._advance_after_repair_agent_result(state, result)
|
|
1891
|
+
if lifecycle_phase == "FINALIZE_AGENT":
|
|
1892
|
+
return self._advance_after_finalization_agent_result(state, result)
|
|
1893
|
+
return self._save_terminal(state, "BLOCKED", "recovered_provider_phase_invalid", "Recovered provider result has an unsupported lifecycle phase.")
|
|
1894
|
+
|
|
1895
|
+
def run(
|
|
1896
|
+
self,
|
|
1897
|
+
prompt_path: Path,
|
|
1898
|
+
run_id: str | None = None,
|
|
1899
|
+
resume: bool = False,
|
|
1900
|
+
owner_authorized: bool = False,
|
|
1901
|
+
transaction_kind: str = "IMPLEMENTATION",
|
|
1902
|
+
) -> TransactionState:
|
|
1903
|
+
# Private helpers remain directly testable, while every public runner
|
|
1904
|
+
# invocation enforces the admission boundary before provider dispatch.
|
|
1905
|
+
self._dispatch_guard_enforced = True
|
|
1906
|
+
objective = prompt_path.read_text(encoding="utf-8")
|
|
1907
|
+
state = self.store.load(run_id) if resume else None
|
|
1908
|
+
if resume and state is not None and dismissal_for_run(self.root, state.run_id):
|
|
1909
|
+
raise RunnerError("This execution has already been dismissed and cannot be resumed.")
|
|
1910
|
+
if (
|
|
1911
|
+
state is not None
|
|
1912
|
+
and state.phase in {"WAIT_FOR_TERMINAL_EVIDENCE", "WAIT_FOR_OPERATOR_MERGE"}
|
|
1913
|
+
and state.pull_request is not None
|
|
1914
|
+
):
|
|
1915
|
+
# A pull-request wait is deliberately passive. Resuming it must
|
|
1916
|
+
# therefore only re-read the remote pull-request state. Re-running workspace admission, repository
|
|
1917
|
+
# synchronization, reviewer selection and memory retrieval here
|
|
1918
|
+
# creates expensive local churn while there is no new work to do.
|
|
1919
|
+
# Once a merge is observed, _poll performs the required
|
|
1920
|
+
# repository reconciliation before cleanup or Finalization.
|
|
1921
|
+
if Path(state.prompt_path) != prompt_path:
|
|
1922
|
+
raise RunnerError("checkpoint conflicts with current prompt")
|
|
1923
|
+
state = self._provider_readiness_gate(
|
|
1924
|
+
state, require_codex=False, require_github=True
|
|
1925
|
+
)
|
|
1926
|
+
if state.next_action == "provider_auth_repair_required":
|
|
1927
|
+
return state
|
|
1928
|
+
self._verify_engineering_platform()
|
|
1929
|
+
# Fresh PR evidence can transition this wait into a bounded
|
|
1930
|
+
# same-PR repair. Reacquire the lease before polling, so that a
|
|
1931
|
+
# running repair has one durable owner and the dashboard reflects
|
|
1932
|
+
# it instead of projecting a false idle state.
|
|
1933
|
+
self.transaction = ExecutionTransaction(state=state, target_repository=self.root)
|
|
1934
|
+
try:
|
|
1935
|
+
self.active_lease = acquire_lease(
|
|
1936
|
+
self.root, state.run_id, identity=self.host_identity,
|
|
1937
|
+
instance_id=self.host_instance_id, process_id=os.getpid(), central_database=self.store.central_database,
|
|
1938
|
+
)
|
|
1939
|
+
except LeaseConflictError as error:
|
|
1940
|
+
raise RunnerError("active-run ownership conflict; execution is refused") from error
|
|
1941
|
+
self.lease_heartbeat = LeaseHeartbeat(self.root, self.active_lease, central_database=self.store.central_database)
|
|
1942
|
+
self.transaction = self.transaction.with_lease(self.active_lease)
|
|
1943
|
+
self.lease_heartbeat.start()
|
|
1944
|
+
write_live_status(self.root, state, state.next_action)
|
|
1945
|
+
try:
|
|
1946
|
+
return self._poll(state)
|
|
1947
|
+
finally:
|
|
1948
|
+
# Terminal and operator-wait saves release their own lease.
|
|
1949
|
+
# A direct passive return must also leave no synthetic owner.
|
|
1950
|
+
if self.active_lease is not None and self.active_lease.run_id == state.run_id:
|
|
1951
|
+
if self.lease_heartbeat is not None:
|
|
1952
|
+
self.active_lease = self.lease_heartbeat.stop()
|
|
1953
|
+
self.lease_heartbeat = None
|
|
1954
|
+
release_lease(self.root, self.active_lease, central_database=self.store.central_database)
|
|
1955
|
+
self.active_lease = None
|
|
1956
|
+
try:
|
|
1957
|
+
# Storage compatibility is verified below before reading the
|
|
1958
|
+
# producer-owned run snapshot. Keep this preflight parser free
|
|
1959
|
+
# of datastore access.
|
|
1960
|
+
context = resolve_execution_context(objective, self.root)
|
|
1961
|
+
except RunnerError as error:
|
|
1962
|
+
evidence = self.repository.inspect(self.root)
|
|
1963
|
+
state = state or TransactionState(
|
|
1964
|
+
run_id or f"run-{uuid.uuid4().hex[:12]}",
|
|
1965
|
+
evidence.repository,
|
|
1966
|
+
str(prompt_path),
|
|
1967
|
+
"INITIALIZE",
|
|
1968
|
+
owner_authorized=owner_authorized,
|
|
1969
|
+
execution_mode="GENESIS"
|
|
1970
|
+
if any(line.strip().casefold() == "execution mode: genesis" for line in objective.splitlines())
|
|
1971
|
+
else "MANAGED",
|
|
1972
|
+
action_intent="MUTATING_DELIVERY",
|
|
1973
|
+
)
|
|
1974
|
+
return self._save_terminal(state, "BLOCKED", "execution_context_resolution", str(error))
|
|
1975
|
+
evidence = self.repository.inspect(self.root)
|
|
1976
|
+
if state is not None:
|
|
1977
|
+
if state.repository != evidence.repository or Path(state.prompt_path) != prompt_path:
|
|
1978
|
+
raise RunnerError("checkpoint conflicts with current repository or prompt")
|
|
1979
|
+
if state.execution_mode != context.execution_mode:
|
|
1980
|
+
raise RunnerError("checkpoint execution mode conflicts with the prompt")
|
|
1981
|
+
if (
|
|
1982
|
+
context.target_repository
|
|
1983
|
+
and state.genesis_repository_path
|
|
1984
|
+
and Path(state.genesis_repository_path) != context.target_repository
|
|
1985
|
+
):
|
|
1986
|
+
raise RunnerError("checkpoint Genesis target conflicts with the prompt")
|
|
1987
|
+
if state.terminal:
|
|
1988
|
+
return state
|
|
1989
|
+
else:
|
|
1990
|
+
state = TransactionState(
|
|
1991
|
+
run_id or f"run-{uuid.uuid4().hex[:12]}",
|
|
1992
|
+
evidence.repository,
|
|
1993
|
+
str(prompt_path),
|
|
1994
|
+
"INITIALIZE",
|
|
1995
|
+
owner_authorized=owner_authorized,
|
|
1996
|
+
transaction_kind=transaction_kind,
|
|
1997
|
+
execution_mode=context.execution_mode,
|
|
1998
|
+
action_intent=context.action_intent,
|
|
1999
|
+
)
|
|
2000
|
+
context = replace(context, run_id=state.run_id)
|
|
2001
|
+
passive_pr_wait = state.pull_request is not None and state.phase in {
|
|
2002
|
+
"WAIT_FOR_TERMINAL_EVIDENCE", "WAIT_FOR_OPERATOR_MERGE"
|
|
2003
|
+
}
|
|
2004
|
+
state = self._provider_readiness_gate(
|
|
2005
|
+
state,
|
|
2006
|
+
require_codex=not passive_pr_wait,
|
|
2007
|
+
require_github=context.execution_mode == "MANAGED",
|
|
2008
|
+
)
|
|
2009
|
+
if state.next_action == "provider_auth_repair_required":
|
|
2010
|
+
return state
|
|
2011
|
+
self.transaction = ExecutionTransaction(
|
|
2012
|
+
state=state,
|
|
2013
|
+
target_repository=context.target_repository or self.root,
|
|
2014
|
+
)
|
|
2015
|
+
# The Inbox watcher admits a run with the schema it has loaded. A
|
|
2016
|
+
# freshly spawned runner reads source files again, so it can otherwise
|
|
2017
|
+
# observe a newer manifest and migrate the database while the watcher
|
|
2018
|
+
# still runs the older code. Verify that compatibility boundary before
|
|
2019
|
+
# StateStore.save() opens (and could migrate) the datastore.
|
|
2020
|
+
if not passive_pr_wait and not self.agent.available():
|
|
2021
|
+
raise RunnerError("Codex CLI is not installed or invokable")
|
|
2022
|
+
self._verify_engineering_platform()
|
|
2023
|
+
recovery_snapshot = self._recovery_state(state.run_id)
|
|
2024
|
+
recovered_resume = (
|
|
2025
|
+
isinstance(recovery_snapshot, dict)
|
|
2026
|
+
and recovery_snapshot.get("state") == "RECOVERED"
|
|
2027
|
+
and recovery_snapshot.get("lifecycle_phase") in {"EXECUTE_AGENT", "QUALITY_CONTROL_AGENT", "REPAIR_AGENT", "FINALIZE_AGENT"}
|
|
2028
|
+
)
|
|
2029
|
+
try:
|
|
2030
|
+
persisted_submission = load_submission_for_run(self.root, state.run_id, central_database=self.store.central_database)
|
|
2031
|
+
except EngineeringStorageError:
|
|
2032
|
+
persisted_submission = None
|
|
2033
|
+
producer_context = persisted_submission.get("execution_context") if isinstance(persisted_submission, dict) else None
|
|
2034
|
+
action_intent = producer_context.get("action_intent", "MUTATING_DELIVERY") if isinstance(producer_context, dict) else "MUTATING_DELIVERY"
|
|
2035
|
+
context = resolve_execution_context(objective, self.root, action_intent=action_intent)
|
|
2036
|
+
if state.action_intent != context.action_intent:
|
|
2037
|
+
if state.phase != "INITIALIZE":
|
|
2038
|
+
raise RunnerError("checkpoint action intent conflicts with the producer execution context")
|
|
2039
|
+
state = replace(state, action_intent=context.action_intent)
|
|
2040
|
+
# Establish canonical transaction identity before persisting readiness evidence.
|
|
2041
|
+
self.store.save(state)
|
|
2042
|
+
# This envelope is deliberately persisted once and can be resumed
|
|
2043
|
+
# after process restart. It is excluded from bottleneck ranking.
|
|
2044
|
+
self._total_phase = self._resume_phase(
|
|
2045
|
+
state.run_id, "TOTAL_EXECUTION", category="EXECUTION"
|
|
2046
|
+
)
|
|
2047
|
+
initialization = self._start_phase(state.run_id, "INITIALIZATION")
|
|
2048
|
+
readiness = evaluate_readiness(
|
|
2049
|
+
selected_profile(context.execution_mode),
|
|
2050
|
+
host_root=self.root,
|
|
2051
|
+
target_root=context.target_repository,
|
|
2052
|
+
managed_clean=lambda candidate: self.repository.inspect(candidate).clean,
|
|
2053
|
+
genesis_preflight=genesis_workspace_preflight,
|
|
2054
|
+
)
|
|
2055
|
+
observed_host = latest_host_preflight(self.root)
|
|
2056
|
+
observed_workspace = latest_workspace_preflight(self.root)
|
|
2057
|
+
observed_capability = latest_capability_preflight(self.root)
|
|
2058
|
+
preflight_facts = ReadinessFacts.from_preflight(
|
|
2059
|
+
host=observed_host,
|
|
2060
|
+
workspace=observed_workspace,
|
|
2061
|
+
capability=observed_capability,
|
|
2062
|
+
lease_available=True,
|
|
2063
|
+
)
|
|
2064
|
+
# Direct runner callers predate admission preflights. Preserve that
|
|
2065
|
+
# public compatibility path while the watcher supplies the complete
|
|
2066
|
+
# observed preflight facts for normal Inbox execution.
|
|
2067
|
+
facts = replace(
|
|
2068
|
+
preflight_facts,
|
|
2069
|
+
host_ready=preflight_facts.host_ready or not observed_host,
|
|
2070
|
+
repository_present=preflight_facts.repository_present or context.target_repository is not None or self.root.is_dir(),
|
|
2071
|
+
repository_clean=preflight_facts.repository_clean if preflight_facts.repository_clean is not None else (evidence.clean if context.execution_mode == "MANAGED" else True),
|
|
2072
|
+
remote_present=preflight_facts.remote_present if preflight_facts.remote_present is not None else True,
|
|
2073
|
+
upstream_present=preflight_facts.upstream_present if preflight_facts.upstream_present is not None else True,
|
|
2074
|
+
branch_present=preflight_facts.branch_present if preflight_facts.branch_present is not None else True,
|
|
2075
|
+
workspace_authorized=preflight_facts.workspace_authorized if preflight_facts.workspace_authorized is not None else True,
|
|
2076
|
+
capabilities_available=preflight_facts.capabilities_available if preflight_facts.capabilities_available is not None else True,
|
|
2077
|
+
providers_available=preflight_facts.providers_available if preflight_facts.providers_available is not None else True,
|
|
2078
|
+
datastore_healthy=preflight_facts.datastore_healthy if preflight_facts.datastore_healthy is not None else True,
|
|
2079
|
+
producer_contract_valid=preflight_facts.producer_contract_valid if preflight_facts.producer_contract_valid is not None else True,
|
|
2080
|
+
)
|
|
2081
|
+
decision = decide_readiness(readiness.profile, facts)
|
|
2082
|
+
record_readiness_evaluation(
|
|
2083
|
+
self.root, run_id=state.run_id, profile_id=decision.profile_id, profile_version=decision.profile_version,
|
|
2084
|
+
execution_mode=context.execution_mode, passed=readiness.ready and decision.passed,
|
|
2085
|
+
failed_requirements=decision.failed_requirements,
|
|
2086
|
+
facts=vars(decision.facts),
|
|
2087
|
+
evaluated_at=decision.evaluated_at, diagnostic=readiness.diagnostic or decision.diagnostic,
|
|
2088
|
+
central_database=self.store.central_database,
|
|
2089
|
+
)
|
|
2090
|
+
complete_phase(self.root, initialization, outcome="COMPLETE" if readiness.ready else "FAILED")
|
|
2091
|
+
if not readiness.ready:
|
|
2092
|
+
if context.execution_mode == "GENESIS":
|
|
2093
|
+
return self._save_terminal(state, "BLOCKED", "genesis_workspace_preflight", readiness.diagnostic)
|
|
2094
|
+
raise RunnerError(readiness.diagnostic or "Execution readiness failed")
|
|
2095
|
+
if context.execution_mode == "GENESIS":
|
|
2096
|
+
state = replace(state, genesis_repository_path=str(context.target_repository))
|
|
2097
|
+
authorization_blocker = target_repository_authorization(self.root, context.target_repository)
|
|
2098
|
+
if authorization_blocker:
|
|
2099
|
+
return self._save_terminal(
|
|
2100
|
+
state,
|
|
2101
|
+
"BLOCKED",
|
|
2102
|
+
"genesis_repository_scope",
|
|
2103
|
+
authorization_blocker,
|
|
2104
|
+
)
|
|
2105
|
+
owner = self._active_genesis_transaction(context.target_repository, state.run_id)
|
|
2106
|
+
if owner:
|
|
2107
|
+
return self._save_terminal(
|
|
2108
|
+
state,
|
|
2109
|
+
"BLOCKED",
|
|
2110
|
+
"genesis_workspace_conflict",
|
|
2111
|
+
f"Genesis preflight blocked: target workspace is owned by active run {owner}.",
|
|
2112
|
+
)
|
|
2113
|
+
reconciliation = self._start_phase(state.run_id, "RECONCILIATION")
|
|
2114
|
+
try:
|
|
2115
|
+
reconcile_stale(self.root, central_database=self.store.central_database)
|
|
2116
|
+
except Exception:
|
|
2117
|
+
complete_phase(self.root, reconciliation, outcome="FAILED")
|
|
2118
|
+
raise
|
|
2119
|
+
complete_phase(self.root, reconciliation)
|
|
2120
|
+
self.store.save(state)
|
|
2121
|
+
try:
|
|
2122
|
+
self.active_lease = acquire_lease(self.root, state.run_id, identity=self.host_identity, instance_id=self.host_instance_id, process_id=os.getpid(), central_database=self.store.central_database)
|
|
2123
|
+
except LeaseConflictError as error:
|
|
2124
|
+
blocked = decide_readiness(
|
|
2125
|
+
readiness.profile,
|
|
2126
|
+
replace(facts, lease_available=False),
|
|
2127
|
+
)
|
|
2128
|
+
record_readiness_evaluation(
|
|
2129
|
+
self.root, run_id=state.run_id, profile_id=blocked.profile_id, profile_version=blocked.profile_version,
|
|
2130
|
+
execution_mode=context.execution_mode, passed=False, failed_requirements=blocked.failed_requirements,
|
|
2131
|
+
facts=vars(blocked.facts),
|
|
2132
|
+
evaluated_at=blocked.evaluated_at, diagnostic=blocked.diagnostic,
|
|
2133
|
+
central_database=self.store.central_database,
|
|
2134
|
+
)
|
|
2135
|
+
raise RunnerError("active-run ownership conflict; execution is refused") from error
|
|
2136
|
+
self.lease_heartbeat = LeaseHeartbeat(self.root, self.active_lease, central_database=self.store.central_database)
|
|
2137
|
+
self.transaction = self.transaction.with_lease(self.active_lease)
|
|
2138
|
+
self.lease_heartbeat.start()
|
|
2139
|
+
if recovered_resume:
|
|
2140
|
+
# Consume immutable attempt-two evidence before any fresh provider
|
|
2141
|
+
# preparation or repository synchronization. This preserves the
|
|
2142
|
+
# original branch/worktree and cannot allocate a new invocation.
|
|
2143
|
+
result = self._invoke_agent_with_timing(state, "")
|
|
2144
|
+
state = self._record_agent_execution_time(state)
|
|
2145
|
+
state = self._record_validation_evidence(state, result)
|
|
2146
|
+
state = self._record_verified_result_commit(
|
|
2147
|
+
state, result, phase="EXECUTE_AGENT", description="implementation_agent_commit_verified",
|
|
2148
|
+
)
|
|
2149
|
+
return self._advance_after_recovered_provider_result(
|
|
2150
|
+
state, result, evidence, str(recovery_snapshot["lifecycle_phase"]),
|
|
2151
|
+
)
|
|
2152
|
+
# Synchronization is a host-owned admission step. Do it while this
|
|
2153
|
+
# run owns the lease so agents never race each other for index.lock,
|
|
2154
|
+
# and so the bounded retry policy in the repository client is used.
|
|
2155
|
+
if context.execution_mode == "MANAGED":
|
|
2156
|
+
try:
|
|
2157
|
+
self.repository.synchronize_main(self.root)
|
|
2158
|
+
evidence = self.repository.inspect(self.root)
|
|
2159
|
+
except RunnerError as error:
|
|
2160
|
+
return self._save_terminal(
|
|
2161
|
+
state,
|
|
2162
|
+
"BLOCKED",
|
|
2163
|
+
"repository_synchronization",
|
|
2164
|
+
f"Repository synchronization failed: {redact_diagnostic(str(error))}",
|
|
2165
|
+
)
|
|
2166
|
+
# The watcher checked the target before claim. Re-check the exact
|
|
2167
|
+
# checkout after the host-owned synchronization while the run lease
|
|
2168
|
+
# is held: an operator/worktree race must never reach reviewers,
|
|
2169
|
+
# providers, or validation on a feature branch.
|
|
2170
|
+
if not (evidence.clean and evidence.branch == "main" and evidence.main_contains_head):
|
|
2171
|
+
return self._save_terminal(
|
|
2172
|
+
state,
|
|
2173
|
+
"BLOCKED",
|
|
2174
|
+
"managed_target_baseline",
|
|
2175
|
+
"Managed target baseline verification failed: expected clean main synchronized with origin/main.",
|
|
2176
|
+
)
|
|
2177
|
+
admission_phase = self._start_phase(state.run_id, "DETERMINISTIC_ADMISSION", category="ADMISSION")
|
|
2178
|
+
state, admission_error = self._confirm_deterministic_admission(state)
|
|
2179
|
+
complete_phase(
|
|
2180
|
+
self.root,
|
|
2181
|
+
admission_phase,
|
|
2182
|
+
outcome="COMPLETE" if admission_error is None else "FAILED",
|
|
2183
|
+
)
|
|
2184
|
+
if admission_error is not None:
|
|
2185
|
+
return self._save_terminal(state, "BLOCKED", "deterministic_admission", admission_error)
|
|
2186
|
+
if state.action_intent == "VALIDATION_ONLY":
|
|
2187
|
+
state = self._bind_validation_only_profile(state, producer_context)
|
|
2188
|
+
if state.terminal:
|
|
2189
|
+
return state
|
|
2190
|
+
if context.execution_mode == "MANAGED":
|
|
2191
|
+
self._managed_action(state, "IMPLEMENTATION" if state.action_intent == "MUTATING_DELIVERY" else "VALIDATION_ONLY")
|
|
2192
|
+
self._provider_dispatch_telemetry = {
|
|
2193
|
+
"provider_dispatch_before_admission": 0,
|
|
2194
|
+
"admission_completed": 1,
|
|
2195
|
+
}
|
|
2196
|
+
state = replace(state, phase="CAPABILITY_REVIEW", next_action="capability_review")
|
|
2197
|
+
self.store.save(state)
|
|
2198
|
+
capability_review = self._start_phase(state.run_id, "CAPABILITY_REVIEW")
|
|
2199
|
+
reviewer_evidence = (
|
|
2200
|
+
ReviewerEvidence.from_repository(state.run_id, state.execution_mode, evidence)
|
|
2201
|
+
if state.execution_mode == "MANAGED" and state.action_intent == "MUTATING_DELIVERY"
|
|
2202
|
+
else None
|
|
2203
|
+
)
|
|
2204
|
+
memory = retrieve_engineering_memory(self.root, prompt_path)
|
|
2205
|
+
selections = select_reviewers(
|
|
2206
|
+
objective,
|
|
2207
|
+
prompt_path,
|
|
2208
|
+
state.transaction_kind if state else "IMPLEMENTATION",
|
|
2209
|
+
load_engineering_memory(self.root),
|
|
2210
|
+
) if state.action_intent == "MUTATING_DELIVERY" else ()
|
|
2211
|
+
self.reviewer_runtime = [
|
|
2212
|
+
{
|
|
2213
|
+
"reviewer": item.reviewer,
|
|
2214
|
+
"capability": item.capability,
|
|
2215
|
+
"selected_because": item.selected_because,
|
|
2216
|
+
"status": "selected",
|
|
2217
|
+
"selected_at": datetime.now(timezone.utc).isoformat(),
|
|
2218
|
+
}
|
|
2219
|
+
for item in selections
|
|
2220
|
+
]
|
|
2221
|
+
write_live_status(
|
|
2222
|
+
self.root,
|
|
2223
|
+
state
|
|
2224
|
+
or TransactionState(
|
|
2225
|
+
run_id or "pending-run", evidence.repository, str(prompt_path), "INITIALIZE"
|
|
2226
|
+
),
|
|
2227
|
+
"Capability Selection: "
|
|
2228
|
+
+ (
|
|
2229
|
+
", ".join(item.reviewer for item in selections)
|
|
2230
|
+
or "No specialist reviewers required."
|
|
2231
|
+
),
|
|
2232
|
+
self.reviewer_runtime,
|
|
2233
|
+
)
|
|
2234
|
+
self._require_provider_dispatch_admission(state)
|
|
2235
|
+
results = run_reviews(
|
|
2236
|
+
self.root,
|
|
2237
|
+
selections,
|
|
2238
|
+
objective,
|
|
2239
|
+
self.agent if hasattr(self.agent, "review") else None,
|
|
2240
|
+
progress=lambda selection, event, result: self._publish_reviewer_progress(
|
|
2241
|
+
state, selection, event, result
|
|
2242
|
+
),
|
|
2243
|
+
evidence=reviewer_evidence,
|
|
2244
|
+
)
|
|
2245
|
+
# Reviewer result objects retain only their own safe structured
|
|
2246
|
+
# telemetry, avoiding shared-client attribution across concurrent work.
|
|
2247
|
+
for reviewer in results:
|
|
2248
|
+
self._persist_provider_invocation(
|
|
2249
|
+
state, phase="CAPABILITY_REVIEW", role=f"reviewer:{reviewer.reviewer}",
|
|
2250
|
+
observed_usage=reviewer.usage, observed_metadata=reviewer.runtime_metadata,
|
|
2251
|
+
observed_churn=reviewer.churn, observed_duration=reviewer.duration_seconds,
|
|
2252
|
+
observed_snapshots=reviewer.usage_snapshots,
|
|
2253
|
+
)
|
|
2254
|
+
self.reviewer_records = records_for_storage(selections, results)
|
|
2255
|
+
# Reviewer reasoning is intentionally not merged into the primary
|
|
2256
|
+
# provider context. Reviewers share the bounded factual snapshot, but
|
|
2257
|
+
# retain independent reasoning responsibility and advisory records.
|
|
2258
|
+
state = (
|
|
2259
|
+
replace(state, phase="EXECUTE_AGENT", next_action="invoke_agent")
|
|
2260
|
+
if context.execution_mode == "GENESIS"
|
|
2261
|
+
else self._reconcile(state, evidence)
|
|
2262
|
+
)
|
|
2263
|
+
self.store.save(state)
|
|
2264
|
+
write_live_status(self.root, state, state.next_action)
|
|
2265
|
+
complete_phase(self.root, capability_review)
|
|
2266
|
+
if state.terminal or state.phase == "WAIT_FOR_TERMINAL_EVIDENCE":
|
|
2267
|
+
return self._poll(state)
|
|
2268
|
+
if state.action_intent == "VALIDATION_ONLY":
|
|
2269
|
+
# The provider-free path still executes the persisted controls;
|
|
2270
|
+
# absent pre-execution evidence is pending work, not terminally
|
|
2271
|
+
# missing evidence.
|
|
2272
|
+
state = self._execute_required_validation_controls(state)
|
|
2273
|
+
if state.terminal:
|
|
2274
|
+
return state
|
|
2275
|
+
try:
|
|
2276
|
+
validation_context = load_validation_context(self.root, state.run_id, central_database=self.store.central_database)
|
|
2277
|
+
except EngineeringStorageError:
|
|
2278
|
+
validation_context = None
|
|
2279
|
+
required = validation_context.get("required_validation_controls", ()) if validation_context else ()
|
|
2280
|
+
controls = validation_context.get("controls", {}) if validation_context else {}
|
|
2281
|
+
results = [controls.get(control, {}).get("result") for control in required]
|
|
2282
|
+
if any(result == "FAIL" for result in results):
|
|
2283
|
+
return self._save_terminal(state, "BLOCKED", "required_validation_failed", "Required validation controls did not pass.")
|
|
2284
|
+
if not required or not all(result == "PASS" for result in results):
|
|
2285
|
+
return self._save_terminal(state, "BLOCKED", "required_validation_unresolved", "Required validation controls do not have authoritative PASS evidence.")
|
|
2286
|
+
return self._poll(state, AgentResult("COMPLETE"))
|
|
2287
|
+
try:
|
|
2288
|
+
if hasattr(self.agent, "set_activity_callback"):
|
|
2289
|
+
self.agent.set_activity_callback(
|
|
2290
|
+
lambda activity: (self._heartbeat(), write_live_status(self.root, state, activity))[1]
|
|
2291
|
+
)
|
|
2292
|
+
if hasattr(self.agent, "set_transient_action_callback"):
|
|
2293
|
+
self.agent.set_transient_action_callback(
|
|
2294
|
+
lambda action: (self._heartbeat(), write_live_status(
|
|
2295
|
+
self.root, state, state.next_action, transient_action=action
|
|
2296
|
+
))[1]
|
|
2297
|
+
)
|
|
2298
|
+
if hasattr(self.agent, "set_process_callback"):
|
|
2299
|
+
self.agent.set_process_callback(
|
|
2300
|
+
lambda process: (
|
|
2301
|
+
write_runner_process(self.root, state.run_id, process),
|
|
2302
|
+
(
|
|
2303
|
+
record_provider_started(
|
|
2304
|
+
self.root, run_id=state.run_id,
|
|
2305
|
+
receipt_id=str(self._recovery_state(state.run_id).get("process_receipt_id")),
|
|
2306
|
+
pid=int(process["pid"]), process_group=int(process["process_group"]),
|
|
2307
|
+
central_database=self.store.central_database,
|
|
2308
|
+
)
|
|
2309
|
+
if isinstance(process, dict)
|
|
2310
|
+
and isinstance(self._recovery_state(state.run_id), dict)
|
|
2311
|
+
and self._recovery_state(state.run_id).get("state") == "RECOVERY_STARTING"
|
|
2312
|
+
and self._recovery_state(state.run_id).get("process_receipt_id")
|
|
2313
|
+
else None
|
|
2314
|
+
),
|
|
2315
|
+
)
|
|
2316
|
+
)
|
|
2317
|
+
if hasattr(self.agent, "set_runtime_metadata_callback"):
|
|
2318
|
+
self.agent.set_runtime_metadata_callback(
|
|
2319
|
+
lambda metadata: write_live_status(
|
|
2320
|
+
self.root, state, state.next_action, runtime_metadata=metadata
|
|
2321
|
+
)
|
|
2322
|
+
)
|
|
2323
|
+
if hasattr(self.agent, "set_workspace_progress_callback"):
|
|
2324
|
+
self.agent.set_workspace_progress_callback(
|
|
2325
|
+
lambda progress: (
|
|
2326
|
+
self._heartbeat(),
|
|
2327
|
+
write_live_status(
|
|
2328
|
+
self.root,
|
|
2329
|
+
state,
|
|
2330
|
+
state.next_action,
|
|
2331
|
+
workspace_progress=progress,
|
|
2332
|
+
),
|
|
2333
|
+
)[1]
|
|
2334
|
+
)
|
|
2335
|
+
result = self._invoke_agent_with_timing(
|
|
2336
|
+
state,
|
|
2337
|
+
assemble_prompt(
|
|
2338
|
+
prompt_path,
|
|
2339
|
+
state,
|
|
2340
|
+
managed_target=self.root if state.execution_mode == "MANAGED" else None,
|
|
2341
|
+
reviewer_evidence=reviewer_evidence,
|
|
2342
|
+
)
|
|
2343
|
+
+ memory,
|
|
2344
|
+
)
|
|
2345
|
+
state = self._record_agent_execution_time(state)
|
|
2346
|
+
state = self._record_validation_evidence(state, result)
|
|
2347
|
+
state = self._record_verified_result_commit(
|
|
2348
|
+
state,
|
|
2349
|
+
result,
|
|
2350
|
+
phase="EXECUTE_AGENT",
|
|
2351
|
+
description="implementation_agent_commit_verified",
|
|
2352
|
+
)
|
|
2353
|
+
self._persist_agent_usage(state.run_id)
|
|
2354
|
+
except ProviderReadinessBlocked as blocked:
|
|
2355
|
+
return blocked.state
|
|
2356
|
+
except CodexInvocationError as error:
|
|
2357
|
+
state = self._record_agent_execution_time(state)
|
|
2358
|
+
self.console_detail = error.console_detail
|
|
2359
|
+
return self._terminalize_provider_invocation_error(state, error)
|
|
2360
|
+
return self._advance_after_primary_agent_result(state, result, evidence)
|
|
2361
|
+
|
|
2362
|
+
def _active_genesis_transaction(self, target: Path, run_id: str) -> str | None:
|
|
2363
|
+
"""Return another active Genesis run that owns the same local workspace."""
|
|
2364
|
+
for checkpoint_id in self.store.run_ids():
|
|
2365
|
+
try:
|
|
2366
|
+
candidate = self.store.load(checkpoint_id)
|
|
2367
|
+
except StateError:
|
|
2368
|
+
continue
|
|
2369
|
+
if (
|
|
2370
|
+
candidate.run_id != run_id
|
|
2371
|
+
and not candidate.terminal
|
|
2372
|
+
and candidate.execution_mode == "GENESIS"
|
|
2373
|
+
and candidate.genesis_repository_path
|
|
2374
|
+
and Path(candidate.genesis_repository_path) == target
|
|
2375
|
+
):
|
|
2376
|
+
return candidate.run_id
|
|
2377
|
+
return None
|
|
2378
|
+
|
|
2379
|
+
def _reconcile_genesis_result(self, state: TransactionState, result: AgentResult) -> TransactionState:
|
|
2380
|
+
if result.terminal_state in {"BLOCKED", "FAILED"}:
|
|
2381
|
+
return self._save_terminal(state, result.terminal_state, "external_action_required", result.diagnostic)
|
|
2382
|
+
if result.terminal_state != "COMPLETE" or result.terminal_condition != "local_commit_reconciled":
|
|
2383
|
+
return self._save_terminal(state, "BLOCKED", "genesis_local_commit_required", "Genesis Mode requires a reconciled local commit.")
|
|
2384
|
+
if not result.repository_path or not result.commit_sha:
|
|
2385
|
+
return self._save_terminal(state, "BLOCKED", "genesis_checkpoint_required", "Genesis Mode requires repository path and commit checkpoint evidence.")
|
|
2386
|
+
target = Path(result.repository_path).expanduser()
|
|
2387
|
+
authorization_blocker = target_repository_authorization(self.root, target)
|
|
2388
|
+
if not target.is_absolute() or authorization_blocker:
|
|
2389
|
+
return self._save_terminal(state, "BLOCKED", "genesis_repository_scope", authorization_blocker or "Genesis preflight blocked: WORKSPACE_TARGET_AUTHORIZED: target path must be absolute.")
|
|
2390
|
+
try:
|
|
2391
|
+
git = getattr(self.repository, "provider", GitProvider())
|
|
2392
|
+
head = git.execute(target, "git", "rev-parse", "HEAD")
|
|
2393
|
+
clean = git.execute(target, "git", "status", "--porcelain", "--untracked-files=all")
|
|
2394
|
+
except OSError as error:
|
|
2395
|
+
return self._save_terminal(state, "BLOCKED", "genesis_local_repository_required", str(error))
|
|
2396
|
+
actual_head = head.stdout.strip()
|
|
2397
|
+
workspace = "clean" if not clean.stdout.strip() else "dirty"
|
|
2398
|
+
if head.returncode or clean.returncode or actual_head != result.commit_sha or workspace != "clean":
|
|
2399
|
+
diagnostic = (
|
|
2400
|
+
"Genesis reconciliation failed: "
|
|
2401
|
+
f"reported commit={result.commit_sha or 'missing'}; "
|
|
2402
|
+
f"actual HEAD={actual_head or 'unavailable'}; workspace={workspace}."
|
|
2403
|
+
)
|
|
2404
|
+
return self._save_terminal(state, "BLOCKED", "genesis_reconciliation_required", diagnostic)
|
|
2405
|
+
reconciled = replace(state, genesis_repository_path=str(target), genesis_commit_sha=result.commit_sha, latest_repository_evidence=f"local genesis commit {result.commit_sha}")
|
|
2406
|
+
reconciled = self._append_verified_commit_evidence(
|
|
2407
|
+
reconciled,
|
|
2408
|
+
phase="EXECUTE_AGENT",
|
|
2409
|
+
commit_sha=result.commit_sha,
|
|
2410
|
+
description="genesis_implementation_commit_verified",
|
|
2411
|
+
)
|
|
2412
|
+
return self._save_terminal(reconciled, "COMPLETE", "genesis_local_commit_reconciled")
|
|
2413
|
+
|
|
2414
|
+
def _verify_engineering_platform(self) -> None:
|
|
2415
|
+
try:
|
|
2416
|
+
self.detected_codex_cli = self.agent.version()
|
|
2417
|
+
self.platform_manifest = EngineeringPlatformManifest.load(
|
|
2418
|
+
package_path("ENGINEERING_PLATFORM_VERSION.json")
|
|
2419
|
+
)
|
|
2420
|
+
validate_compatibility(
|
|
2421
|
+
self.platform_manifest, self.compatibility, self.detected_codex_cli
|
|
2422
|
+
)
|
|
2423
|
+
configuration = PlatformConfiguration.load(self.root)
|
|
2424
|
+
if configuration.platform.version != self.platform_manifest.platform_version:
|
|
2425
|
+
raise EngineeringPlatformCompatibilityError("Platform identity and manifest version mismatch")
|
|
2426
|
+
except (EngineeringPlatformCompatibilityError, PlatformConfigurationError) as error:
|
|
2427
|
+
raise RunnerError(str(error)) from error
|
|
2428
|
+
|
|
2429
|
+
def _reconcile(self, state: TransactionState, evidence: RepositoryEvidence) -> TransactionState:
|
|
2430
|
+
if state.branch and evidence.branch not in {"main", state.branch}:
|
|
2431
|
+
raise RunnerError("current branch conflicts with active transaction")
|
|
2432
|
+
if state.pull_request:
|
|
2433
|
+
return replace(
|
|
2434
|
+
state,
|
|
2435
|
+
phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2436
|
+
last_verified_sha=evidence.head_sha,
|
|
2437
|
+
next_action="poll_required_checks",
|
|
2438
|
+
)
|
|
2439
|
+
if state.transaction_kind == "FINALIZATION" and state.phase == "FINALIZE_AGENT":
|
|
2440
|
+
if state.finalization_pull_request is None:
|
|
2441
|
+
# Finalization entry is persisted before its provider handoff.
|
|
2442
|
+
# A new host must continue that exact same entry rather than
|
|
2443
|
+
# requiring a PR that has not yet been created.
|
|
2444
|
+
return replace(
|
|
2445
|
+
state,
|
|
2446
|
+
last_verified_sha=evidence.head_sha,
|
|
2447
|
+
next_action="create_finalization",
|
|
2448
|
+
)
|
|
2449
|
+
return self._recover_finalization_pull_request(state, evidence)
|
|
2450
|
+
if (
|
|
2451
|
+
state.transaction_kind == "FINALIZATION"
|
|
2452
|
+
and state.finalization_pull_request is None
|
|
2453
|
+
and state.finalization_branch is not None
|
|
2454
|
+
):
|
|
2455
|
+
# Resumed-host setup temporarily projects CAPABILITY_REVIEW. Keep
|
|
2456
|
+
# the already durable Finalization entry instead of falling back
|
|
2457
|
+
# into the implementation execution phase.
|
|
2458
|
+
return replace(
|
|
2459
|
+
state,
|
|
2460
|
+
phase="FINALIZE_AGENT",
|
|
2461
|
+
last_verified_sha=evidence.head_sha,
|
|
2462
|
+
next_action="create_finalization",
|
|
2463
|
+
)
|
|
2464
|
+
if (
|
|
2465
|
+
state.transaction_kind == "FINALIZATION"
|
|
2466
|
+
and state.implementation_pull_request is None
|
|
2467
|
+
and not state.finalization_pull_request
|
|
2468
|
+
):
|
|
2469
|
+
return replace(
|
|
2470
|
+
state,
|
|
2471
|
+
phase="FINALIZE_AGENT",
|
|
2472
|
+
last_verified_sha=evidence.head_sha,
|
|
2473
|
+
next_action="create_finalization",
|
|
2474
|
+
)
|
|
2475
|
+
if state.transaction_kind == "RECONCILIATION" and not state.reconciliation_pull_request:
|
|
2476
|
+
return replace(
|
|
2477
|
+
state, phase="RECONCILE_AGENT", last_verified_sha=evidence.head_sha,
|
|
2478
|
+
next_action="create_reconciliation",
|
|
2479
|
+
)
|
|
2480
|
+
if (
|
|
2481
|
+
state.transaction_kind == "FINALIZATION"
|
|
2482
|
+
and state.finalization_merge_commit
|
|
2483
|
+
and self.repository.main_contains(self.root, state.finalization_merge_commit)
|
|
2484
|
+
):
|
|
2485
|
+
return self._cleanup(state)
|
|
2486
|
+
if (
|
|
2487
|
+
state.transaction_kind == "IMPLEMENTATION"
|
|
2488
|
+
and state.implementation_merge_commit
|
|
2489
|
+
and self.repository.main_contains(self.root, state.implementation_merge_commit)
|
|
2490
|
+
):
|
|
2491
|
+
if state.owner_authorized:
|
|
2492
|
+
return self._start_finalization(state, state.implementation_pull_request or 0)
|
|
2493
|
+
return replace(
|
|
2494
|
+
state,
|
|
2495
|
+
phase="EXECUTE_AGENT",
|
|
2496
|
+
last_verified_sha=evidence.head_sha,
|
|
2497
|
+
next_action="invoke_agent",
|
|
2498
|
+
)
|
|
2499
|
+
|
|
2500
|
+
def _recover_finalization_pull_request(
|
|
2501
|
+
self, state: TransactionState, evidence: RepositoryEvidence,
|
|
2502
|
+
) -> TransactionState:
|
|
2503
|
+
"""Persist an existing Finalization PR only after current proof.
|
|
2504
|
+
|
|
2505
|
+
The checkpointed branch is the recovery identity. A missing or
|
|
2506
|
+
ambiguous match is terminally blocked rather than retried, so recovery
|
|
2507
|
+
never invokes an agent or creates a replacement Finalization PR.
|
|
2508
|
+
"""
|
|
2509
|
+
if state.finalization_pull_request:
|
|
2510
|
+
return replace(
|
|
2511
|
+
state, pull_request=state.finalization_pull_request,
|
|
2512
|
+
branch=state.finalization_branch, phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2513
|
+
next_action="poll_required_checks",
|
|
2514
|
+
)
|
|
2515
|
+
if (
|
|
2516
|
+
not state.finalization_branch
|
|
2517
|
+
or not state.implementation_merge_commit
|
|
2518
|
+
or evidence.branch != "main"
|
|
2519
|
+
or not evidence.clean
|
|
2520
|
+
or not evidence.main_contains_head
|
|
2521
|
+
or not self.repository.main_contains(self.root, state.implementation_merge_commit)
|
|
2522
|
+
):
|
|
2523
|
+
return self._save_terminal(
|
|
2524
|
+
state, "BLOCKED", "finalization_recovery_evidence_required",
|
|
2525
|
+
"Finalization recovery requires a clean current main checkout and a verified implementation merge.",
|
|
2526
|
+
)
|
|
2527
|
+
candidate = self.github.pull_request_for_head_branch(state.finalization_branch)
|
|
2528
|
+
if candidate is None:
|
|
2529
|
+
return self._save_terminal(
|
|
2530
|
+
state, "BLOCKED", "finalization_recovery_evidence_required",
|
|
2531
|
+
"No Finalization pull request matches the checkpointed branch; no replacement was created.",
|
|
2532
|
+
)
|
|
2533
|
+
if (
|
|
2534
|
+
candidate.head_branch != state.finalization_branch
|
|
2535
|
+
or candidate.base_branch != "main"
|
|
2536
|
+
or candidate.state not in {"OPEN", "MERGED"}
|
|
2537
|
+
):
|
|
2538
|
+
return self._save_terminal(
|
|
2539
|
+
state, "BLOCKED", "finalization_recovery_evidence_invalid",
|
|
2540
|
+
"Finalization pull request evidence does not match the checkpointed branch and main base.",
|
|
2541
|
+
)
|
|
2542
|
+
recovered = replace(
|
|
2543
|
+
state,
|
|
2544
|
+
branch=state.finalization_branch,
|
|
2545
|
+
pull_request=candidate.number,
|
|
2546
|
+
finalization_pull_request=candidate.number,
|
|
2547
|
+
phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2548
|
+
next_action="poll_required_checks",
|
|
2549
|
+
last_verified_sha=evidence.head_sha,
|
|
2550
|
+
latest_repository_evidence=_repository_summary(evidence),
|
|
2551
|
+
latest_github_evidence=_pull_request_summary(candidate),
|
|
2552
|
+
)
|
|
2553
|
+
self.store.save(recovered)
|
|
2554
|
+
write_live_status(self.root, recovered, recovered.next_action)
|
|
2555
|
+
return self._poll(recovered)
|
|
2556
|
+
|
|
2557
|
+
def _poll(self, state: TransactionState, result: AgentResult | None = None) -> TransactionState:
|
|
2558
|
+
if state.pull_request:
|
|
2559
|
+
state = self._provider_readiness_gate(
|
|
2560
|
+
state, require_codex=False, require_github=True
|
|
2561
|
+
)
|
|
2562
|
+
if state.next_action == "provider_auth_repair_required":
|
|
2563
|
+
return state
|
|
2564
|
+
if result and result.terminal_state in {"BLOCKED", "FAILED"}:
|
|
2565
|
+
return self._save_terminal(
|
|
2566
|
+
state, result.terminal_state, "external_action_required", result.diagnostic
|
|
2567
|
+
)
|
|
2568
|
+
if result and result.pull_request and result.branch in {None, "main"}:
|
|
2569
|
+
return self._save_terminal(
|
|
2570
|
+
state,
|
|
2571
|
+
"BLOCKED",
|
|
2572
|
+
"invalid_pull_request_evidence",
|
|
2573
|
+
"Agent result referenced a pull request without a transaction branch; the current main branch cannot be reused as execution evidence.",
|
|
2574
|
+
)
|
|
2575
|
+
if not state.pull_request:
|
|
2576
|
+
if state.transaction_kind == "RECONCILIATION" and result and result.pull_request:
|
|
2577
|
+
return self._save_terminal(
|
|
2578
|
+
state, "BLOCKED", "reconciliation_pull_request_forbidden",
|
|
2579
|
+
"Automatic reconciliation must commit directly to main and must not create a pull request.",
|
|
2580
|
+
)
|
|
2581
|
+
if state.transaction_kind == "RECONCILIATION" and result and result.terminal_state == "COMPLETE":
|
|
2582
|
+
evidence = self.repository.inspect(self.root)
|
|
2583
|
+
if (
|
|
2584
|
+
result.terminal_condition == "repository_reconciled"
|
|
2585
|
+
and result.commit_sha
|
|
2586
|
+
and evidence.clean
|
|
2587
|
+
and evidence.branch == "main"
|
|
2588
|
+
and evidence.head_sha == result.commit_sha
|
|
2589
|
+
and evidence.main_contains_head
|
|
2590
|
+
):
|
|
2591
|
+
reconciled = replace(state, last_verified_sha=result.commit_sha)
|
|
2592
|
+
reconciled = self._append_verified_commit_evidence(
|
|
2593
|
+
reconciled,
|
|
2594
|
+
phase="RECONCILE_AGENT",
|
|
2595
|
+
commit_sha=result.commit_sha,
|
|
2596
|
+
description="end_reconciliation_commit_verified",
|
|
2597
|
+
)
|
|
2598
|
+
return self._cleanup(reconciled)
|
|
2599
|
+
return self._save_terminal(
|
|
2600
|
+
state, "BLOCKED", "automatic_reconciliation_evidence_required",
|
|
2601
|
+
"Automatic reconciliation requires a clean main checkout containing its reported commit.",
|
|
2602
|
+
)
|
|
2603
|
+
if result and result.terminal_state == "COMPLETE":
|
|
2604
|
+
evidence = self.repository.inspect(self.root)
|
|
2605
|
+
if evidence.clean and evidence.main_contains_head:
|
|
2606
|
+
return self._cleanup(state)
|
|
2607
|
+
return replace(
|
|
2608
|
+
state, phase="WAIT_FOR_TERMINAL_EVIDENCE", next_action="obtain_repository_evidence"
|
|
2609
|
+
)
|
|
2610
|
+
attempts = 0
|
|
2611
|
+
while True:
|
|
2612
|
+
pr_operation = self._start_phase(state.run_id, "PR_OR_MERGE")
|
|
2613
|
+
try:
|
|
2614
|
+
pr = self.github.pull_request(state.pull_request)
|
|
2615
|
+
except RunnerError:
|
|
2616
|
+
complete_phase(self.root, pr_operation, outcome="FAILED")
|
|
2617
|
+
attempts += 1
|
|
2618
|
+
if attempts >= 3:
|
|
2619
|
+
# No provider or foreground GitHub operation remains after
|
|
2620
|
+
# the bounded evidence retries. This is a durable passive
|
|
2621
|
+
# wait, so it must relinquish the run lease just like the
|
|
2622
|
+
# operator-merge hand-off. Otherwise a lifecycle worker
|
|
2623
|
+
# cannot resume the same run after the transient outage.
|
|
2624
|
+
return self._save_operator_merge_wait(
|
|
2625
|
+
replace(
|
|
2626
|
+
state,
|
|
2627
|
+
phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2628
|
+
next_action="retry_github_evidence",
|
|
2629
|
+
)
|
|
2630
|
+
)
|
|
2631
|
+
wait = self._start_phase(state.run_id, "EXTERNAL_CI_WAIT", metadata={"reason": "github_evidence_retry"})
|
|
2632
|
+
self.sleep(min(30, 2**attempts))
|
|
2633
|
+
complete_phase(self.root, wait)
|
|
2634
|
+
continue
|
|
2635
|
+
complete_phase(self.root, pr_operation)
|
|
2636
|
+
self._managed_pr_check(state, pr)
|
|
2637
|
+
# GitHub can omit or retire a check rollup once a pull request is
|
|
2638
|
+
# merged. The merge is authoritative remote evidence and must
|
|
2639
|
+
# therefore take precedence over pre-merge check polling.
|
|
2640
|
+
if pr.state != "MERGED" and not pr.checks_terminal:
|
|
2641
|
+
self._managed_action(state, "GITHUB_REQUIRED_CHECK", "EXTERNAL_PLATFORM_EVENT", actor="github", evidence_ref="required_check_waiting")
|
|
2642
|
+
wait = self._start_phase(state.run_id, "EXTERNAL_CI_WAIT", metadata={"reason": "github_checks"})
|
|
2643
|
+
self.sleep(15)
|
|
2644
|
+
complete_phase(self.root, wait)
|
|
2645
|
+
continue
|
|
2646
|
+
repairable_mergeability = pr.state == "OPEN" and pr.merge_state_status in {"BEHIND", "DIRTY", "UNSTABLE"}
|
|
2647
|
+
if pr.state != "MERGED" and (not pr.checks_passed or repairable_mergeability):
|
|
2648
|
+
if state.owner_authorized:
|
|
2649
|
+
failed = (
|
|
2650
|
+
", ".join(pr.failed_checks) or "required CI check"
|
|
2651
|
+
if not pr.checks_passed
|
|
2652
|
+
else "pull request is behind or cannot merge cleanly with main"
|
|
2653
|
+
)
|
|
2654
|
+
if state.repair_iterations >= MAX_PR_CHECK_REPAIR_ATTEMPTS:
|
|
2655
|
+
return self._save_terminal(
|
|
2656
|
+
state,
|
|
2657
|
+
"BLOCKED",
|
|
2658
|
+
"repair_attempt_limit_reached",
|
|
2659
|
+
"Required CI checks still failed after "
|
|
2660
|
+
f"{MAX_PR_CHECK_REPAIR_ATTEMPTS} bounded repair attempts: {failed}.",
|
|
2661
|
+
terminal_condition="repair_attempt_limit_reached",
|
|
2662
|
+
)
|
|
2663
|
+
return self._repair(
|
|
2664
|
+
state,
|
|
2665
|
+
f"{failed} failed. Repair only the bounded transaction defects, commit and push the repair, then return the same pull request number.",
|
|
2666
|
+
)
|
|
2667
|
+
return self._save_terminal(
|
|
2668
|
+
state, "FAILED", "required_checks_failed", "Required CI check failed."
|
|
2669
|
+
)
|
|
2670
|
+
if pr.state == "MERGED":
|
|
2671
|
+
# A merge is remote evidence. Refresh origin/main before
|
|
2672
|
+
# verifying ancestry, without switching or fast-forwarding
|
|
2673
|
+
# the shared checkout during a passive wait poll. Local main
|
|
2674
|
+
# synchronization belongs to finalization/cleanup, after the
|
|
2675
|
+
# remote merge has been verified.
|
|
2676
|
+
try:
|
|
2677
|
+
self.repository.refresh_main_reference(self.root)
|
|
2678
|
+
evidence = self.repository.inspect(self.root)
|
|
2679
|
+
except RunnerError:
|
|
2680
|
+
return self._save_operator_merge_wait(state)
|
|
2681
|
+
if pr.merge_commit and self.repository.remote_main_contains(self.root, pr.merge_commit):
|
|
2682
|
+
gate_type = {"IMPLEMENTATION": "IMPLEMENTATION_MERGE_APPROVAL", "FINALIZATION": "FINALIZATION_MERGE_APPROVAL", "RECONCILIATION": "RECONCILIATION_MERGE_APPROVAL"}[state.transaction_kind]
|
|
2683
|
+
self._managed_gate(state, gate_type, "SATISFIED", pr.number, resolved=True)
|
|
2684
|
+
action = {"IMPLEMENTATION": "IMPLEMENTATION_MERGE", "FINALIZATION": "FINALIZATION_MERGE", "RECONCILIATION": "RECONCILIATION_MERGE"}[state.transaction_kind]
|
|
2685
|
+
self._managed_action(state, action, "EXPECTED_OPERATOR_GATE", actor="operator", evidence_ref="github_merge")
|
|
2686
|
+
state = self._record_merged_evidence(state, pr, evidence)
|
|
2687
|
+
if state.owner_authorized and state.transaction_kind == "IMPLEMENTATION":
|
|
2688
|
+
return self._start_finalization(state, pr.number)
|
|
2689
|
+
if state.owner_authorized and state.transaction_kind == "FINALIZATION":
|
|
2690
|
+
return self._start_automatic_reconciliation(state)
|
|
2691
|
+
return self._cleanup(state)
|
|
2692
|
+
# A green PR is an explicit hand-off to the operator. The runner
|
|
2693
|
+
# must not turn that durable waiting state into a synthetic failure
|
|
2694
|
+
# merely because its foreground process has ended.
|
|
2695
|
+
waiting = replace(
|
|
2696
|
+
state,
|
|
2697
|
+
phase="WAIT_FOR_OPERATOR_MERGE",
|
|
2698
|
+
next_action="await_operator_pr_merge",
|
|
2699
|
+
terminal_condition="operator_merge_required",
|
|
2700
|
+
diagnostic=None,
|
|
2701
|
+
waiting_for_merge_since=state.waiting_for_merge_since
|
|
2702
|
+
or datetime.now(timezone.utc).isoformat(),
|
|
2703
|
+
)
|
|
2704
|
+
gate_type = {"IMPLEMENTATION": "IMPLEMENTATION_MERGE_APPROVAL", "FINALIZATION": "FINALIZATION_MERGE_APPROVAL", "RECONCILIATION": "RECONCILIATION_MERGE_APPROVAL"}[state.transaction_kind]
|
|
2705
|
+
self._managed_gate(waiting, gate_type, "WAITING", state.pull_request)
|
|
2706
|
+
return self._save_operator_merge_wait(waiting)
|
|
2707
|
+
|
|
2708
|
+
def _repair(self, state: TransactionState, objective: str) -> TransactionState:
|
|
2709
|
+
failed_checks = objective.split(" failed.", 1)[0]
|
|
2710
|
+
repair = replace(
|
|
2711
|
+
state,
|
|
2712
|
+
phase="REPAIR_AGENT",
|
|
2713
|
+
next_action="repair_bounded_validation_failure",
|
|
2714
|
+
repair_iterations=state.repair_iterations + 1,
|
|
2715
|
+
)
|
|
2716
|
+
# This plan is the canonical, bounded context needed to apply a
|
|
2717
|
+
# successful recovered result after a host restart. It is persisted
|
|
2718
|
+
# before the provider can begin; it is never rebuilt from prompt text.
|
|
2719
|
+
repair = self._record_repair_audit(
|
|
2720
|
+
repair, failed_checks=failed_checks, objective=objective, result=None, outcome="planned",
|
|
2721
|
+
)
|
|
2722
|
+
self.store.save(repair)
|
|
2723
|
+
write_live_status(self.root, repair, repair.next_action)
|
|
2724
|
+
try:
|
|
2725
|
+
result = self._invoke_agent_with_timing(
|
|
2726
|
+
repair,
|
|
2727
|
+
assemble_prompt(
|
|
2728
|
+
Path(repair.prompt_path),
|
|
2729
|
+
repair,
|
|
2730
|
+
managed_target=self.root if repair.execution_mode == "MANAGED" else None,
|
|
2731
|
+
)
|
|
2732
|
+
+ f"\n\nRepair objective: {objective}",
|
|
2733
|
+
repair=True,
|
|
2734
|
+
)
|
|
2735
|
+
repair = self._record_agent_execution_time(repair)
|
|
2736
|
+
repair = self._record_validation_evidence(repair, result)
|
|
2737
|
+
repair = self._record_verified_result_commit(
|
|
2738
|
+
repair,
|
|
2739
|
+
result,
|
|
2740
|
+
phase="REPAIR_AGENT",
|
|
2741
|
+
description="pull_request_repair_commit_verified",
|
|
2742
|
+
)
|
|
2743
|
+
self._persist_agent_usage(repair.run_id)
|
|
2744
|
+
except CodexHandoffTimeout:
|
|
2745
|
+
repair = self._record_agent_execution_time(repair)
|
|
2746
|
+
repair = self._record_repair_audit(
|
|
2747
|
+
repair, failed_checks=failed_checks, objective=objective, result=None,
|
|
2748
|
+
outcome="agent_timed_out",
|
|
2749
|
+
)
|
|
2750
|
+
return self._save_terminal(
|
|
2751
|
+
repair,
|
|
2752
|
+
"BLOCKED",
|
|
2753
|
+
"repair_agent_timeout",
|
|
2754
|
+
"Repair agent exceeded the host-owned deadline; no further repair was started.",
|
|
2755
|
+
terminal_condition="repair_agent_timeout",
|
|
2756
|
+
)
|
|
2757
|
+
except ProviderReadinessBlocked as blocked:
|
|
2758
|
+
return blocked.state
|
|
2759
|
+
except CodexInvocationError as error:
|
|
2760
|
+
repair = self._record_agent_execution_time(repair)
|
|
2761
|
+
self.console_detail = error.console_detail
|
|
2762
|
+
repair = self._record_repair_audit(repair, failed_checks=failed_checks, objective=objective, result=None, outcome="agent_failed")
|
|
2763
|
+
return self._terminalize_provider_invocation_error(repair, error)
|
|
2764
|
+
return self._advance_after_repair_agent_result(repair, result)
|
|
2765
|
+
|
|
2766
|
+
def _start_finalization(
|
|
2767
|
+
self, state: TransactionState, implementation_pr: int
|
|
2768
|
+
) -> TransactionState:
|
|
2769
|
+
self._managed_action(state, "POST_IMPLEMENTATION_MERGE")
|
|
2770
|
+
if state.finalization_pull_request:
|
|
2771
|
+
return replace(
|
|
2772
|
+
state,
|
|
2773
|
+
transaction_kind="FINALIZATION",
|
|
2774
|
+
pull_request=state.finalization_pull_request,
|
|
2775
|
+
branch=state.finalization_branch,
|
|
2776
|
+
phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2777
|
+
next_action="poll_required_checks",
|
|
2778
|
+
)
|
|
2779
|
+
finalization_phase = self._start_phase(state.run_id, "REPOSITORY_FINALIZATION")
|
|
2780
|
+
synchronize = getattr(self.repository, "synchronize_main", None)
|
|
2781
|
+
if callable(synchronize):
|
|
2782
|
+
synchronize(self.root)
|
|
2783
|
+
evidence = self.repository.inspect(self.root)
|
|
2784
|
+
if not evidence.clean or evidence.branch != "main":
|
|
2785
|
+
complete_phase(self.root, finalization_phase, outcome="FAILED")
|
|
2786
|
+
return self._save_post_merge_sync_wait(
|
|
2787
|
+
state,
|
|
2788
|
+
"Finalization is waiting for a clean, synchronized main checkout.",
|
|
2789
|
+
)
|
|
2790
|
+
expected_branch = state.finalization_branch or f"codex/finalize-{state.run_id}"
|
|
2791
|
+
finalization = replace(
|
|
2792
|
+
state,
|
|
2793
|
+
phase="FINALIZE_AGENT",
|
|
2794
|
+
transaction_kind="FINALIZATION",
|
|
2795
|
+
pull_request=None,
|
|
2796
|
+
branch=expected_branch,
|
|
2797
|
+
finalization_branch=expected_branch,
|
|
2798
|
+
next_action="create_finalization",
|
|
2799
|
+
implementation_pull_request=implementation_pr or state.implementation_pull_request,
|
|
2800
|
+
latest_repository_evidence=_repository_summary(evidence),
|
|
2801
|
+
waiting_for_merge_since=None,
|
|
2802
|
+
)
|
|
2803
|
+
self._managed_action(finalization, "FINALIZATION")
|
|
2804
|
+
self.store.save(finalization)
|
|
2805
|
+
write_live_status(self.root, finalization, finalization.next_action)
|
|
2806
|
+
complete_phase(self.root, finalization_phase)
|
|
2807
|
+
instruction = (
|
|
2808
|
+
f"\n\nThe implementation PR #{implementation_pr} is merged. Execute only its mandatory "
|
|
2809
|
+
"governance-only Finalization: reconcile the four rolling records and immutable Prompt "
|
|
2810
|
+
f"History, then create a draft Finalization PR on exactly `{expected_branch}`. After GitHub assigns its number, run "
|
|
2811
|
+
f"`python3 -m engineering_platform.repository_handoff --run-id {finalization.run_id} "
|
|
2812
|
+
f"--platform-version {self.platform_manifest.platform_version if self.platform_manifest else 'unknown'} "
|
|
2813
|
+
f"--implementation-pr {implementation_pr} --finalization-pr <PR_NUMBER>`, commit the "
|
|
2814
|
+
"resulting `docs/engineering/runs/` handoff records to that same Finalization branch, "
|
|
2815
|
+
"push it, and only then return that PR number."
|
|
2816
|
+
)
|
|
2817
|
+
finalization_span = self._start_phase(state.run_id, "FINALIZATION")
|
|
2818
|
+
handoff_started = time.monotonic()
|
|
2819
|
+
set_handoff_deadline = getattr(self.agent, "set_handoff_deadline_callback", None)
|
|
2820
|
+
if callable(set_handoff_deadline):
|
|
2821
|
+
set_handoff_deadline(lambda: time.monotonic() - handoff_started >= FINALIZATION_PR_HANDOFF_MAX_SECONDS)
|
|
2822
|
+
try:
|
|
2823
|
+
result = self._invoke_agent_with_timing(
|
|
2824
|
+
finalization,
|
|
2825
|
+
assemble_prompt(
|
|
2826
|
+
Path(finalization.prompt_path),
|
|
2827
|
+
finalization,
|
|
2828
|
+
managed_target=self.root if finalization.execution_mode == "MANAGED" else None,
|
|
2829
|
+
)
|
|
2830
|
+
+ instruction,
|
|
2831
|
+
)
|
|
2832
|
+
finalization = self._record_agent_execution_time(finalization)
|
|
2833
|
+
finalization = self._record_validation_evidence(finalization, result)
|
|
2834
|
+
finalization = self._record_verified_result_commit(
|
|
2835
|
+
finalization,
|
|
2836
|
+
result,
|
|
2837
|
+
phase="FINALIZE_AGENT",
|
|
2838
|
+
description="finalization_commit_verified",
|
|
2839
|
+
)
|
|
2840
|
+
self._persist_agent_usage(finalization.run_id)
|
|
2841
|
+
except CodexHandoffTimeout:
|
|
2842
|
+
complete_phase(self.root, finalization_span, outcome="FAILED")
|
|
2843
|
+
finalization = self._record_agent_execution_time(finalization)
|
|
2844
|
+
# The timeout is not a terminal agent result. Reconcile only a
|
|
2845
|
+
# PR that current evidence proves already exists; otherwise the
|
|
2846
|
+
# recovery helper blocks fail-closed without another invocation.
|
|
2847
|
+
return self._recover_finalization_pull_request(finalization, self.repository.inspect(self.root))
|
|
2848
|
+
except ProviderReadinessBlocked as blocked:
|
|
2849
|
+
complete_phase(self.root, finalization_span, outcome="BLOCKED")
|
|
2850
|
+
return blocked.state
|
|
2851
|
+
except CodexInvocationError as error:
|
|
2852
|
+
complete_phase(
|
|
2853
|
+
self.root, finalization_span,
|
|
2854
|
+
outcome="INTERRUPTED" if error.provider_turn_interrupted else "FAILED",
|
|
2855
|
+
)
|
|
2856
|
+
finalization = self._record_agent_execution_time(finalization)
|
|
2857
|
+
self.console_detail = error.console_detail
|
|
2858
|
+
return self._terminalize_provider_invocation_error(finalization, error)
|
|
2859
|
+
finally:
|
|
2860
|
+
if callable(set_handoff_deadline):
|
|
2861
|
+
set_handoff_deadline(None)
|
|
2862
|
+
complete_phase(self.root, finalization_span)
|
|
2863
|
+
return self._advance_after_finalization_agent_result(finalization, result)
|
|
2864
|
+
|
|
2865
|
+
def _advance_after_finalization_agent_result(
|
|
2866
|
+
self, finalization: TransactionState, result: AgentResult,
|
|
2867
|
+
) -> TransactionState:
|
|
2868
|
+
"""Apply a live or durably recovered Finalization result.
|
|
2869
|
+
|
|
2870
|
+
Timing spans belong exclusively to the live caller. This method uses
|
|
2871
|
+
only checkpointed lifecycle identity and the validated AgentResult so
|
|
2872
|
+
a restarted host cannot fabricate Finalization timing evidence.
|
|
2873
|
+
"""
|
|
2874
|
+
if result.terminal_state in {"BLOCKED", "FAILED"} or not result.pull_request:
|
|
2875
|
+
return self._save_terminal(
|
|
2876
|
+
finalization,
|
|
2877
|
+
result.terminal_state
|
|
2878
|
+
if result.terminal_state in {"BLOCKED", "FAILED"}
|
|
2879
|
+
else "BLOCKED",
|
|
2880
|
+
"finalization_pr_required",
|
|
2881
|
+
result.diagnostic or "Finalization pull request was not created.",
|
|
2882
|
+
)
|
|
2883
|
+
finalization = replace(
|
|
2884
|
+
finalization,
|
|
2885
|
+
phase="WAIT_FOR_TERMINAL_EVIDENCE",
|
|
2886
|
+
branch=result.branch,
|
|
2887
|
+
pull_request=result.pull_request,
|
|
2888
|
+
finalization_branch=result.branch,
|
|
2889
|
+
finalization_pull_request=result.pull_request,
|
|
2890
|
+
terminal_condition="repository_reconciled",
|
|
2891
|
+
next_action="poll_required_checks",
|
|
2892
|
+
)
|
|
2893
|
+
self.store.save(finalization)
|
|
2894
|
+
write_live_status(self.root, finalization, finalization.next_action)
|
|
2895
|
+
# A resumed transaction can discover that its mandatory Finalization
|
|
2896
|
+
# PR was already merged before the agent returned it. It is valid
|
|
2897
|
+
# evidence for this same transaction, so reconcile it through the
|
|
2898
|
+
# normal merge/cleanup path instead of trying to mark a closed PR
|
|
2899
|
+
# ready for review.
|
|
2900
|
+
finalization_evidence = self.github.pull_request(result.pull_request)
|
|
2901
|
+
if finalization_evidence.state == "MERGED":
|
|
2902
|
+
return self._poll(finalization, result)
|
|
2903
|
+
self.github.normalize_markdown_body(result.pull_request)
|
|
2904
|
+
self.github.ready(result.pull_request)
|
|
2905
|
+
return self._poll(finalization, result)
|
|
2906
|
+
|
|
2907
|
+
def _start_automatic_reconciliation(self, state: TransactionState) -> TransactionState:
|
|
2908
|
+
"""Apply the bounded rolling-record update after Finalization merges.
|
|
2909
|
+
|
|
2910
|
+
This is deliberately a direct, post-merge `main` commit: it has no PR,
|
|
2911
|
+
review, approval, or operator merge boundary. Its scope is enforced by
|
|
2912
|
+
the supplied reconciliation prompt and the exact commit evidence below.
|
|
2913
|
+
"""
|
|
2914
|
+
synchronize = getattr(self.repository, "synchronize_main", None)
|
|
2915
|
+
if callable(synchronize):
|
|
2916
|
+
synchronize(self.root)
|
|
2917
|
+
evidence = self.repository.inspect(self.root)
|
|
2918
|
+
if not evidence.clean or evidence.branch != "main":
|
|
2919
|
+
return self._save_post_merge_sync_wait(
|
|
2920
|
+
state,
|
|
2921
|
+
"End reconciliation is waiting for a clean, synchronized main checkout.",
|
|
2922
|
+
)
|
|
2923
|
+
reconciliation = replace(
|
|
2924
|
+
state, phase="RECONCILE_AGENT", transaction_kind="RECONCILIATION",
|
|
2925
|
+
branch=None, pull_request=None, reconciliation_pull_request=None,
|
|
2926
|
+
next_action="reconcile_rolling_records_on_main", waiting_for_merge_since=None,
|
|
2927
|
+
latest_repository_evidence=_repository_summary(evidence),
|
|
2928
|
+
)
|
|
2929
|
+
self._managed_action(reconciliation, "AUTOMATIC_RECONCILIATION")
|
|
2930
|
+
self.store.save(reconciliation)
|
|
2931
|
+
write_live_status(self.root, reconciliation, reconciliation.next_action)
|
|
2932
|
+
try:
|
|
2933
|
+
result = self._invoke_agent_with_timing(
|
|
2934
|
+
reconciliation,
|
|
2935
|
+
assemble_prompt(
|
|
2936
|
+
Path(reconciliation.prompt_path), reconciliation,
|
|
2937
|
+
managed_target=self.root if reconciliation.execution_mode == "MANAGED" else None,
|
|
2938
|
+
) + "\n\nReconcile only the four canonical rolling current-state records after the verified Finalization merge. Preserve immutable Prompt History.",
|
|
2939
|
+
)
|
|
2940
|
+
reconciliation = self._record_agent_execution_time(reconciliation)
|
|
2941
|
+
reconciliation = self._record_validation_evidence(reconciliation, result)
|
|
2942
|
+
reconciliation = self._record_verified_result_commit(
|
|
2943
|
+
reconciliation,
|
|
2944
|
+
result,
|
|
2945
|
+
phase="RECONCILE_AGENT",
|
|
2946
|
+
description="end_reconciliation_commit_verified",
|
|
2947
|
+
)
|
|
2948
|
+
self._persist_agent_usage(reconciliation.run_id)
|
|
2949
|
+
except ProviderReadinessBlocked as blocked:
|
|
2950
|
+
return blocked.state
|
|
2951
|
+
except CodexInvocationError as error:
|
|
2952
|
+
reconciliation = self._record_agent_execution_time(reconciliation)
|
|
2953
|
+
self.console_detail = error.console_detail
|
|
2954
|
+
return self._terminalize_provider_invocation_error(reconciliation, error)
|
|
2955
|
+
return self._poll(reconciliation, result)
|
|
2956
|
+
|
|
2957
|
+
def _save_terminal(
|
|
2958
|
+
self,
|
|
2959
|
+
state: TransactionState,
|
|
2960
|
+
phase: str,
|
|
2961
|
+
action: str,
|
|
2962
|
+
diagnostic: str | None = None,
|
|
2963
|
+
*,
|
|
2964
|
+
terminal_condition: str | None = None,
|
|
2965
|
+
) -> TransactionState:
|
|
2966
|
+
terminal = replace(
|
|
2967
|
+
state,
|
|
2968
|
+
phase=phase,
|
|
2969
|
+
terminal=True,
|
|
2970
|
+
next_action=action,
|
|
2971
|
+
terminal_condition=terminal_condition or state.terminal_condition,
|
|
2972
|
+
diagnostic=redact_diagnostic(diagnostic) if diagnostic else None,
|
|
2973
|
+
)
|
|
2974
|
+
self.store.save(terminal)
|
|
2975
|
+
if self.active_lease is not None and self.active_lease.run_id == terminal.run_id:
|
|
2976
|
+
lease = self.active_lease
|
|
2977
|
+
try:
|
|
2978
|
+
if self.lease_heartbeat is not None:
|
|
2979
|
+
lease = self.lease_heartbeat.stop()
|
|
2980
|
+
self.lease_heartbeat = None
|
|
2981
|
+
release_lease(self.root, lease, central_database=self.store.central_database)
|
|
2982
|
+
except Exception:
|
|
2983
|
+
# A durable terminal checkpoint is authoritative even when
|
|
2984
|
+
# post-terminal lease cleanup is unavailable. Stale lease
|
|
2985
|
+
# reconciliation records that separate cleanup concern.
|
|
2986
|
+
LOGGER.exception("Terminal lease release failed for run %s", terminal.run_id)
|
|
2987
|
+
finally:
|
|
2988
|
+
self.active_lease = None
|
|
2989
|
+
if phase == "COMPLETE":
|
|
2990
|
+
capture_engineering_memory(self.root, terminal, self.reviewer_records)
|
|
2991
|
+
write_live_status(self.root, terminal, action)
|
|
2992
|
+
print(f"[{terminal.phase}] {action}")
|
|
2993
|
+
return terminal
|
|
2994
|
+
|
|
2995
|
+
def _terminalize_provider_invocation_error(
|
|
2996
|
+
self, state: TransactionState, error: CodexInvocationError
|
|
2997
|
+
) -> TransactionState:
|
|
2998
|
+
"""Turn provider-proven interruption into one durable terminal truth."""
|
|
2999
|
+
if error.provider_turn_interrupted:
|
|
3000
|
+
return self._save_terminal(
|
|
3001
|
+
state,
|
|
3002
|
+
"FAILED",
|
|
3003
|
+
"NONE",
|
|
3004
|
+
str(error),
|
|
3005
|
+
terminal_condition="provider_turn_interrupted",
|
|
3006
|
+
)
|
|
3007
|
+
return self._save_terminal(
|
|
3008
|
+
state,
|
|
3009
|
+
"BLOCKED",
|
|
3010
|
+
error.next_action,
|
|
3011
|
+
str(error),
|
|
3012
|
+
terminal_condition=error.terminal_condition,
|
|
3013
|
+
)
|
|
3014
|
+
|
|
3015
|
+
def _save_operator_merge_wait(self, state: TransactionState) -> TransactionState:
|
|
3016
|
+
"""Persist a PR hand-off and release the foreground lease.
|
|
3017
|
+
|
|
3018
|
+
The wait is deliberately durable, but there is no running agent to
|
|
3019
|
+
own a liveness lease while the human reviews or merges the pull
|
|
3020
|
+
request. The watcher recognises this checkpoint as queue-owning.
|
|
3021
|
+
"""
|
|
3022
|
+
self.store.save(state)
|
|
3023
|
+
if self.active_lease is not None and self.active_lease.run_id == state.run_id:
|
|
3024
|
+
if self.lease_heartbeat is not None:
|
|
3025
|
+
self.active_lease = self.lease_heartbeat.stop()
|
|
3026
|
+
self.lease_heartbeat = None
|
|
3027
|
+
release_lease(self.root, self.active_lease, central_database=self.store.central_database)
|
|
3028
|
+
self.active_lease = None
|
|
3029
|
+
write_live_status(self.root, state, state.next_action)
|
|
3030
|
+
return state
|
|
3031
|
+
|
|
3032
|
+
def _save_post_merge_sync_wait(
|
|
3033
|
+
self, state: TransactionState, diagnostic: str
|
|
3034
|
+
) -> TransactionState:
|
|
3035
|
+
"""Keep post-merge closure resumable when the shared checkout is busy.
|
|
3036
|
+
|
|
3037
|
+
A dirty or non-main checkout is never force-cleaned. The verified
|
|
3038
|
+
merge remains durable evidence and the watcher retries the same
|
|
3039
|
+
transaction through its normal merge-poll path once the checkout is
|
|
3040
|
+
available again.
|
|
3041
|
+
"""
|
|
3042
|
+
waiting = replace(
|
|
3043
|
+
state,
|
|
3044
|
+
phase="WAIT_FOR_OPERATOR_MERGE",
|
|
3045
|
+
terminal=False,
|
|
3046
|
+
next_action="await_clean_synchronized_main",
|
|
3047
|
+
terminal_condition="post_merge_workspace_sync_required",
|
|
3048
|
+
diagnostic=redact_diagnostic(diagnostic),
|
|
3049
|
+
)
|
|
3050
|
+
return self._save_operator_merge_wait(waiting)
|
|
3051
|
+
|
|
3052
|
+
def _cleanup(self, state: TransactionState) -> TransactionState:
|
|
3053
|
+
print("[REPOSITORY_CLEANUP] Repository cleanup in progress")
|
|
3054
|
+
self._managed_action(state, "RECONCILIATION")
|
|
3055
|
+
self._managed_action(state, "CLEANUP")
|
|
3056
|
+
cleanup = self._start_phase(state.run_id, "REPOSITORY_CLEANUP")
|
|
3057
|
+
try:
|
|
3058
|
+
result = self.finalization.cleanup(
|
|
3059
|
+
root=self.root,
|
|
3060
|
+
store=self.store,
|
|
3061
|
+
repository=self.repository,
|
|
3062
|
+
state=state,
|
|
3063
|
+
save_terminal=self._save_terminal,
|
|
3064
|
+
)
|
|
3065
|
+
except Exception:
|
|
3066
|
+
complete_phase(self.root, cleanup, outcome="FAILED")
|
|
3067
|
+
raise
|
|
3068
|
+
complete_phase(self.root, cleanup, outcome="COMPLETE" if result.phase == "COMPLETE" else "FAILED")
|
|
3069
|
+
return result
|
|
3070
|
+
|
|
3071
|
+
def _record_merged_evidence(
|
|
3072
|
+
self, state: TransactionState, pr: PullRequestEvidence, evidence: RepositoryEvidence
|
|
3073
|
+
) -> TransactionState:
|
|
3074
|
+
common = {
|
|
3075
|
+
"last_verified_sha": evidence.head_sha,
|
|
3076
|
+
"latest_repository_evidence": _repository_summary(evidence),
|
|
3077
|
+
"latest_github_evidence": _pull_request_summary(pr),
|
|
3078
|
+
}
|
|
3079
|
+
if state.transaction_kind == "IMPLEMENTATION":
|
|
3080
|
+
recorded = replace(
|
|
3081
|
+
state,
|
|
3082
|
+
implementation_branch=state.branch,
|
|
3083
|
+
implementation_pull_request=pr.number,
|
|
3084
|
+
implementation_head_sha=state.last_verified_sha,
|
|
3085
|
+
implementation_merge_commit=pr.merge_commit,
|
|
3086
|
+
**common,
|
|
3087
|
+
)
|
|
3088
|
+
description = "implementation_merge_verified"
|
|
3089
|
+
elif state.transaction_kind == "RECONCILIATION":
|
|
3090
|
+
recorded = replace(state, reconciliation_pull_request=pr.number, **common)
|
|
3091
|
+
description = "reconciliation_merge_verified"
|
|
3092
|
+
else:
|
|
3093
|
+
recorded = replace(
|
|
3094
|
+
state,
|
|
3095
|
+
finalization_branch=state.branch or state.finalization_branch,
|
|
3096
|
+
finalization_pull_request=pr.number,
|
|
3097
|
+
finalization_head_sha=state.last_verified_sha,
|
|
3098
|
+
finalization_merge_commit=pr.merge_commit,
|
|
3099
|
+
**common,
|
|
3100
|
+
)
|
|
3101
|
+
description = "finalization_merge_verified"
|
|
3102
|
+
return self._append_verified_commit_evidence(
|
|
3103
|
+
recorded,
|
|
3104
|
+
phase="WAIT_FOR_OPERATOR_MERGE",
|
|
3105
|
+
commit_sha=pr.merge_commit or "",
|
|
3106
|
+
description=description,
|
|
3107
|
+
)
|
|
3108
|
+
|
|
3109
|
+
|
|
3110
|
+
def build_parser() -> argparse.ArgumentParser:
|
|
3111
|
+
parser = argparse.ArgumentParser(
|
|
3112
|
+
prog="engineering-execution-host",
|
|
3113
|
+
description="Run one bounded Engineering Platform execution transaction",
|
|
3114
|
+
)
|
|
3115
|
+
parser.add_argument("prompt", type=Path)
|
|
3116
|
+
parser.add_argument("--run-id")
|
|
3117
|
+
parser.add_argument(
|
|
3118
|
+
"--central-database", type=Path,
|
|
3119
|
+
help="installation-owned engineering.db selected by the lifecycle composition root",
|
|
3120
|
+
)
|
|
3121
|
+
parser.add_argument(
|
|
3122
|
+
"--transaction-kind",
|
|
3123
|
+
choices=("IMPLEMENTATION", "FINALIZATION", "RECONCILIATION"),
|
|
3124
|
+
default="IMPLEMENTATION",
|
|
3125
|
+
help="internal watcher-selected transaction kind",
|
|
3126
|
+
)
|
|
3127
|
+
parser.add_argument(
|
|
3128
|
+
"--admitted-storage-schema",
|
|
3129
|
+
type=int,
|
|
3130
|
+
help="storage schema admitted by the watcher that spawned this run",
|
|
3131
|
+
)
|
|
3132
|
+
parser.add_argument("--resume", action="store_true")
|
|
3133
|
+
parser.add_argument(
|
|
3134
|
+
"--owner-authorized",
|
|
3135
|
+
action="store_true",
|
|
3136
|
+
help="record the owner's bounded branch and pull-request authorization; merges remain operator-owned",
|
|
3137
|
+
)
|
|
3138
|
+
return parser
|
|
3139
|
+
|
|
3140
|
+
|
|
3141
|
+
def main(argv: list[str] | None = None) -> int:
|
|
3142
|
+
raw_args = argv if argv is not None else __import__("sys").argv[1:]
|
|
3143
|
+
root = Path.cwd().resolve()
|
|
3144
|
+
runtime_workspace(root)
|
|
3145
|
+
if raw_args == ["status"]:
|
|
3146
|
+
return print_live_status(root)
|
|
3147
|
+
if raw_args == ["qualify"]:
|
|
3148
|
+
report = execute_qualification(root)
|
|
3149
|
+
print(dashboard(report))
|
|
3150
|
+
return 0 if report["qualification"] == "PASS" else 1
|
|
3151
|
+
args = build_parser().parse_args(raw_args)
|
|
3152
|
+
central_database = args.central_database.resolve() if args.central_database is not None else None
|
|
3153
|
+
if central_database is None:
|
|
3154
|
+
raise SystemExit("CENTRAL_OPERATIONAL_DATABASE_REQUIRED")
|
|
3155
|
+
if central_database is not None and central_database.name != "engineering.db":
|
|
3156
|
+
raise SystemExit("--central-database must name engineering.db")
|
|
3157
|
+
if central_database is not None and not central_database.is_file():
|
|
3158
|
+
raise SystemExit("--central-database does not exist")
|
|
3159
|
+
prompt_path = args.prompt.resolve()
|
|
3160
|
+
if not prompt_path.is_file():
|
|
3161
|
+
raise SystemExit(f"prompt does not exist: {prompt_path}")
|
|
3162
|
+
if args.resume and not args.run_id:
|
|
3163
|
+
raise SystemExit("--resume requires --run-id")
|
|
3164
|
+
if args.admitted_storage_schema is not None and args.admitted_storage_schema < 1:
|
|
3165
|
+
raise SystemExit("--admitted-storage-schema must be positive")
|
|
3166
|
+
if args.admitted_storage_schema is not None:
|
|
3167
|
+
# Keep the watcher admission boundary with every child process the
|
|
3168
|
+
# runner starts. Source may change during execution, but the canonical
|
|
3169
|
+
# live database must remain readable by the admitting components.
|
|
3170
|
+
os.environ["ENGINEERING_PLATFORM_ADMITTED_STORAGE_SCHEMA"] = str(args.admitted_storage_schema)
|
|
3171
|
+
os.environ["ENGINEERING_PLATFORM_ADMITTED_STORAGE_ROOT"] = str(root)
|
|
3172
|
+
compatibility = (
|
|
3173
|
+
RunnerCompatibility(storage_schemas=frozenset({args.admitted_storage_schema}))
|
|
3174
|
+
if args.admitted_storage_schema is not None
|
|
3175
|
+
else RunnerCompatibility()
|
|
3176
|
+
)
|
|
3177
|
+
try:
|
|
3178
|
+
runtime = PlatformConfiguration.load(root).resolver(root).resolve_runtime()
|
|
3179
|
+
except PlatformConfigurationError:
|
|
3180
|
+
runtime = None
|
|
3181
|
+
runner = EngineeringRunner(
|
|
3182
|
+
root,
|
|
3183
|
+
StateStore(
|
|
3184
|
+
root / ".engineering" / "engineering-runs",
|
|
3185
|
+
central_database=central_database,
|
|
3186
|
+
emit_local_projection=False,
|
|
3187
|
+
),
|
|
3188
|
+
SubprocessRepositoryClient(),
|
|
3189
|
+
GhCliClient(),
|
|
3190
|
+
CodexCliClient(CodexCliProvider(str(runtime)) if runtime is not None else CodexCliProvider()),
|
|
3191
|
+
compatibility=compatibility,
|
|
3192
|
+
)
|
|
3193
|
+
# Execution is hosted by the canonical lifecycle worker; it must not
|
|
3194
|
+
# create an ad-hoc component identity in CENTRAL operational logs.
|
|
3195
|
+
logger = component_logger(root, "lifecycle_worker", central_database=central_database)
|
|
3196
|
+
lifecycle_context = {"application_version": CURRENT_PLATFORM_VERSION, "target_component": "lifecycle_worker"}
|
|
3197
|
+
try:
|
|
3198
|
+
with shutdown_signal_logging(logger, lifecycle_context):
|
|
3199
|
+
state = runner.run(
|
|
3200
|
+
prompt_path,
|
|
3201
|
+
args.run_id,
|
|
3202
|
+
args.resume,
|
|
3203
|
+
args.owner_authorized,
|
|
3204
|
+
args.transaction_kind,
|
|
3205
|
+
)
|
|
3206
|
+
except (RunnerError, StateError) as error:
|
|
3207
|
+
print(f"BLOCKED: {error}")
|
|
3208
|
+
return 2
|
|
3209
|
+
report_phase = (
|
|
3210
|
+
start_phase(root, state.run_id, "REPORT_GENERATION", central_database=central_database)
|
|
3211
|
+
if state.terminal else None
|
|
3212
|
+
)
|
|
3213
|
+
try:
|
|
3214
|
+
report_path = (
|
|
3215
|
+
generate_terminal_report(
|
|
3216
|
+
root,
|
|
3217
|
+
state,
|
|
3218
|
+
runner.platform_manifest,
|
|
3219
|
+
runner.detected_codex_cli,
|
|
3220
|
+
runner.reviewer_records,
|
|
3221
|
+
getattr(runner.agent, "last_runtime_metadata", None),
|
|
3222
|
+
getattr(runner.agent, "last_execution_metadata", None),
|
|
3223
|
+
central_database=central_database,
|
|
3224
|
+
)
|
|
3225
|
+
if state.terminal
|
|
3226
|
+
else None
|
|
3227
|
+
)
|
|
3228
|
+
except Exception:
|
|
3229
|
+
if report_phase is not None:
|
|
3230
|
+
complete_phase(root, report_phase, outcome="FAILED")
|
|
3231
|
+
raise
|
|
3232
|
+
if report_phase is not None:
|
|
3233
|
+
complete_phase(root, report_phase)
|
|
3234
|
+
if report_path:
|
|
3235
|
+
evidence_phase = start_phase(
|
|
3236
|
+
root, state.run_id, "EVIDENCE_PERSISTENCE", central_database=central_database,
|
|
3237
|
+
)
|
|
3238
|
+
try:
|
|
3239
|
+
record_terminal_report(root, report_path, central_database=central_database)
|
|
3240
|
+
analyze_terminal_report(root, state.run_id, report_path)
|
|
3241
|
+
except Exception:
|
|
3242
|
+
complete_phase(root, evidence_phase, outcome="FAILED")
|
|
3243
|
+
raise
|
|
3244
|
+
complete_phase(root, evidence_phase)
|
|
3245
|
+
if runner.platform_manifest:
|
|
3246
|
+
publish_canonical_status(
|
|
3247
|
+
root / ".engineering" / "status",
|
|
3248
|
+
build_canonical_status(
|
|
3249
|
+
runner.platform_manifest,
|
|
3250
|
+
current_phase=state.phase,
|
|
3251
|
+
current_action=state.next_action,
|
|
3252
|
+
run_id=state.run_id,
|
|
3253
|
+
repair_iteration=state.repair_iterations,
|
|
3254
|
+
implementation_pr=state.implementation_pull_request,
|
|
3255
|
+
finalization_pr=state.finalization_pull_request,
|
|
3256
|
+
repository_state="MERGED_RECONCILED" if state.phase == "COMPLETE" else "ACTIVE",
|
|
3257
|
+
workspace_state="WORKSPACE_READY" if state.phase == "COMPLETE" else "ACTIVE",
|
|
3258
|
+
owner_authorized=state.owner_authorized,
|
|
3259
|
+
resume_available=not state.terminal,
|
|
3260
|
+
latest_report=str(report_path) if report_path else None,
|
|
3261
|
+
diagnostic=state.diagnostic,
|
|
3262
|
+
),
|
|
3263
|
+
)
|
|
3264
|
+
if report_path:
|
|
3265
|
+
print(
|
|
3266
|
+
f"Engineering report generated:\n\n{report_path}\n\nAvailable in the Engineering Status dashboard."
|
|
3267
|
+
)
|
|
3268
|
+
if state.phase in {"BLOCKED", "FAILED"}:
|
|
3269
|
+
print(_format_terminal_report(state))
|
|
3270
|
+
if runner.console_detail:
|
|
3271
|
+
record_redacted_codex_cli_diagnostic(
|
|
3272
|
+
root, state.run_id, runner.console_detail, central_database=central_database,
|
|
3273
|
+
)
|
|
3274
|
+
print("\nCodex CLI-diagnostiek is veilig in CENTRAL vastgelegd.")
|
|
3275
|
+
print(f"\nCodex CLI details:\n{runner.console_detail}")
|
|
3276
|
+
elif state.phase == "COMPLETE" and state.owner_authorized and state.finalization_merge_commit:
|
|
3277
|
+
print(format_management_summary(state))
|
|
3278
|
+
else:
|
|
3279
|
+
print(json.dumps(state.to_dict(), indent=2, sort_keys=True))
|
|
3280
|
+
# A watcher owns the outer envelope until it archives and persists its
|
|
3281
|
+
# evidence. Direct invocations own that final boundary themselves.
|
|
3282
|
+
if args.admitted_storage_schema is None and state.terminal:
|
|
3283
|
+
complete_active_phase(
|
|
3284
|
+
root, state.run_id, "TOTAL_EXECUTION",
|
|
3285
|
+
outcome="COMPLETE" if state.phase == "COMPLETE" else "FAILED",
|
|
3286
|
+
central_database=central_database,
|
|
3287
|
+
)
|
|
3288
|
+
return 0 if state.phase == "COMPLETE" else 1
|
|
3289
|
+
|
|
3290
|
+
|
|
3291
|
+
# Reporting compatibility exports are implemented in execution_reporting.py.
|
|
3292
|
+
from .execution_reporting import ( # noqa: E402, F401
|
|
3293
|
+
_format_engineering_outcome,
|
|
3294
|
+
_format_reviewer_records,
|
|
3295
|
+
_format_terminal_report,
|
|
3296
|
+
_next_action_message,
|
|
3297
|
+
_pull_request_summary,
|
|
3298
|
+
_repository_summary,
|
|
3299
|
+
collect_terminal_evidence,
|
|
3300
|
+
corrected_terminal_report,
|
|
3301
|
+
format_management_summary,
|
|
3302
|
+
format_terminal_management_summary,
|
|
3303
|
+
generate_terminal_report,
|
|
3304
|
+
report_consistency_errors,
|
|
3305
|
+
terminal_report_matches_state,
|
|
3306
|
+
)
|