froid-loop 0.11.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- froid_loop/__init__.py +11 -0
- froid_loop/__main__.py +12 -0
- froid_loop/adapters/__init__.py +3 -0
- froid_loop/adapters/base.py +254 -0
- froid_loop/adapters/entrypoints.py +63 -0
- froid_loop/adapters/env_fault.py +290 -0
- froid_loop/adapters/generic.py +2013 -0
- froid_loop/adapters/mock.py +49 -0
- froid_loop/adapters/multiplexer.py +914 -0
- froid_loop/adapters/opencode_http.py +1687 -0
- froid_loop/adapters/profile.py +650 -0
- froid_loop/adapters/psmux_backend.py +1428 -0
- froid_loop/adapters/registry.py +322 -0
- froid_loop/adapters/tmux_backend.py +35 -0
- froid_loop/adapters/tmux_base.py +630 -0
- froid_loop/checks.py +187 -0
- froid_loop/cli.py +5041 -0
- froid_loop/data/__init__.py +0 -0
- froid_loop/data/froid_loop_hook.py +228 -0
- froid_loop/data/froid_loop_probe_hook.py +88 -0
- froid_loop/data/plugins/example/plugin.toml +21 -0
- froid_loop/data/plugins/tea/plugin.toml +184 -0
- froid_loop/data/plugins/tea/tea_plugin.py +258 -0
- froid_loop/data/plugins/unity/plugin.toml +140 -0
- froid_loop/data/plugins/unity/unity_assets/FroidLoop.Unity.Editor.asmdef +16 -0
- froid_loop/data/plugins/unity/unity_assets/FroidLoop.Unity.Editor.asmdef.meta +7 -0
- froid_loop/data/plugins/unity/unity_assets/SceneAutoSaveGuard.cs +221 -0
- froid_loop/data/plugins/unity/unity_assets/SceneAutoSaveGuard.cs.meta +11 -0
- froid_loop/data/plugins/unity/unity_assets/_folders/Editor.meta +8 -0
- froid_loop/data/plugins/unity/unity_assets/_folders/FroidLoop.meta +8 -0
- froid_loop/data/plugins/unity/unity_cleanup.py +125 -0
- froid_loop/data/plugins/unity/unity_dialog_probe.py +239 -0
- froid_loop/data/plugins/unity/unity_facts.md +17 -0
- froid_loop/data/plugins/unity/unity_plugin.py +415 -0
- froid_loop/data/plugins/unity/unity_quiesce.py +234 -0
- froid_loop/data/plugins/unity/unity_ready.py +230 -0
- froid_loop/data/plugins/unity/unity_seed_assets.py +298 -0
- froid_loop/data/plugins/unity/unity_setup.py +551 -0
- froid_loop/data/plugins/unity/unity_teardown.py +362 -0
- froid_loop/data/profiles/antigravity.toml +52 -0
- froid_loop/data/profiles/claude.toml +85 -0
- froid_loop/data/profiles/codex.toml +22 -0
- froid_loop/data/profiles/copilot.toml +52 -0
- froid_loop/data/profiles/gemini.toml +26 -0
- froid_loop/data/profiles/opencode.toml +54 -0
- froid_loop/data/settings/core.toml +458 -0
- froid_loop/data/skills/README.md +93 -0
- froid_loop/data/skills/froid-loop-resolve/SKILL.md +288 -0
- froid_loop/data/skills/froid-loop-setup/SKILL.md +161 -0
- froid_loop/data/skills/froid-loop-setup/assets/module-help.csv +3 -0
- froid_loop/data/skills/froid-loop-setup/assets/module.yaml +19 -0
- froid_loop/data/skills/froid-loop-sweep/SKILL.md +100 -0
- froid_loop/data/skills/froid-loop-sweep/automation-mode.md +127 -0
- froid_loop/data/skills/froid-loop-sweep/deferred-work-format.md +302 -0
- froid_loop/data/skills/froid-loop-sweep/migration-mode.md +86 -0
- froid_loop/decisions.py +202 -0
- froid_loop/deferredwork.py +2282 -0
- froid_loop/devcontract.py +892 -0
- froid_loop/diagnostics.py +1104 -0
- froid_loop/documents.py +532 -0
- froid_loop/engine.py +7732 -0
- froid_loop/envvars.py +111 -0
- froid_loop/escalation.py +225 -0
- froid_loop/events.py +266 -0
- froid_loop/fences.py +103 -0
- froid_loop/froidconfig.py +226 -0
- froid_loop/frontmatter.py +526 -0
- froid_loop/gates.py +133 -0
- froid_loop/install.py +2936 -0
- froid_loop/journal.py +178 -0
- froid_loop/machine.py +148 -0
- froid_loop/model.py +898 -0
- froid_loop/operatoractions.py +474 -0
- froid_loop/platform_util.py +1490 -0
- froid_loop/plugins/__init__.py +64 -0
- froid_loop/plugins/bus.py +259 -0
- froid_loop/plugins/context.py +319 -0
- froid_loop/plugins/loader.py +145 -0
- froid_loop/plugins/manifest.py +279 -0
- froid_loop/plugins/model.py +296 -0
- froid_loop/plugins/registry.py +245 -0
- froid_loop/plugins/trust.py +75 -0
- froid_loop/policy.py +1569 -0
- froid_loop/probe.py +1044 -0
- froid_loop/process_host.py +408 -0
- froid_loop/recovery_flow.py +1561 -0
- froid_loop/resolve.py +283 -0
- froid_loop/runs.py +4715 -0
- froid_loop/runsetup.py +1293 -0
- froid_loop/sanitize.py +593 -0
- froid_loop/settings_schema.py +276 -0
- froid_loop/signals.py +160 -0
- froid_loop/sprintstatus.py +609 -0
- froid_loop/statemachine.py +57 -0
- froid_loop/stories.py +615 -0
- froid_loop/stories_engine.py +796 -0
- froid_loop/sweep.py +1892 -0
- froid_loop/tokens.py +196 -0
- froid_loop/tui/__init__.py +11 -0
- froid_loop/tui/app.py +1584 -0
- froid_loop/tui/data.py +840 -0
- froid_loop/tui/launch.py +1003 -0
- froid_loop/tui/screens/__init__.py +1 -0
- froid_loop/tui/screens/dashboard.py +1071 -0
- froid_loop/tui/screens/modals.py +943 -0
- froid_loop/tui/screens/settings_screen.py +477 -0
- froid_loop/tui/settings.py +135 -0
- froid_loop/tui/widgets.py +981 -0
- froid_loop/verify.py +4545 -0
- froid_loop/workspace.py +320 -0
- froid_loop/worktree_flow.py +2301 -0
- froid_loop-0.11.1.dist-info/METADATA +728 -0
- froid_loop-0.11.1.dist-info/RECORD +116 -0
- froid_loop-0.11.1.dist-info/WHEEL +4 -0
- froid_loop-0.11.1.dist-info/entry_points.txt +2 -0
- froid_loop-0.11.1.dist-info/licenses/LICENSE +30 -0
|
@@ -0,0 +1,1561 @@
|
|
|
1
|
+
"""Attempt rollback + recovery-ref preservation flow.
|
|
2
|
+
|
|
3
|
+
Extracted from :class:`froid_loop.engine.Engine` (issue #244, PR 2/2): the
|
|
4
|
+
rollback/preserve cluster is an independent recovery state machine. It lives
|
|
5
|
+
here as a collaborator built from narrow dependencies (repo paths, the policy,
|
|
6
|
+
run state, journal, run dir) plus a getter for the engine's swappable active
|
|
7
|
+
workspace and a handful of engine callbacks (emit a plugin hook, save state,
|
|
8
|
+
escalate a task, and escalation-pause). The collaborator never receives the
|
|
9
|
+
whole ``Engine`` — it cannot reach engine internals beyond those callables.
|
|
10
|
+
|
|
11
|
+
``Engine`` keeps same-name private methods that delegate here, so its tests and
|
|
12
|
+
the ``SweepEngine``/``StoriesEngine`` subclasses (which override nothing in this
|
|
13
|
+
cluster) see an unchanged surface.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import TYPE_CHECKING, Callable, NoReturn
|
|
20
|
+
|
|
21
|
+
from . import gates, verify
|
|
22
|
+
from .model import Phase
|
|
23
|
+
from .platform_util import atomic_write_bytes, safe_ref_segment
|
|
24
|
+
from .statemachine import advance
|
|
25
|
+
|
|
26
|
+
if TYPE_CHECKING:
|
|
27
|
+
from .froidconfig import ProjectPaths
|
|
28
|
+
from .journal import Journal
|
|
29
|
+
from .model import RunState, StoryTask
|
|
30
|
+
from .policy import Policy
|
|
31
|
+
from .workspace import Workspace
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
# How many candidate `refs/attempt-preserve-dirty/*` names one rollback may probe
|
|
35
|
+
# before giving up (the base name plus -r2..-r100). Deliberately not a policy
|
|
36
|
+
# field: it is a runaway backstop, not a tuning knob — `scm.preserve_keep`
|
|
37
|
+
# (default 20) already prunes this namespace at every run start, so needing even
|
|
38
|
+
# a dozen candidates for ONE {slug}-{baseline}-{attempt} triple means something
|
|
39
|
+
# upstream is wrong. The cost of the bound is one git spawn per candidate on a
|
|
40
|
+
# path that only runs while a crashed attempt is being rolled back.
|
|
41
|
+
PRESERVE_REF_PROBE_LIMIT = 100
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _OwnedSpecAuthorityError(RuntimeError):
|
|
45
|
+
"""A previously canonical owned-spec name became unsafe to restore."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class RecoveryFlow:
|
|
49
|
+
"""Roll back or pause a stopped/abandoned attempt, parking any work it did on
|
|
50
|
+
named recovery refs before the reset.
|
|
51
|
+
|
|
52
|
+
Built once per engine from narrow deps + engine callbacks (see module
|
|
53
|
+
docstring). Behavior is identical to the cluster it was carved out of; the
|
|
54
|
+
only structural changes are that engine-owned effects go through injected
|
|
55
|
+
callables: ``emit`` fires a plugin hook (late-bound so a monkeypatched
|
|
56
|
+
``Engine._emit`` still wins), ``save`` persists run state, ``escalate``
|
|
57
|
+
routes an intent-gap restore failure through the engine's escalation, and
|
|
58
|
+
``escalation_pause`` raises the engine's ``RunPaused`` (injected so this
|
|
59
|
+
module need not import ``engine`` — that would reintroduce a runtime<->engine
|
|
60
|
+
import cycle). ``workspace_get`` reads the engine's live (worktree-swappable)
|
|
61
|
+
active workspace."""
|
|
62
|
+
|
|
63
|
+
def __init__(
|
|
64
|
+
self,
|
|
65
|
+
*,
|
|
66
|
+
paths: ProjectPaths,
|
|
67
|
+
policy: Policy,
|
|
68
|
+
state: RunState,
|
|
69
|
+
journal: Journal,
|
|
70
|
+
run_dir: Path,
|
|
71
|
+
workspace_get: Callable[[], Workspace],
|
|
72
|
+
emit: Callable[..., object],
|
|
73
|
+
save: Callable[[], None],
|
|
74
|
+
escalate: Callable[[StoryTask, str], None],
|
|
75
|
+
escalation_pause: Callable[..., NoReturn],
|
|
76
|
+
) -> None:
|
|
77
|
+
self.paths = paths
|
|
78
|
+
self.policy = policy
|
|
79
|
+
self.state = state
|
|
80
|
+
self.journal = journal
|
|
81
|
+
self.run_dir = run_dir
|
|
82
|
+
# Read live (a getter, not a captured ref) so a run that swaps the
|
|
83
|
+
# engine's `self.workspace` to a mounted unit worktree is seen here.
|
|
84
|
+
self._workspace_get = workspace_get
|
|
85
|
+
# Injected late-bound so a test patching `engine._emit` still wins
|
|
86
|
+
# (recovery_flow's own binding wouldn't).
|
|
87
|
+
self._emit = emit
|
|
88
|
+
self._save = save
|
|
89
|
+
self._escalate = escalate
|
|
90
|
+
self._pause = escalation_pause
|
|
91
|
+
|
|
92
|
+
def protected_relpaths(self) -> tuple[str, ...]:
|
|
93
|
+
"""Repo-relative posix paths of the FROID artifact folders. These are
|
|
94
|
+
preserved through a resolved re-drive's reset so a human correction is
|
|
95
|
+
not reverted. They are deliberately not attempt-dirtiness exclusions;
|
|
96
|
+
only the exact attempt-bound spec can serve that separate recognition
|
|
97
|
+
job. Folders configured outside the repo are skipped — nothing to
|
|
98
|
+
preserve through Git there."""
|
|
99
|
+
workspace = self._workspace_get()
|
|
100
|
+
out: list[str] = []
|
|
101
|
+
for protected in (
|
|
102
|
+
workspace.paths.output_folder,
|
|
103
|
+
workspace.paths.implementation_artifacts,
|
|
104
|
+
workspace.paths.planning_artifacts,
|
|
105
|
+
):
|
|
106
|
+
try:
|
|
107
|
+
rel = protected.relative_to(workspace.root).as_posix()
|
|
108
|
+
except ValueError:
|
|
109
|
+
continue # configured outside the repo; nothing to protect here
|
|
110
|
+
# "." (folder == repo root) as a keep/preserve prefix would cover the
|
|
111
|
+
# whole tree — drop it so a misconfig can't disable the reset.
|
|
112
|
+
if rel and rel != ".":
|
|
113
|
+
out.append(rel)
|
|
114
|
+
return tuple(out)
|
|
115
|
+
|
|
116
|
+
def _attempt_owned_spec(self, task: StoryTask) -> tuple[Path, str | None] | None:
|
|
117
|
+
"""Resolve this attempt's bound spec and its exact Git exclusion.
|
|
118
|
+
|
|
119
|
+
Relative persisted paths may name either a project-relative file or a
|
|
120
|
+
basename under the configured implementation-artifacts directory. The
|
|
121
|
+
binding is usable only when exactly one such regular file exists and its
|
|
122
|
+
resolved target is inside a trusted project/artifact root. An artifact
|
|
123
|
+
root configured outside the Git workspace remains a trusted repair target,
|
|
124
|
+
but cannot contribute a pathspec to a Git command running in the workspace.
|
|
125
|
+
"""
|
|
126
|
+
if not task.dispatched_spec_file:
|
|
127
|
+
return None
|
|
128
|
+
|
|
129
|
+
workspace = self._workspace_get()
|
|
130
|
+
raw = Path(task.dispatched_spec_file)
|
|
131
|
+
candidates = (
|
|
132
|
+
(raw,)
|
|
133
|
+
if raw.is_absolute()
|
|
134
|
+
else (
|
|
135
|
+
workspace.paths.project / raw,
|
|
136
|
+
workspace.paths.implementation_artifacts / raw,
|
|
137
|
+
)
|
|
138
|
+
)
|
|
139
|
+
resolved_files: list[Path] = []
|
|
140
|
+
for candidate in candidates:
|
|
141
|
+
try:
|
|
142
|
+
# New attempts persist a canonical regular-file path. Refuse a
|
|
143
|
+
# post-launch symlink replacement before resolving it: following
|
|
144
|
+
# the link here would let a failed child retarget snapshot restore
|
|
145
|
+
# into an unrelated file that happens to share a trusted root.
|
|
146
|
+
if candidate.is_symlink():
|
|
147
|
+
return None
|
|
148
|
+
resolved = candidate.resolve()
|
|
149
|
+
if raw.is_absolute() and resolved != candidate:
|
|
150
|
+
# Snapshot-bearing bindings are persisted canonically. A
|
|
151
|
+
# changed result here means a parent component was replaced
|
|
152
|
+
# by a symlink after launch, which is the same retargeting
|
|
153
|
+
# hazard as a link at the final component.
|
|
154
|
+
return None
|
|
155
|
+
is_file = resolved.is_file()
|
|
156
|
+
except (OSError, RuntimeError):
|
|
157
|
+
return None
|
|
158
|
+
if is_file and resolved not in resolved_files:
|
|
159
|
+
resolved_files.append(resolved)
|
|
160
|
+
if len(resolved_files) != 1:
|
|
161
|
+
return None
|
|
162
|
+
|
|
163
|
+
spec_path = resolved_files[0]
|
|
164
|
+
if not verify.spec_within_roots(spec_path, workspace.paths):
|
|
165
|
+
return None
|
|
166
|
+
|
|
167
|
+
try:
|
|
168
|
+
rel = spec_path.relative_to(workspace.root.resolve()).as_posix()
|
|
169
|
+
except (OSError, RuntimeError, ValueError):
|
|
170
|
+
return spec_path, None
|
|
171
|
+
return spec_path, rel if rel and rel != "." else None
|
|
172
|
+
|
|
173
|
+
@staticmethod
|
|
174
|
+
def _normalize_attempt_owned_spec(
|
|
175
|
+
spec_path: Path, target_status: str, *, confine_root: Path
|
|
176
|
+
) -> None:
|
|
177
|
+
"""Write and verify the lifecycle route recovery promises to dispatch.
|
|
178
|
+
|
|
179
|
+
``confine_root`` is the project that owns the binding
|
|
180
|
+
(``workspace.paths.project`` — the same root `_attempt_owned_spec`
|
|
181
|
+
resolves candidates under), threaded down rather than re-derived here:
|
|
182
|
+
this is a staticmethod on purpose, and `_workspace_get` is a live getter
|
|
183
|
+
precisely because a unit worktree swaps the root mid-run (``rebased``
|
|
184
|
+
makes ``paths.project`` the worktree root there). It must NOT be
|
|
185
|
+
``workspace.root``: under the `repo_root` override that is the separate
|
|
186
|
+
code repo, an in-project spec fails its `is_relative_to` test, and the
|
|
187
|
+
chokepoint silently takes the plain arm — dropping the parent walk the
|
|
188
|
+
confinement exists for. It reaches the spec-writer chokepoint rule
|
|
189
|
+
stated in `frontmatter.set_frontmatter_status` — an artifacts folder
|
|
190
|
+
configured outside the project is a trusted repair target here
|
|
191
|
+
(`_attempt_owned_spec`) and keeps the plain no-follow write."""
|
|
192
|
+
verify.set_frontmatter_status(spec_path, target_status, confine_root=confine_root)
|
|
193
|
+
if verify.status_of(verify.read_frontmatter(spec_path)) != target_status:
|
|
194
|
+
raise verify.FrontmatterWriteError(
|
|
195
|
+
f"could not normalize attempt-owned spec {spec_path} "
|
|
196
|
+
f"to status {target_status!r}"
|
|
197
|
+
)
|
|
198
|
+
|
|
199
|
+
@staticmethod
|
|
200
|
+
def _restore_attempt_owned_spec_bytes(spec_path: Path, snapshot: bytes) -> None:
|
|
201
|
+
"""Restore and verify the byte-exact pre-attempt input."""
|
|
202
|
+
try:
|
|
203
|
+
parent = spec_path.parent
|
|
204
|
+
existing_parent = parent
|
|
205
|
+
while not existing_parent.exists() and not existing_parent.is_symlink():
|
|
206
|
+
if existing_parent == existing_parent.parent:
|
|
207
|
+
break
|
|
208
|
+
existing_parent = existing_parent.parent
|
|
209
|
+
# `Path.is_absolute()` on purpose, NOT `platform_util.is_absolute_path`
|
|
210
|
+
# (#480 item 4). That family predicate is built for "must stay INSIDE
|
|
211
|
+
# the project" config guards, where the answer must not vary by host.
|
|
212
|
+
# This is the opposite question: a live path this process is about to
|
|
213
|
+
# `resolve(strict=True)` and write through on the host it is running on,
|
|
214
|
+
# so the platform's own notion of absolute is the operative one. They
|
|
215
|
+
# diverge in the direction that matters -- on Windows a POSIX-absolute
|
|
216
|
+
# `/spec.md` reads as NOT absolute, so this REFUSES it and fails CLOSED,
|
|
217
|
+
# while `is_absolute_path` answers True and would let it through. The
|
|
218
|
+
# swap #480 proposes would loosen the only genuine refusal guard it
|
|
219
|
+
# named. Measured on POSIX: the `resolve(strict=True)` fixed-point term
|
|
220
|
+
# below already refuses every relative spelling on its own (a relative
|
|
221
|
+
# path never equals its own resolve), so on this platform the term
|
|
222
|
+
# states the intent rather than carrying it alone -- which is exactly
|
|
223
|
+
# why it needs saying here.
|
|
224
|
+
if (
|
|
225
|
+
not spec_path.is_absolute()
|
|
226
|
+
or not existing_parent.is_dir()
|
|
227
|
+
or existing_parent.is_symlink()
|
|
228
|
+
or existing_parent.resolve(strict=True) != existing_parent
|
|
229
|
+
or spec_path.is_symlink()
|
|
230
|
+
):
|
|
231
|
+
raise _OwnedSpecAuthorityError(
|
|
232
|
+
f"attempt-owned spec target became unsafe: {spec_path}"
|
|
233
|
+
)
|
|
234
|
+
parent.mkdir(parents=True, exist_ok=True)
|
|
235
|
+
if not parent.is_dir() or parent.is_symlink() or parent.resolve(strict=True) != parent:
|
|
236
|
+
raise _OwnedSpecAuthorityError(
|
|
237
|
+
f"attempt-owned spec target became unsafe: {spec_path}"
|
|
238
|
+
)
|
|
239
|
+
if spec_path.exists() and (
|
|
240
|
+
not spec_path.is_file() or spec_path.resolve(strict=True) != spec_path
|
|
241
|
+
):
|
|
242
|
+
raise _OwnedSpecAuthorityError(
|
|
243
|
+
f"attempt-owned spec target became unsafe: {spec_path}"
|
|
244
|
+
)
|
|
245
|
+
except _OwnedSpecAuthorityError:
|
|
246
|
+
raise
|
|
247
|
+
except (OSError, RuntimeError) as exc:
|
|
248
|
+
raise _OwnedSpecAuthorityError(
|
|
249
|
+
f"attempt-owned spec target could not be revalidated: {spec_path}"
|
|
250
|
+
) from exc
|
|
251
|
+
# `require_writable_target=True` (#597): the spec this puts back is
|
|
252
|
+
# operator-editable, and a temp-and-replace write needs write permission on
|
|
253
|
+
# the PARENT DIRECTORY, never on the entry it replaces — so a spec marked
|
|
254
|
+
# read-only was rewritten anyway. NOT the confined writer: the authority
|
|
255
|
+
# walk above is stricter than the cohort walk (it demands
|
|
256
|
+
# `resolve(strict=True)` fixed points for the parent and the target, which
|
|
257
|
+
# refuses a link ANYWHERE above, inside the checkout as well as out), so a
|
|
258
|
+
# confined write would relax this site rather than harden it.
|
|
259
|
+
atomic_write_bytes(spec_path, snapshot, follow_symlinks=False, require_writable_target=True)
|
|
260
|
+
if spec_path.read_bytes() != snapshot:
|
|
261
|
+
raise verify.FrontmatterWriteError(
|
|
262
|
+
f"could not restore pre-attempt contents of owned spec {spec_path}"
|
|
263
|
+
)
|
|
264
|
+
|
|
265
|
+
@classmethod
|
|
266
|
+
def _restore_attempt_owned_spec(
|
|
267
|
+
cls,
|
|
268
|
+
spec_path: Path,
|
|
269
|
+
snapshot: bytes,
|
|
270
|
+
target_status: str,
|
|
271
|
+
*,
|
|
272
|
+
confine_root: Path,
|
|
273
|
+
) -> None:
|
|
274
|
+
"""Restore exact pre-attempt bytes, then verify the promised route."""
|
|
275
|
+
cls._restore_attempt_owned_spec_bytes(spec_path, snapshot)
|
|
276
|
+
# The durable snapshot should already carry this route. Keep the status
|
|
277
|
+
# repair as a fail-safe for a legacy or externally edited state record;
|
|
278
|
+
# it is the only permitted difference from the exact snapshot.
|
|
279
|
+
cls._normalize_attempt_owned_spec(spec_path, target_status, confine_root=confine_root)
|
|
280
|
+
|
|
281
|
+
def pause_for_owned_spec_recovery(
|
|
282
|
+
self,
|
|
283
|
+
task: StoryTask,
|
|
284
|
+
spec: str,
|
|
285
|
+
problem: str,
|
|
286
|
+
) -> NoReturn:
|
|
287
|
+
"""Pause once on unsafe snapshot authority, with a convergent remedy.
|
|
288
|
+
|
|
289
|
+
The current checkout may contain either operator intent, failed-child
|
|
290
|
+
output, or a partially completed reset, so recovery cannot infer which
|
|
291
|
+
paths are safe to mutate next. Clear the unusable authority pair before
|
|
292
|
+
saving: after the operator restores/verifies the intended spec, resume can
|
|
293
|
+
recover the remaining tree as an unbound redrive instead of repeating the
|
|
294
|
+
same impossible snapshot check forever.
|
|
295
|
+
"""
|
|
296
|
+
task.dispatched_spec_file = None
|
|
297
|
+
task.dispatched_spec_snapshot = None
|
|
298
|
+
root = self._workspace_get().root
|
|
299
|
+
notice = (
|
|
300
|
+
"**ACTION REQUIRED — attempt-owned spec needs manual recovery**\n"
|
|
301
|
+
f"Story **{task.story_key}** cannot safely restore its pre-attempt spec "
|
|
302
|
+
f"at `{spec}`: {problem}. The working tree at `{root}` now requires "
|
|
303
|
+
"inspection because froid-loop cannot safely distinguish operator "
|
|
304
|
+
"intent, failed-session output, and any rollback already completed.\n"
|
|
305
|
+
" 1. Save any failed-session work you may want to inspect.\n"
|
|
306
|
+
f" 2. Restore or verify the operator-approved contents of `{spec}` and "
|
|
307
|
+
"remove/reset any other rejected attempt residue.\n"
|
|
308
|
+
f" 3. Run `froid-loop resume {self.state.run_id}`. The unusable binding "
|
|
309
|
+
"was cleared, so the corrected spec will be observed afresh before the "
|
|
310
|
+
"next child launches."
|
|
311
|
+
)
|
|
312
|
+
self.journal.append(
|
|
313
|
+
"rollback-owned-spec-manual-required",
|
|
314
|
+
story_key=task.story_key,
|
|
315
|
+
spec=spec,
|
|
316
|
+
problem=problem,
|
|
317
|
+
)
|
|
318
|
+
gates.notify(
|
|
319
|
+
self.policy,
|
|
320
|
+
self.run_dir,
|
|
321
|
+
f"ACTION REQUIRED: recover attempt-owned spec for {task.story_key}",
|
|
322
|
+
notice,
|
|
323
|
+
)
|
|
324
|
+
self._save()
|
|
325
|
+
self._pause(notice, task.story_key)
|
|
326
|
+
|
|
327
|
+
def rollback_or_pause(self, task: StoryTask, *, cause: str = "stopped") -> None:
|
|
328
|
+
"""Recover from an attempt that won't proceed.
|
|
329
|
+
|
|
330
|
+
No-op when the real tree is proven to be at the attempt's baseline:
|
|
331
|
+
neither a reset nor a pause is needed, and an unchanged bound spec is
|
|
332
|
+
never rewritten. The one recognition exception is the exact regular file
|
|
333
|
+
bound in ``task.dispatched_spec_file`` for this attempt. If the real tree
|
|
334
|
+
is dirty but no debris remains after excluding that file, its lifecycle
|
|
335
|
+
status is normalized and the real checkout is probed again without
|
|
336
|
+
exclusions. A genuinely clean checkout with no commits above the attempt
|
|
337
|
+
baseline emits ``rollback-skipped-clean``. If a lifecycle-only attempt
|
|
338
|
+
committed its flip, recovery parks that commit and resets HEAD before
|
|
339
|
+
retrying instead of mistaking the baseline-shaped worktree for a clean
|
|
340
|
+
branch.
|
|
341
|
+
A resolved re-drive whose authorized human-corrected spec remains dirty
|
|
342
|
+
instead emits ``rollback-owned-spec-normalized`` and continues without
|
|
343
|
+
pretending the checkout is clean. Snapshot-backed changes to a pre-existing
|
|
344
|
+
untracked/ignored owned spec are parked explicitly and restored byte-exactly;
|
|
345
|
+
other plain-attempt substantive residue follows ordinary reset/pause policy.
|
|
346
|
+
|
|
347
|
+
The clean outcome also lets manual-recovery instructions terminate —
|
|
348
|
+
after the operator resets and resumes, the now-clean tree skips straight
|
|
349
|
+
through instead of re-pausing on the still-set ``baseline_commit``.
|
|
350
|
+
|
|
351
|
+
A ``cause="resolved"`` re-drive is human-initiated (the operator ran the
|
|
352
|
+
resolve workflow and re-armed the story), so it always auto-recovers and
|
|
353
|
+
never pauses, regardless of ``scm.rollback_on_failure``. For the entire
|
|
354
|
+
re-drive (``task.resolved_redrive``, latched at resume and cleared once the
|
|
355
|
+
correction is committed) the FROID artifact folders are preserved through
|
|
356
|
+
every reset — so a later mid-re-drive retry/defer reset can't silently
|
|
357
|
+
revert the correction. Whole folders never participate in the dirtiness
|
|
358
|
+
decision; sibling artifact residue remains visible there.
|
|
359
|
+
|
|
360
|
+
Otherwise (a stopped/abandoned attempt) recovery depends on where the
|
|
361
|
+
attempt ran. Inside a mounted unit worktree it auto-recovers instead of
|
|
362
|
+
pausing on policy: the worktree is disposable, the attempt's work is parked
|
|
363
|
+
on preserve refs before the reset, and ``scm.rollback_on_failure`` gates
|
|
364
|
+
*in-place* (isolation="none") recovery only (#161). In the main checkout the
|
|
365
|
+
flag governs: OFF (default) leaves the working tree untouched and emits a
|
|
366
|
+
bold manual-recovery notice that pauses the run (stop-and-wait); ON does a
|
|
367
|
+
clean reset to baseline. Either way pre-existing untracked files are
|
|
368
|
+
preserved; there is no blanket ``git clean``.
|
|
369
|
+
|
|
370
|
+
The preserve steps and unsafe attempt-owned snapshot authority are the only
|
|
371
|
+
things that can still pause a rollback the branching above chose to
|
|
372
|
+
auto-recover, worktree included: when the attempt's
|
|
373
|
+
committed or uncommitted work cannot be parked and the reset would destroy
|
|
374
|
+
it, they refuse rather than reset (#340). That is a preservation failure
|
|
375
|
+
rather than a policy decision, so it does not weaken #161 — the notice
|
|
376
|
+
targets ``workspace.root``, which is the mounted worktree when there is
|
|
377
|
+
one."""
|
|
378
|
+
workspace = self._workspace_get()
|
|
379
|
+
resolved = cause == "resolved"
|
|
380
|
+
# preserve the corrected spec for the whole re-drive, not just the first
|
|
381
|
+
# reset; the auto-recover (pause-vs-reset) decision below is unaffected.
|
|
382
|
+
redrive = resolved or task.resolved_redrive
|
|
383
|
+
# Whole-folder protection is reset-only. Attempt recognition below gets
|
|
384
|
+
# at most one literal regular-file exclusion from the attempt binding.
|
|
385
|
+
protected = self.protected_relpaths() if redrive else ()
|
|
386
|
+
owned_spec = self._attempt_owned_spec(task)
|
|
387
|
+
owned_exclude = (owned_spec[1],) if owned_spec and owned_spec[1] else ()
|
|
388
|
+
# Un-determinable dirty check (git timeout/failure, #156) ⇒ assume dirty:
|
|
389
|
+
# never skip recovery on an unproven "clean", never crash the run. The
|
|
390
|
+
# normal branching below then decides — OFF pauses (worktree kept), ON /
|
|
391
|
+
# resolved auto-recovers behind its preserve steps. A missing, unreadable,
|
|
392
|
+
# or retargeted owned-spec snapshot is a separate fail-closed boundary.
|
|
393
|
+
dirty = True
|
|
394
|
+
dirty_probe_succeeded = False
|
|
395
|
+
normalized_status: str | None = None
|
|
396
|
+
normalized_commits_present = False
|
|
397
|
+
owned_snapshot_changed = False
|
|
398
|
+
owned_snapshot_restored = False
|
|
399
|
+
owned_current_bytes: bytes | None = None
|
|
400
|
+
owned_baseline_bytes: bytes | None = None
|
|
401
|
+
owned_index_changed = False
|
|
402
|
+
snapshot_restore_pending = False
|
|
403
|
+
# ``cause=resolved`` is the initial unwind of the abandoned escalated
|
|
404
|
+
# attempt. Its persisted snapshot predates the operator's correction and
|
|
405
|
+
# must never overwrite that correction. Only a later failed attempt in
|
|
406
|
+
# the latched re-drive owns a pre-launch snapshot that is safe to restore.
|
|
407
|
+
restore_attempt_snapshot = not resolved and task.dispatched_spec_snapshot is not None
|
|
408
|
+
restore_redrive_snapshot = task.resolved_redrive and restore_attempt_snapshot
|
|
409
|
+
if not resolved and task.dispatched_spec_file and owned_spec is None:
|
|
410
|
+
# The bound path was trusted and regular at launch. A child-side
|
|
411
|
+
# deletion, directory/symlink replacement, or later resolution fault
|
|
412
|
+
# must not turn that authority into an unowned generic reset: protected
|
|
413
|
+
# artifact replay could otherwise preserve the replacement or lose the
|
|
414
|
+
# operator's only corrected copy.
|
|
415
|
+
self.journal.append(
|
|
416
|
+
"rollback-owned-spec-unavailable",
|
|
417
|
+
story_key=task.story_key,
|
|
418
|
+
)
|
|
419
|
+
self.pause_for_owned_spec_recovery(
|
|
420
|
+
task,
|
|
421
|
+
task.dispatched_spec_file,
|
|
422
|
+
"the bound path is missing, unreadable, non-regular, or was retargeted",
|
|
423
|
+
)
|
|
424
|
+
if task.baseline_commit:
|
|
425
|
+
try:
|
|
426
|
+
dirty = verify.attempt_dirty(
|
|
427
|
+
workspace.root,
|
|
428
|
+
task.baseline_commit,
|
|
429
|
+
task.baseline_untracked,
|
|
430
|
+
)
|
|
431
|
+
dirty_probe_succeeded = True
|
|
432
|
+
except (verify.GitError, OSError) as exc:
|
|
433
|
+
self.journal.append(
|
|
434
|
+
"rollback-dirty-check-failed", story_key=task.story_key, error=str(exc)
|
|
435
|
+
)
|
|
436
|
+
if not resolved and owned_spec:
|
|
437
|
+
if task.dispatched_spec_snapshot is None and task.resolved_redrive:
|
|
438
|
+
# A pre-upgrade/incomplete state cannot distinguish the operator's
|
|
439
|
+
# correction from child-authored body edits. Refuse every reset,
|
|
440
|
+
# even when rollback_on_failure is enabled: either choice could
|
|
441
|
+
# silently preserve bad bytes or discard the human correction.
|
|
442
|
+
self.journal.append(
|
|
443
|
+
"rollback-owned-spec-snapshot-missing",
|
|
444
|
+
story_key=task.story_key,
|
|
445
|
+
spec=str(owned_spec[0]),
|
|
446
|
+
)
|
|
447
|
+
self.pause_for_owned_spec_recovery(
|
|
448
|
+
task,
|
|
449
|
+
str(owned_spec[0]),
|
|
450
|
+
"the persisted retry chain predates its required byte snapshot",
|
|
451
|
+
)
|
|
452
|
+
if task.dispatched_spec_snapshot is not None:
|
|
453
|
+
try:
|
|
454
|
+
owned_current_bytes = owned_spec[0].read_bytes()
|
|
455
|
+
owned_snapshot_changed = owned_current_bytes != task.dispatched_spec_snapshot
|
|
456
|
+
except OSError as exc:
|
|
457
|
+
self.journal.append(
|
|
458
|
+
"rollback-owned-spec-unreadable",
|
|
459
|
+
story_key=task.story_key,
|
|
460
|
+
spec=str(owned_spec[0]),
|
|
461
|
+
error=str(exc),
|
|
462
|
+
)
|
|
463
|
+
self.pause_for_owned_spec_recovery(
|
|
464
|
+
task,
|
|
465
|
+
str(owned_spec[0]),
|
|
466
|
+
"its current bytes could not be read for comparison",
|
|
467
|
+
)
|
|
468
|
+
if task.baseline_commit and owned_exclude:
|
|
469
|
+
try:
|
|
470
|
+
owned_baseline_bytes = verify.worktree_file_bytes_at_revision(
|
|
471
|
+
workspace.root,
|
|
472
|
+
task.baseline_commit,
|
|
473
|
+
owned_exclude[0],
|
|
474
|
+
)
|
|
475
|
+
owned_index_changed = verify.index_path_changed_since(
|
|
476
|
+
workspace.root,
|
|
477
|
+
task.baseline_commit,
|
|
478
|
+
owned_exclude[0],
|
|
479
|
+
)
|
|
480
|
+
if verify.path_has_non_tree_ancestor_at_revision(
|
|
481
|
+
workspace.root,
|
|
482
|
+
task.baseline_commit,
|
|
483
|
+
owned_exclude[0],
|
|
484
|
+
) or verify.path_is_non_regular_at_revision(
|
|
485
|
+
workspace.root,
|
|
486
|
+
task.baseline_commit,
|
|
487
|
+
owned_exclude[0],
|
|
488
|
+
):
|
|
489
|
+
self.pause_for_owned_spec_recovery(
|
|
490
|
+
task,
|
|
491
|
+
str(owned_spec[0]),
|
|
492
|
+
"the attempt baseline would replace its canonical path "
|
|
493
|
+
"or one of its parent directories with an unsafe shape",
|
|
494
|
+
)
|
|
495
|
+
except (verify.GitError, OSError) as exc:
|
|
496
|
+
self.journal.append(
|
|
497
|
+
"rollback-owned-spec-baseline-read-failed",
|
|
498
|
+
story_key=task.story_key,
|
|
499
|
+
spec=str(owned_spec[0]),
|
|
500
|
+
error=str(exc),
|
|
501
|
+
)
|
|
502
|
+
self.pause_for_owned_spec_recovery(
|
|
503
|
+
task,
|
|
504
|
+
str(owned_spec[0]),
|
|
505
|
+
"its baseline tracking state could not be verified",
|
|
506
|
+
)
|
|
507
|
+
# Git intentionally ignores the contents of names present in
|
|
508
|
+
# baseline_untracked. The byte snapshot is the missing oracle for
|
|
509
|
+
# a child edit to a pre-existing untracked/ignored spec.
|
|
510
|
+
dirty = dirty or owned_snapshot_changed
|
|
511
|
+
if owned_snapshot_changed and not owned_exclude:
|
|
512
|
+
# An attempt-bound spec under a configured external artifact
|
|
513
|
+
# root cannot be captured by a repository recovery ref. A
|
|
514
|
+
# reset or direct redrive restoration would otherwise overwrite
|
|
515
|
+
# the only failed-child copy with the pre-launch snapshot. Leave
|
|
516
|
+
# it for explicit operator adoption instead.
|
|
517
|
+
self.journal.append(
|
|
518
|
+
"rollback-owned-spec-unpreservable",
|
|
519
|
+
story_key=task.story_key,
|
|
520
|
+
spec=str(owned_spec[0]),
|
|
521
|
+
)
|
|
522
|
+
self.pause_for_owned_spec_recovery(
|
|
523
|
+
task,
|
|
524
|
+
str(owned_spec[0]),
|
|
525
|
+
"its failed-session bytes are outside Git and cannot be parked",
|
|
526
|
+
)
|
|
527
|
+
# Establish that this attempt actually changed the checkout before the
|
|
528
|
+
# bound spec can confer any normalization authority. Stories may dispatch
|
|
529
|
+
# an already-resumable draft/in-progress/in-review spec; a session that
|
|
530
|
+
# dies without touching it must not rewrite that clean baseline.
|
|
531
|
+
if task.baseline_commit and dirty_probe_succeeded and not dirty:
|
|
532
|
+
if (
|
|
533
|
+
not resolved
|
|
534
|
+
and owned_spec
|
|
535
|
+
and task.dispatched_spec_file
|
|
536
|
+
and task.dispatched_spec_snapshot is None
|
|
537
|
+
):
|
|
538
|
+
spec_rel = owned_spec[1]
|
|
539
|
+
try:
|
|
540
|
+
baseline_status = (
|
|
541
|
+
verify.frontmatter_status_at_revision(
|
|
542
|
+
workspace.root,
|
|
543
|
+
task.baseline_commit,
|
|
544
|
+
spec_rel,
|
|
545
|
+
)
|
|
546
|
+
if spec_rel
|
|
547
|
+
else None
|
|
548
|
+
)
|
|
549
|
+
except verify.GitError:
|
|
550
|
+
baseline_status = None
|
|
551
|
+
if baseline_status is None:
|
|
552
|
+
self.journal.append(
|
|
553
|
+
"rollback-owned-spec-snapshot-missing",
|
|
554
|
+
story_key=task.story_key,
|
|
555
|
+
spec=str(owned_spec[0]),
|
|
556
|
+
)
|
|
557
|
+
self.pause_for_owned_spec_recovery(
|
|
558
|
+
task,
|
|
559
|
+
str(owned_spec[0]),
|
|
560
|
+
"no byte snapshot or tracked baseline can prove it unchanged",
|
|
561
|
+
)
|
|
562
|
+
self.journal.append("rollback-skipped-clean", story_key=task.story_key)
|
|
563
|
+
return
|
|
564
|
+
|
|
565
|
+
# The real tree is dirty. Only now ask whether the exact in-workspace
|
|
566
|
+
# binding accounts for all of it. An out-of-workspace trusted spec has no
|
|
567
|
+
# Git pathspec and therefore cannot manufacture evidence of a lifecycle
|
|
568
|
+
# delta that Git cannot observe.
|
|
569
|
+
if task.baseline_commit and dirty_probe_succeeded and owned_spec and owned_exclude:
|
|
570
|
+
dirty = True
|
|
571
|
+
dirty_probe_succeeded = False
|
|
572
|
+
try:
|
|
573
|
+
dirty = verify.attempt_dirty(
|
|
574
|
+
workspace.root,
|
|
575
|
+
task.baseline_commit,
|
|
576
|
+
task.baseline_untracked,
|
|
577
|
+
exclude=owned_exclude,
|
|
578
|
+
)
|
|
579
|
+
dirty_probe_succeeded = True
|
|
580
|
+
except (verify.GitError, OSError) as exc:
|
|
581
|
+
self.journal.append(
|
|
582
|
+
"rollback-dirty-check-failed", story_key=task.story_key, error=str(exc)
|
|
583
|
+
)
|
|
584
|
+
if task.baseline_commit and dirty_probe_succeeded and not dirty and owned_spec:
|
|
585
|
+
spec_path, spec_rel = owned_spec
|
|
586
|
+
try:
|
|
587
|
+
original_spec = spec_path.read_bytes()
|
|
588
|
+
except OSError as exc:
|
|
589
|
+
self.journal.append(
|
|
590
|
+
"rollback-owned-spec-unreadable",
|
|
591
|
+
story_key=task.story_key,
|
|
592
|
+
spec=str(spec_path),
|
|
593
|
+
error=str(exc),
|
|
594
|
+
)
|
|
595
|
+
self.pause_for_owned_spec_recovery(
|
|
596
|
+
task,
|
|
597
|
+
str(spec_path),
|
|
598
|
+
"its current bytes could not be read before lifecycle repair",
|
|
599
|
+
)
|
|
600
|
+
if redrive:
|
|
601
|
+
target_status = "in-review" if task.restore_patch else "ready-for-dev"
|
|
602
|
+
else:
|
|
603
|
+
try:
|
|
604
|
+
target_status = (
|
|
605
|
+
verify.frontmatter_status_at_revision(
|
|
606
|
+
workspace.root, task.baseline_commit, spec_rel
|
|
607
|
+
)
|
|
608
|
+
if spec_rel
|
|
609
|
+
else None
|
|
610
|
+
)
|
|
611
|
+
except verify.GitError as exc:
|
|
612
|
+
self.journal.append(
|
|
613
|
+
"rollback-owned-spec-baseline-status-failed",
|
|
614
|
+
story_key=task.story_key,
|
|
615
|
+
error=str(exc),
|
|
616
|
+
)
|
|
617
|
+
target_status = None
|
|
618
|
+
if (
|
|
619
|
+
target_status is None
|
|
620
|
+
and restore_attempt_snapshot
|
|
621
|
+
and owned_snapshot_changed
|
|
622
|
+
and not redrive
|
|
623
|
+
):
|
|
624
|
+
# A baseline-untracked or ignored spec has no Git blob whose
|
|
625
|
+
# lifecycle can be normalized. Its durable pre-launch bytes are
|
|
626
|
+
# authoritative, but preserve the failed child's current bytes on
|
|
627
|
+
# a recovery ref before restoring them in the auto-recovery arm.
|
|
628
|
+
snapshot_restore_pending = True
|
|
629
|
+
dirty = True
|
|
630
|
+
normalized_status = None
|
|
631
|
+
elif target_status is None:
|
|
632
|
+
# The exclusion proved only that the bound spec accounts for the
|
|
633
|
+
# checkout diff. Without a readable baseline status, that is not
|
|
634
|
+
# authority to mutate it or to call the real checkout clean.
|
|
635
|
+
dirty = True
|
|
636
|
+
normalized_status = None
|
|
637
|
+
else:
|
|
638
|
+
repair_probe_succeeded = True
|
|
639
|
+
if restore_redrive_snapshot:
|
|
640
|
+
# Preserve refs must capture the failed child's bytes, not the
|
|
641
|
+
# restored operator snapshot. Detect committed child residue
|
|
642
|
+
# before mutation and defer restoration until after both
|
|
643
|
+
# preserve steps when a reset is required.
|
|
644
|
+
try:
|
|
645
|
+
normalized_commits_present = bool(
|
|
646
|
+
verify.commits_above(workspace.root, task.baseline_commit)
|
|
647
|
+
)
|
|
648
|
+
except (verify.GitError, OSError) as exc:
|
|
649
|
+
dirty = True
|
|
650
|
+
dirty_probe_succeeded = False
|
|
651
|
+
repair_probe_succeeded = False
|
|
652
|
+
self.journal.append(
|
|
653
|
+
"rollback-dirty-check-failed",
|
|
654
|
+
story_key=task.story_key,
|
|
655
|
+
error=str(exc),
|
|
656
|
+
)
|
|
657
|
+
if repair_probe_succeeded:
|
|
658
|
+
if restore_redrive_snapshot and (owned_snapshot_changed or owned_index_changed):
|
|
659
|
+
# Even a spec-only failed child must be recoverable before
|
|
660
|
+
# its body is replaced. Route through the auto arm so both
|
|
661
|
+
# committed and uncommitted bytes are parked first.
|
|
662
|
+
snapshot_restore_pending = True
|
|
663
|
+
dirty = True
|
|
664
|
+
normalized_status = target_status
|
|
665
|
+
else:
|
|
666
|
+
if restore_redrive_snapshot:
|
|
667
|
+
assert task.dispatched_spec_snapshot is not None
|
|
668
|
+
self._restore_attempt_owned_spec(
|
|
669
|
+
spec_path,
|
|
670
|
+
task.dispatched_spec_snapshot,
|
|
671
|
+
target_status,
|
|
672
|
+
confine_root=workspace.paths.project,
|
|
673
|
+
)
|
|
674
|
+
owned_snapshot_restored = True
|
|
675
|
+
else:
|
|
676
|
+
self._normalize_attempt_owned_spec(
|
|
677
|
+
spec_path,
|
|
678
|
+
target_status,
|
|
679
|
+
confine_root=workspace.paths.project,
|
|
680
|
+
)
|
|
681
|
+
normalized_status = target_status
|
|
682
|
+
|
|
683
|
+
# The exclusion answered only whether anything besides the
|
|
684
|
+
# owned spec changed. Re-probe the real checkout after repair
|
|
685
|
+
# before calling it clean or choosing a recovery policy.
|
|
686
|
+
dirty = True
|
|
687
|
+
dirty_probe_succeeded = False
|
|
688
|
+
try:
|
|
689
|
+
dirty = verify.attempt_dirty(
|
|
690
|
+
workspace.root,
|
|
691
|
+
task.baseline_commit,
|
|
692
|
+
task.baseline_untracked,
|
|
693
|
+
)
|
|
694
|
+
dirty_probe_succeeded = True
|
|
695
|
+
except (verify.GitError, OSError) as exc:
|
|
696
|
+
self.journal.append(
|
|
697
|
+
"rollback-dirty-check-failed",
|
|
698
|
+
story_key=task.story_key,
|
|
699
|
+
error=str(exc),
|
|
700
|
+
)
|
|
701
|
+
if dirty_probe_succeeded and not restore_redrive_snapshot:
|
|
702
|
+
# A plain lifecycle-only attempt can have committed its
|
|
703
|
+
# flip. The normalized tree then matches baseline even
|
|
704
|
+
# though HEAD still carries abandoned history.
|
|
705
|
+
try:
|
|
706
|
+
normalized_commits_present = bool(
|
|
707
|
+
verify.commits_above(workspace.root, task.baseline_commit)
|
|
708
|
+
)
|
|
709
|
+
except (verify.GitError, OSError) as exc:
|
|
710
|
+
dirty = True
|
|
711
|
+
dirty_probe_succeeded = False
|
|
712
|
+
self.journal.append(
|
|
713
|
+
"rollback-dirty-check-failed",
|
|
714
|
+
story_key=task.story_key,
|
|
715
|
+
error=str(exc),
|
|
716
|
+
)
|
|
717
|
+
if (
|
|
718
|
+
dirty_probe_succeeded
|
|
719
|
+
and not dirty
|
|
720
|
+
and restore_attempt_snapshot
|
|
721
|
+
and owned_snapshot_changed
|
|
722
|
+
and not redrive
|
|
723
|
+
):
|
|
724
|
+
# Git-clean after lifecycle normalization means the
|
|
725
|
+
# failed child left only baseline bytes. If this attempt
|
|
726
|
+
# began with pre-existing operator dirt, restore that
|
|
727
|
+
# exact input instead of accepting the child's deletion
|
|
728
|
+
# as a clean rollback. A commit above baseline must be
|
|
729
|
+
# parked first; an uncommitted baseline copy is already
|
|
730
|
+
# durable in HEAD and needs no redundant recovery ref.
|
|
731
|
+
assert task.dispatched_spec_snapshot is not None
|
|
732
|
+
if normalized_commits_present:
|
|
733
|
+
snapshot_restore_pending = True
|
|
734
|
+
dirty = True
|
|
735
|
+
normalized_status = None
|
|
736
|
+
elif spec_path.read_bytes() != task.dispatched_spec_snapshot:
|
|
737
|
+
self._restore_attempt_owned_spec_bytes(
|
|
738
|
+
spec_path, task.dispatched_spec_snapshot
|
|
739
|
+
)
|
|
740
|
+
owned_snapshot_restored = True
|
|
741
|
+
normalized_status = None
|
|
742
|
+
dirty = True
|
|
743
|
+
dirty_probe_succeeded = False
|
|
744
|
+
try:
|
|
745
|
+
dirty = verify.attempt_dirty(
|
|
746
|
+
workspace.root,
|
|
747
|
+
task.baseline_commit,
|
|
748
|
+
task.baseline_untracked,
|
|
749
|
+
)
|
|
750
|
+
dirty_probe_succeeded = True
|
|
751
|
+
except (verify.GitError, OSError) as exc:
|
|
752
|
+
self.journal.append(
|
|
753
|
+
"rollback-dirty-check-failed",
|
|
754
|
+
story_key=task.story_key,
|
|
755
|
+
error=str(exc),
|
|
756
|
+
)
|
|
757
|
+
if (
|
|
758
|
+
dirty
|
|
759
|
+
and not redrive
|
|
760
|
+
and not snapshot_restore_pending
|
|
761
|
+
and not owned_snapshot_restored
|
|
762
|
+
):
|
|
763
|
+
# A plain attempt may be inspected or parked by the ordinary
|
|
764
|
+
# recovery policy below. If baseline-status normalization did
|
|
765
|
+
# not prove the checkout clean, put its spec back byte-for-byte
|
|
766
|
+
# before that policy claims the tree was left untouched.
|
|
767
|
+
try:
|
|
768
|
+
self._restore_attempt_owned_spec_bytes(spec_path, original_spec)
|
|
769
|
+
except _OwnedSpecAuthorityError as exc:
|
|
770
|
+
self.pause_for_owned_spec_recovery(
|
|
771
|
+
task,
|
|
772
|
+
str(spec_path),
|
|
773
|
+
"its path became unsafe while undoing a tentative "
|
|
774
|
+
f"lifecycle repair ({exc})",
|
|
775
|
+
)
|
|
776
|
+
normalized_status = None
|
|
777
|
+
if (
|
|
778
|
+
owned_snapshot_restored
|
|
779
|
+
and not redrive
|
|
780
|
+
and owned_spec
|
|
781
|
+
and task.baseline_commit
|
|
782
|
+
and dirty_probe_succeeded
|
|
783
|
+
):
|
|
784
|
+
self.journal.append(
|
|
785
|
+
"rollback-owned-spec-restored",
|
|
786
|
+
story_key=task.story_key,
|
|
787
|
+
spec=str(owned_spec[0]),
|
|
788
|
+
checkout_dirty=dirty,
|
|
789
|
+
)
|
|
790
|
+
return
|
|
791
|
+
if task.baseline_commit and not dirty and not normalized_commits_present:
|
|
792
|
+
if owned_snapshot_restored and owned_spec:
|
|
793
|
+
self.journal.append(
|
|
794
|
+
"rollback-owned-spec-restored",
|
|
795
|
+
story_key=task.story_key,
|
|
796
|
+
spec=str(owned_spec[0]),
|
|
797
|
+
checkout_dirty=False,
|
|
798
|
+
)
|
|
799
|
+
else:
|
|
800
|
+
self.journal.append("rollback-skipped-clean", story_key=task.story_key)
|
|
801
|
+
return
|
|
802
|
+
if (
|
|
803
|
+
task.baseline_commit
|
|
804
|
+
and dirty_probe_succeeded
|
|
805
|
+
and dirty
|
|
806
|
+
and redrive
|
|
807
|
+
and owned_spec
|
|
808
|
+
and normalized_status is not None
|
|
809
|
+
and not normalized_commits_present
|
|
810
|
+
and not snapshot_restore_pending
|
|
811
|
+
):
|
|
812
|
+
self.journal.append(
|
|
813
|
+
"rollback-owned-spec-normalized",
|
|
814
|
+
story_key=task.story_key,
|
|
815
|
+
spec=str(owned_spec[0]),
|
|
816
|
+
status=normalized_status,
|
|
817
|
+
checkout_dirty=True,
|
|
818
|
+
)
|
|
819
|
+
return
|
|
820
|
+
# A mounted unit worktree is disposable by design: its branch never
|
|
821
|
+
# touches the operator's checkout and the attempt's work is parked on
|
|
822
|
+
# recovery refs before any reset. `scm.rollback_on_failure` gates
|
|
823
|
+
# *in-place* (isolation="none") recovery only — pausing here would emit
|
|
824
|
+
# main-checkout reset instructions for a tree the operator never works
|
|
825
|
+
# in (#161). Compared by path, not by policy: a worktree-mode call that
|
|
826
|
+
# reaches this method *outside* a mounted unit (e.g. resume with no
|
|
827
|
+
# worktree recorded) still targets the main checkout and must pause.
|
|
828
|
+
in_unit_worktree = workspace.root != self.paths.repo_root
|
|
829
|
+
normalized_attempt_commits = (
|
|
830
|
+
task.baseline_commit is not None
|
|
831
|
+
and normalized_status is not None
|
|
832
|
+
and normalized_commits_present
|
|
833
|
+
)
|
|
834
|
+
if (
|
|
835
|
+
normalized_attempt_commits
|
|
836
|
+
or snapshot_restore_pending
|
|
837
|
+
or resolved
|
|
838
|
+
or in_unit_worktree
|
|
839
|
+
or self.policy.scm.rollback_on_failure
|
|
840
|
+
):
|
|
841
|
+
# `preserve_ref` names where *this* rollback parked the attempt. Clear
|
|
842
|
+
# it first: a later attempt that parks nothing (no commits above
|
|
843
|
+
# baseline, or a preserve failure) must not inherit the previous
|
|
844
|
+
# attempt's ref — the defer notice would then send the operator to work
|
|
845
|
+
# that is not the deferred attempt's. A *clean*-tree rollback never gets
|
|
846
|
+
# here (the `rollback-skipped-clean` return above fires first), so it
|
|
847
|
+
# deliberately keeps the earlier ref: nothing new was parked, and that
|
|
848
|
+
# ref is then the only place the story's work survives.
|
|
849
|
+
task.preserve_ref = None
|
|
850
|
+
task.preserve_partial = False
|
|
851
|
+
self.journal.append(
|
|
852
|
+
"rollback-auto",
|
|
853
|
+
story_key=task.story_key,
|
|
854
|
+
baseline=task.baseline_commit or "",
|
|
855
|
+
note="reverting tracked changes + run-created untracked files",
|
|
856
|
+
)
|
|
857
|
+
# Give a plugin (the Unity engine) a chance to quiesce before the reset
|
|
858
|
+
# rewrites tracked files under it — e.g. save + close open scenes so a
|
|
859
|
+
# git reset --hard can't leave a shared Editor showing a run-freezing
|
|
860
|
+
# "scene changed on disk" modal. Observe-only, like pre_worktree_teardown:
|
|
861
|
+
# the returned ctx is ignored and never routed through _vetoed — a failed
|
|
862
|
+
# quiesce must never block a rollback.
|
|
863
|
+
self._emit("pre_rollback", task)
|
|
864
|
+
force_owned_snapshot = (
|
|
865
|
+
owned_exclude
|
|
866
|
+
if restore_attempt_snapshot and (owned_snapshot_changed or owned_index_changed)
|
|
867
|
+
else ()
|
|
868
|
+
)
|
|
869
|
+
# A re-drive ordinarily preserves best-effort, but restoration of a
|
|
870
|
+
# changed bound spec is destructive unless both its committed and
|
|
871
|
+
# uncommitted child state can be parked first.
|
|
872
|
+
self.preserve_attempt_commits(
|
|
873
|
+
task,
|
|
874
|
+
allow_pause=not redrive or bool(force_owned_snapshot),
|
|
875
|
+
)
|
|
876
|
+
# Park the attempt's uncommitted diff too, so the reset below (and its
|
|
877
|
+
# untracked cleanup) can't silently destroy in-progress work. Runs only
|
|
878
|
+
# if preserve_attempt_commits did not pause (plain-rollback preserve
|
|
879
|
+
# failure), and refuses the reset on the same terms when a failed
|
|
880
|
+
# capture would cost unparked work (#340).
|
|
881
|
+
# After owned-spec normalization proved the checkout byte-equivalent
|
|
882
|
+
# to the baseline, only branch ancestry remains. Capturing the
|
|
883
|
+
# worktree here would park the normalization's inverse diff against
|
|
884
|
+
# the failed HEAD even though the target reset cannot discard any
|
|
885
|
+
# checkout content. The commits were parked above; reset them directly.
|
|
886
|
+
if restore_redrive_snapshot or not (normalized_attempt_commits and not dirty):
|
|
887
|
+
self.preserve_attempt_worktree(
|
|
888
|
+
task,
|
|
889
|
+
allow_pause=not redrive or bool(force_owned_snapshot),
|
|
890
|
+
force_include=force_owned_snapshot,
|
|
891
|
+
)
|
|
892
|
+
if (
|
|
893
|
+
restore_attempt_snapshot
|
|
894
|
+
and (owned_snapshot_changed or owned_index_changed)
|
|
895
|
+
and owned_spec
|
|
896
|
+
):
|
|
897
|
+
# Both refs now contain the untouched failed attempt. Restore the
|
|
898
|
+
# pre-launch input before reset. Redrives also re-establish the
|
|
899
|
+
# route promised by the next prompt; plain attempts restore exact
|
|
900
|
+
# bytes and repeat that restoration after reset so pre-existing
|
|
901
|
+
# tracked operator dirt is not erased with the child.
|
|
902
|
+
assert task.dispatched_spec_snapshot is not None
|
|
903
|
+
if redrive:
|
|
904
|
+
target_status = "in-review" if task.restore_patch else "ready-for-dev"
|
|
905
|
+
self._restore_attempt_owned_spec(
|
|
906
|
+
owned_spec[0],
|
|
907
|
+
task.dispatched_spec_snapshot,
|
|
908
|
+
target_status,
|
|
909
|
+
confine_root=workspace.paths.project,
|
|
910
|
+
)
|
|
911
|
+
else:
|
|
912
|
+
self._restore_attempt_owned_spec_bytes(
|
|
913
|
+
owned_spec[0], task.dispatched_spec_snapshot
|
|
914
|
+
)
|
|
915
|
+
owned_snapshot_restored = True
|
|
916
|
+
self.safe_reset(task, preserve=protected)
|
|
917
|
+
if restore_attempt_snapshot and owned_spec and owned_exclude and not redrive:
|
|
918
|
+
assert task.dispatched_spec_snapshot is not None
|
|
919
|
+
try:
|
|
920
|
+
self._restore_attempt_owned_spec_bytes(
|
|
921
|
+
owned_spec[0], task.dispatched_spec_snapshot
|
|
922
|
+
)
|
|
923
|
+
except _OwnedSpecAuthorityError as exc:
|
|
924
|
+
self.pause_for_owned_spec_recovery(
|
|
925
|
+
task,
|
|
926
|
+
str(owned_spec[0]),
|
|
927
|
+
f"its path became unsafe after the baseline reset ({exc})",
|
|
928
|
+
)
|
|
929
|
+
owned_snapshot_restored = True
|
|
930
|
+
if redrive and task.baseline_commit and owned_spec:
|
|
931
|
+
# A sibling source/artifact change bypasses the earlier spec-only
|
|
932
|
+
# normalization and reaches this reset. The protected artifact
|
|
933
|
+
# folders deliberately retain the corrected spec, so re-establish
|
|
934
|
+
# the route promised by the next prompt after resetting the
|
|
935
|
+
# sibling residue. Patch restores keep their review route.
|
|
936
|
+
target_status = "in-review" if task.restore_patch else "ready-for-dev"
|
|
937
|
+
if restore_redrive_snapshot and owned_index_changed and owned_spec[1]:
|
|
938
|
+
# A whole-folder preserve checkout can stage a spec that the
|
|
939
|
+
# failed child force-added or committed even though the
|
|
940
|
+
# pre-launch binding was ignored/untracked. Restore baseline
|
|
941
|
+
# index ownership before writing the operator snapshot back.
|
|
942
|
+
verify.reset_index_path(
|
|
943
|
+
workspace.root,
|
|
944
|
+
task.baseline_commit,
|
|
945
|
+
owned_spec[1],
|
|
946
|
+
)
|
|
947
|
+
if restore_redrive_snapshot and task.dispatched_spec_snapshot is not None:
|
|
948
|
+
try:
|
|
949
|
+
self._restore_attempt_owned_spec(
|
|
950
|
+
owned_spec[0],
|
|
951
|
+
task.dispatched_spec_snapshot,
|
|
952
|
+
target_status,
|
|
953
|
+
confine_root=workspace.paths.project,
|
|
954
|
+
)
|
|
955
|
+
except _OwnedSpecAuthorityError as exc:
|
|
956
|
+
self.pause_for_owned_spec_recovery(
|
|
957
|
+
task,
|
|
958
|
+
str(owned_spec[0]),
|
|
959
|
+
f"its path became unsafe after the baseline reset ({exc})",
|
|
960
|
+
)
|
|
961
|
+
else:
|
|
962
|
+
self._normalize_attempt_owned_spec(
|
|
963
|
+
owned_spec[0],
|
|
964
|
+
target_status,
|
|
965
|
+
confine_root=workspace.paths.project,
|
|
966
|
+
)
|
|
967
|
+
try:
|
|
968
|
+
checkout_dirty = verify.attempt_dirty(
|
|
969
|
+
workspace.root,
|
|
970
|
+
task.baseline_commit,
|
|
971
|
+
task.baseline_untracked,
|
|
972
|
+
)
|
|
973
|
+
except (verify.GitError, OSError) as exc:
|
|
974
|
+
self.journal.append(
|
|
975
|
+
"rollback-dirty-check-failed",
|
|
976
|
+
story_key=task.story_key,
|
|
977
|
+
error=str(exc),
|
|
978
|
+
)
|
|
979
|
+
else:
|
|
980
|
+
if checkout_dirty:
|
|
981
|
+
self.journal.append(
|
|
982
|
+
"rollback-owned-spec-normalized",
|
|
983
|
+
story_key=task.story_key,
|
|
984
|
+
spec=str(owned_spec[0]),
|
|
985
|
+
status=target_status,
|
|
986
|
+
checkout_dirty=True,
|
|
987
|
+
)
|
|
988
|
+
elif owned_snapshot_restored:
|
|
989
|
+
self.journal.append(
|
|
990
|
+
"rollback-owned-spec-restored",
|
|
991
|
+
story_key=task.story_key,
|
|
992
|
+
spec=str(owned_spec[0]),
|
|
993
|
+
checkout_dirty=False,
|
|
994
|
+
)
|
|
995
|
+
elif owned_snapshot_restored and owned_spec and task.baseline_commit:
|
|
996
|
+
try:
|
|
997
|
+
checkout_dirty = verify.attempt_dirty(
|
|
998
|
+
workspace.root,
|
|
999
|
+
task.baseline_commit,
|
|
1000
|
+
task.baseline_untracked,
|
|
1001
|
+
)
|
|
1002
|
+
except (verify.GitError, OSError) as exc:
|
|
1003
|
+
self.journal.append(
|
|
1004
|
+
"rollback-dirty-check-failed",
|
|
1005
|
+
story_key=task.story_key,
|
|
1006
|
+
error=str(exc),
|
|
1007
|
+
)
|
|
1008
|
+
else:
|
|
1009
|
+
self.journal.append(
|
|
1010
|
+
"rollback-owned-spec-restored",
|
|
1011
|
+
story_key=task.story_key,
|
|
1012
|
+
spec=str(owned_spec[0]),
|
|
1013
|
+
checkout_dirty=checkout_dirty,
|
|
1014
|
+
)
|
|
1015
|
+
# Refresh the plugin's view of the now-reset tree (the Unity engine
|
|
1016
|
+
# re-imports assets). Observe-only for the same reason as pre_rollback.
|
|
1017
|
+
self._emit("post_rollback", task)
|
|
1018
|
+
return
|
|
1019
|
+
restored_before_pause: str | None = None
|
|
1020
|
+
if (
|
|
1021
|
+
restore_attempt_snapshot
|
|
1022
|
+
and owned_snapshot_changed
|
|
1023
|
+
and owned_spec
|
|
1024
|
+
and owned_current_bytes is not None
|
|
1025
|
+
and owned_baseline_bytes is not None
|
|
1026
|
+
and owned_current_bytes == owned_baseline_bytes
|
|
1027
|
+
):
|
|
1028
|
+
# The failed child put a tracked, pre-edited spec back at the exact
|
|
1029
|
+
# baseline blob. Those child bytes are already durable in Git, so
|
|
1030
|
+
# restoring the byte-exact pre-launch operator input cannot destroy
|
|
1031
|
+
# evidence even though sibling residue still requires manual policy.
|
|
1032
|
+
assert task.dispatched_spec_snapshot is not None
|
|
1033
|
+
self._restore_attempt_owned_spec_bytes(owned_spec[0], task.dispatched_spec_snapshot)
|
|
1034
|
+
restored_before_pause = str(owned_spec[0])
|
|
1035
|
+
self.journal.append(
|
|
1036
|
+
"rollback-owned-spec-restored",
|
|
1037
|
+
story_key=task.story_key,
|
|
1038
|
+
spec=restored_before_pause,
|
|
1039
|
+
checkout_dirty=True,
|
|
1040
|
+
)
|
|
1041
|
+
self.pause_for_manual_recovery(
|
|
1042
|
+
task,
|
|
1043
|
+
task.baseline_commit or "",
|
|
1044
|
+
restored_spec=restored_before_pause,
|
|
1045
|
+
)
|
|
1046
|
+
return # unreachable: pause_for_manual_recovery always raises
|
|
1047
|
+
|
|
1048
|
+
def safe_reset(self, task: StoryTask, *, preserve: tuple[str, ...] = ()) -> None:
|
|
1049
|
+
"""Revert tracked changes to the task baseline and remove only the
|
|
1050
|
+
untracked files this run created — never a blanket `git clean`. Used by
|
|
1051
|
+
the gated/resolved rollback and by internal ledger recovery (sweep
|
|
1052
|
+
migration), which restores the orchestrator's own state and must not
|
|
1053
|
+
pause. The FROID artifact folders are always kept from untracked deletion;
|
|
1054
|
+
``preserve`` (set only on a resolved re-drive) additionally keeps their
|
|
1055
|
+
*tracked* content alive through the reset, so a just-corrected spec is not
|
|
1056
|
+
reverted. Sweep passes no ``preserve`` — it wants the broken ledger gone.
|
|
1057
|
+
A cleanup-preflight refusal is journaled and routed through the injected
|
|
1058
|
+
pause before the re-drive can continue."""
|
|
1059
|
+
workspace = self._workspace_get()
|
|
1060
|
+
try:
|
|
1061
|
+
verify.safe_rollback(
|
|
1062
|
+
workspace.root,
|
|
1063
|
+
task.baseline_commit or "",
|
|
1064
|
+
baseline_untracked=task.baseline_untracked,
|
|
1065
|
+
keep=(".froid-loop", *self.protected_relpaths()),
|
|
1066
|
+
preserve=preserve,
|
|
1067
|
+
)
|
|
1068
|
+
except verify.RollbackPreflightError as e:
|
|
1069
|
+
self.journal.append("rollback-reset-failed", story_key=task.story_key, error=str(e))
|
|
1070
|
+
self._pause(
|
|
1071
|
+
f"automatic rollback for {task.story_key} could not safely start: {e}. "
|
|
1072
|
+
"Fix the underlying filesystem fault, then resume the run.",
|
|
1073
|
+
task.story_key,
|
|
1074
|
+
cause=e,
|
|
1075
|
+
)
|
|
1076
|
+
|
|
1077
|
+
def restore_patch(self, task: StoryTask) -> None:
|
|
1078
|
+
"""Re-apply the latched intent-gap patch (Froid Plane #2564) onto the
|
|
1079
|
+
baseline tree so the re-driven session resumes review (step-04) on the
|
|
1080
|
+
restored diff instead of re-implementing. No-op unless a restore is latched.
|
|
1081
|
+
|
|
1082
|
+
Applied from inside `_dev_phase`'s loop, right before each dispatch that
|
|
1083
|
+
runs against a fresh baseline (the first attempt and every non-fixable
|
|
1084
|
+
rollback retry — the loop gates this on ``feedback is None``, so a
|
|
1085
|
+
fixable-feedback retry that KEEPS the attempt's tree is never double-applied,
|
|
1086
|
+
and the patch file's own untracked/tracked content is excluded from
|
|
1087
|
+
`baseline_untracked` because that snapshot is taken before the first apply).
|
|
1088
|
+
This is the plan's "apply after every baseline reset" seam, placed here
|
|
1089
|
+
rather than in `_rollback_or_pause` so the patch always lands after the
|
|
1090
|
+
clean baseline_untracked snapshot — avoiding a mid-re-drive reset preserving
|
|
1091
|
+
the patch's own new files and then colliding with the re-apply.
|
|
1092
|
+
|
|
1093
|
+
On apply failure we escalate rather than dispatch a session onto a
|
|
1094
|
+
half-restored tree; the task is mid-dispatch (DEV_RUNNING), so step it to
|
|
1095
|
+
the escalatable DEV_VERIFY phase first (`_escalate` raises RunPaused)."""
|
|
1096
|
+
if not task.restore_patch:
|
|
1097
|
+
return
|
|
1098
|
+
workspace = self._workspace_get()
|
|
1099
|
+
patch = verify.resolve_restore_path(task.restore_patch, workspace.root)
|
|
1100
|
+
try:
|
|
1101
|
+
verify.apply_patch(workspace.root, patch)
|
|
1102
|
+
except (verify.GitError, OSError) as e:
|
|
1103
|
+
# OSError joins GitError because the patch file is read from disk
|
|
1104
|
+
# here, so an ENOENT/EACCES/ENOSPC arrives untyped — a non-spawn FS
|
|
1105
|
+
# fault the #343 chokepoint cannot translate. Crashing would skip
|
|
1106
|
+
# the escalation this branch exists to perform and leave the tree
|
|
1107
|
+
# half-restored with no attention file.
|
|
1108
|
+
self.journal.append(
|
|
1109
|
+
"attempt-restore-failed",
|
|
1110
|
+
story_key=task.story_key,
|
|
1111
|
+
patch=task.restore_patch,
|
|
1112
|
+
error=str(e),
|
|
1113
|
+
)
|
|
1114
|
+
# Call-site invariant: `_dev_phase` advances the task to DEV_RUNNING
|
|
1115
|
+
# immediately before dispatch, and this runs on that path only — so the
|
|
1116
|
+
# step to DEV_VERIFY is unconditional. It is required because `_escalate`
|
|
1117
|
+
# cannot transition out of DEV_RUNNING directly.
|
|
1118
|
+
advance(task, Phase.DEV_VERIFY)
|
|
1119
|
+
self._escalate(task, f"intent-gap restore patch failed to apply: {e}")
|
|
1120
|
+
self.journal.append("attempt-restored", story_key=task.story_key, patch=task.restore_patch)
|
|
1121
|
+
|
|
1122
|
+
def prune_preserve_refs(self) -> None:
|
|
1123
|
+
"""Bounded retention for both recovery-ref families at run start — the
|
|
1124
|
+
attempt-preserve/* branches and the refs/attempt-preserve-dirty/*
|
|
1125
|
+
worktree snapshots: keep the newest scm.preserve_keep of each by
|
|
1126
|
+
committer date, delete the tail (mirrors the runs/cleanup retention
|
|
1127
|
+
knobs — without it the refs grow unbounded on a long-lived project).
|
|
1128
|
+
Best-effort: a git failure is journalled per family and never blocks or
|
|
1129
|
+
pauses the run — the refs are a safety net, not run state — and a
|
|
1130
|
+
failure in one family never skips the other. preserve_keep = 0 disables
|
|
1131
|
+
pruning entirely."""
|
|
1132
|
+
keep = self.policy.scm.preserve_keep
|
|
1133
|
+
if keep <= 0:
|
|
1134
|
+
return
|
|
1135
|
+
workspace = self._workspace_get()
|
|
1136
|
+
for family, prune in (
|
|
1137
|
+
("attempt-preserve", verify.prune_preserve_refs),
|
|
1138
|
+
("attempt-preserve-dirty", verify.prune_preserve_dirty_refs),
|
|
1139
|
+
):
|
|
1140
|
+
try:
|
|
1141
|
+
deleted = prune(workspace.root, keep)
|
|
1142
|
+
except Exception as exc: # housekeeping must never crash the
|
|
1143
|
+
# run: a git timeout/OSError here would otherwise escape to the crash
|
|
1144
|
+
# handler, so anything beyond the expected GitError is journalled too
|
|
1145
|
+
# A partial prune (PrunePreserveError) already deleted refs before one
|
|
1146
|
+
# stuck — that destructive half must stay structurally auditable, not
|
|
1147
|
+
# buried in the error string.
|
|
1148
|
+
partial = getattr(exc, "deleted", [])
|
|
1149
|
+
if partial:
|
|
1150
|
+
self.journal.append(f"{family}-pruned", count=len(partial), refs=partial)
|
|
1151
|
+
failed = getattr(exc, "failed", [])
|
|
1152
|
+
if failed:
|
|
1153
|
+
self.journal.append(f"{family}-prune-failed", error=str(exc), failed=failed)
|
|
1154
|
+
else:
|
|
1155
|
+
self.journal.append(f"{family}-prune-failed", error=str(exc))
|
|
1156
|
+
continue
|
|
1157
|
+
if deleted:
|
|
1158
|
+
self.journal.append(f"{family}-pruned", count=len(deleted), refs=deleted)
|
|
1159
|
+
|
|
1160
|
+
def preserve_attempt_commits(self, task: StoryTask, *, allow_pause: bool) -> None:
|
|
1161
|
+
"""Before an auto-rollback's hard reset, park any commits the attempt made
|
|
1162
|
+
above its baseline under a named recovery ref, so `reset --hard baseline`
|
|
1163
|
+
can't silently orphan committed work (it survives `git gc` and is
|
|
1164
|
+
recoverable by name, not just the reflog). No-op when the attempt added no
|
|
1165
|
+
commits — an uncommitted-only revert is the intended, non-destructive case.
|
|
1166
|
+
|
|
1167
|
+
If commits exist but the ref cannot be created — or the range cannot be
|
|
1168
|
+
enumerated at all, which is the same thing one step earlier: with
|
|
1169
|
+
``allow_pause`` (a plain rollback) refuse to reset — pause for manual
|
|
1170
|
+
recovery rather than destroy the work. Ordinary re-drive preservation uses
|
|
1171
|
+
``allow_pause=False`` and journals before proceeding; a caller that will
|
|
1172
|
+
replace a changed owned-spec snapshot passes True because that destructive
|
|
1173
|
+
write is unsafe until the child commit is parked. The two failures journal under distinct events
|
|
1174
|
+
(``attempt-preserve-enumerate-failed`` vs ``attempt-preserve-failed``) so a
|
|
1175
|
+
post-mortem can tell "could not count the work" from "counted it but could
|
|
1176
|
+
not park it" — only the latter can report a HEAD."""
|
|
1177
|
+
baseline = task.baseline_commit
|
|
1178
|
+
if not baseline:
|
|
1179
|
+
return
|
|
1180
|
+
workspace = self._workspace_get()
|
|
1181
|
+
# Enumerating the range is what decides whether the reset is safe, so a
|
|
1182
|
+
# fault here is not a no-op: an un-determinable range must read as "there
|
|
1183
|
+
# may be work above baseline" and take the preservation-failure path below,
|
|
1184
|
+
# never the `not commits` early return (which would let the reset run
|
|
1185
|
+
# blind). These two calls carried no guard at all, so until #343 a plain
|
|
1186
|
+
# git *timeout* — which `_run_git` does translate, and which every sibling
|
|
1187
|
+
# here already treats as routine — crashed the rollback outright; OSError
|
|
1188
|
+
# joins it because the translation stops at timeouts. The `not commits`
|
|
1189
|
+
# return sits inside the try to keep the original call order: HEAD is still
|
|
1190
|
+
# only read once there is something to park.
|
|
1191
|
+
try:
|
|
1192
|
+
commits = verify.commits_above(workspace.root, baseline)
|
|
1193
|
+
if not commits:
|
|
1194
|
+
return
|
|
1195
|
+
head = verify.rev_parse_head(workspace.root) # the tip the ref parks at
|
|
1196
|
+
except (verify.GitError, OSError) as exc:
|
|
1197
|
+
self.journal.append(
|
|
1198
|
+
"attempt-preserve-enumerate-failed", story_key=task.story_key, error=str(exc)
|
|
1199
|
+
)
|
|
1200
|
+
if allow_pause:
|
|
1201
|
+
# Same refusal as an un-parked ref: the notice must not tell the
|
|
1202
|
+
# operator to `reset --hard` past work we could not even count.
|
|
1203
|
+
self.pause_for_manual_recovery(task, baseline, preserve_failed=True)
|
|
1204
|
+
return # re-drive: never pause — proceed to the (human-directed) reset
|
|
1205
|
+
# run_id can be an arbitrary user `--run-id`; ref-sanitize it (same
|
|
1206
|
+
# identity-for-clean-ids / digest-for-dirty contract as the unit branches) so
|
|
1207
|
+
# an exotic/overlong id can't blow the ref-name limit, fail `git branch`, and
|
|
1208
|
+
# drop the recovery ref (which on a re-drive would then reset past the work
|
|
1209
|
+
# anyway).
|
|
1210
|
+
slug = safe_ref_segment(self.state.run_id)
|
|
1211
|
+
try:
|
|
1212
|
+
ref = verify.preserve_commits(
|
|
1213
|
+
workspace.root,
|
|
1214
|
+
baseline,
|
|
1215
|
+
f"attempt-preserve/{slug}-{head[:8]}",
|
|
1216
|
+
commits=commits,
|
|
1217
|
+
)
|
|
1218
|
+
except (verify.GitError, OSError):
|
|
1219
|
+
ref = None # branch creation failed — treat as a preservation failure
|
|
1220
|
+
if ref is None:
|
|
1221
|
+
# commits exist (just enumerated) but the ref did not take.
|
|
1222
|
+
self.journal.append("attempt-preserve-failed", story_key=task.story_key, head=head)
|
|
1223
|
+
if allow_pause:
|
|
1224
|
+
# the commits at HEAD could not be parked — the notice must NOT tell
|
|
1225
|
+
# the operator to blindly `reset --hard` (that would discard them).
|
|
1226
|
+
self.pause_for_manual_recovery(task, baseline, preserve_failed=True)
|
|
1227
|
+
return # re-drive: never pause — proceed to the (human-directed) reset
|
|
1228
|
+
task.preserve_ref = ref
|
|
1229
|
+
self.journal.append(
|
|
1230
|
+
"attempt-commits-preserved", story_key=task.story_key, ref=ref, count=len(commits)
|
|
1231
|
+
)
|
|
1232
|
+
|
|
1233
|
+
def preserve_attempt_worktree(
|
|
1234
|
+
self,
|
|
1235
|
+
task: StoryTask,
|
|
1236
|
+
*,
|
|
1237
|
+
allow_pause: bool,
|
|
1238
|
+
force_include: tuple[str, ...] = (),
|
|
1239
|
+
) -> None:
|
|
1240
|
+
"""Before an auto-rollback's hard reset, park the attempt's *uncommitted*
|
|
1241
|
+
working-tree changes (tracked edits + run-created untracked files) under a
|
|
1242
|
+
named recovery ref, so `reset --hard baseline` and its untracked cleanup
|
|
1243
|
+
can't silently destroy in-progress work. Complements
|
|
1244
|
+
`_preserve_attempt_commits` (which parks *committed* work above baseline);
|
|
1245
|
+
together they cover the whole attempt. No-op when the tree is clean vs HEAD
|
|
1246
|
+
— the intended non-destructive uncommitted-revert case.
|
|
1247
|
+
|
|
1248
|
+
A capture failure is a gate, not a footnote (#340 — this reverses the
|
|
1249
|
+
original best-effort contract). The two preserve steps used to be
|
|
1250
|
+
asymmetric: the commits path refused to reset past work it could not park,
|
|
1251
|
+
while a failed snapshot journaled and let the reset run. That protected the
|
|
1252
|
+
*more* recoverable half — orphaned commits stay in the object store,
|
|
1253
|
+
reachable by reflog/`git fsck` until gc, whereas an uncommitted edit a
|
|
1254
|
+
`reset --hard` discards is gone permanently. Both paths now refuse on the
|
|
1255
|
+
same terms: with ``allow_pause`` (a plain rollback) pause for manual
|
|
1256
|
+
recovery rather than reset; ordinary re-drive preservation remains
|
|
1257
|
+
best-effort with ``allow_pause=False``. Snapshot-backed Git-invisible specs
|
|
1258
|
+
pass ``allow_pause=True`` even on a re-drive because restoration would
|
|
1259
|
+
otherwise overwrite the only child copy. Both paths guard
|
|
1260
|
+
``(GitError, OSError)`` too: spawn faults
|
|
1261
|
+
arrive typed as ``GitSpawnError`` since #343, but ``snapshot_worktree``'s
|
|
1262
|
+
``TemporaryDirectory`` can still raise a plain ``OSError`` (ENOSPC),
|
|
1263
|
+
which would otherwise crash the rollback rather than refuse it.
|
|
1264
|
+
|
|
1265
|
+
The refusal is gated on :meth:`_reset_would_destroy`, so a capture failure
|
|
1266
|
+
over a tree with nothing left to lose (commits already parked, nothing
|
|
1267
|
+
uncommitted) still resets instead of halting an unattended run. The failure
|
|
1268
|
+
is journaled either way, and ``preserve_partial`` is latched either way —
|
|
1269
|
+
on the best-effort re-drive path the reset still runs, so the defer notice
|
|
1270
|
+
must still downgrade its claim to the committed half (#338)."""
|
|
1271
|
+
baseline = task.baseline_commit
|
|
1272
|
+
if not baseline:
|
|
1273
|
+
return
|
|
1274
|
+
workspace = self._workspace_get()
|
|
1275
|
+
# Same ref-sanitized slug as preserve_attempt_commits so an exotic/overlong
|
|
1276
|
+
# --run-id can't blow the ref-name limit and drop the ref.
|
|
1277
|
+
slug = safe_ref_segment(self.state.run_id)
|
|
1278
|
+
# ``baseline_commit`` is fixed across the whole dev retry loop, so keying the
|
|
1279
|
+
# ref on the baseline alone would make a 2nd dirty rollback reuse the name and
|
|
1280
|
+
# orphan the 1st attempt's snapshot. ``task.attempt`` discriminates the
|
|
1281
|
+
# retries of one arming but is NOT monotonic across the story's life:
|
|
1282
|
+
# runs.rearm_escalation resets it to 0, and a resolve session that commits
|
|
1283
|
+
# nothing leaves HEAD == baseline, so the post-resolve re-drive's rollback
|
|
1284
|
+
# recomputes the exact {slug}-{baseline}-{attempt} name of the pre-resolve
|
|
1285
|
+
# rollback and would overwrite that snapshot, destroying the only copy of
|
|
1286
|
+
# the first attempt's work. Probe for a free name instead of trusting the
|
|
1287
|
+
# counter: uniqueness is enforced against the refs that actually exist.
|
|
1288
|
+
# The probe runs INSIDE the try: `ref_exists` spawns git, and a timeout or
|
|
1289
|
+
# spawn fault arrives as GitError/GitSpawnError rather than a return code.
|
|
1290
|
+
# Uncaught it would crash the rollback here — the one thing this handler
|
|
1291
|
+
# exists to prevent — so a probe that cannot run degrades into the same
|
|
1292
|
+
# "preservation is observation" path as a snapshot that cannot be written.
|
|
1293
|
+
# The scan is BOUNDED: it terminates on its own (the ref set is finite and
|
|
1294
|
+
# `serial` only climbs), but termination is not a bound — the iteration
|
|
1295
|
+
# count is whatever the namespace happens to hold, one git spawn apiece, in
|
|
1296
|
+
# the middle of a crash-recovery path. PROBE_LIMIT turns "trust the
|
|
1297
|
+
# namespace is small" into an enforced invariant, and exhausting it raises
|
|
1298
|
+
# rather than reusing the last candidate: falling through to an occupied
|
|
1299
|
+
# name is the precise data loss this probe exists to prevent (#349).
|
|
1300
|
+
#
|
|
1301
|
+
# The probe is check-then-write, not atomic: `snapshot_worktree` finishes on a
|
|
1302
|
+
# plain two-arg `update-ref`, which overwrites whatever is there rather than
|
|
1303
|
+
# failing if the name were taken in between. Safe here because each name has
|
|
1304
|
+
# exactly one possible writer, on two independent grounds — the control loop
|
|
1305
|
+
# is sequential (nothing in this path threads, so one run never has two
|
|
1306
|
+
# rollbacks in flight), and the name is keyed on `run_id`, so separate runs
|
|
1307
|
+
# address disjoint namespaces. Only two processes driving the SAME run could
|
|
1308
|
+
# collide, and they would already be racing on run state, worktrees and mux
|
|
1309
|
+
# sessions; the remedy for that is run-level exclusion, not a compare-and-swap
|
|
1310
|
+
# on this one ref.
|
|
1311
|
+
base_ref = f"refs/attempt-preserve-dirty/{slug}-{baseline[:8]}-{task.attempt}"
|
|
1312
|
+
ref = base_ref
|
|
1313
|
+
serial = 2
|
|
1314
|
+
try:
|
|
1315
|
+
while verify.ref_exists(workspace.root, ref):
|
|
1316
|
+
if serial > PRESERVE_REF_PROBE_LIMIT:
|
|
1317
|
+
# Remedy has to hold at BOTH ends of the preserve_keep range:
|
|
1318
|
+
# "lower it" is impossible at 0, which is precisely the setting
|
|
1319
|
+
# (pruning disabled) that lets this namespace grow far enough to
|
|
1320
|
+
# exhaust the probe in the first place.
|
|
1321
|
+
raise verify.PreserveRefExhaustedError(
|
|
1322
|
+
f"no free snapshot refname for {base_ref}: "
|
|
1323
|
+
f"{PRESERVE_REF_PROBE_LIMIT} candidates through -r{serial - 1} "
|
|
1324
|
+
f"are all taken (prune refs/attempt-preserve-dirty/*, or set "
|
|
1325
|
+
f"scm.preserve_keep to a positive value below that limit — "
|
|
1326
|
+
f"0 disables pruning entirely)"
|
|
1327
|
+
)
|
|
1328
|
+
ref = f"{base_ref}-r{serial}"
|
|
1329
|
+
serial += 1
|
|
1330
|
+
parked = verify.snapshot_worktree(
|
|
1331
|
+
workspace.root,
|
|
1332
|
+
ref,
|
|
1333
|
+
baseline_untracked=task.baseline_untracked,
|
|
1334
|
+
force_include=force_include,
|
|
1335
|
+
)
|
|
1336
|
+
except (verify.GitError, OSError) as exc:
|
|
1337
|
+
# OSError alongside GitError: spawn faults arrive typed as GitSpawnError
|
|
1338
|
+
# since #343, but `snapshot_worktree`'s `TemporaryDirectory` can raise a
|
|
1339
|
+
# plain OSError (ENOSPC) — a non-spawn FS fault the chokepoint cannot
|
|
1340
|
+
# translate, so this arm stays load-bearing. Uncaught it crashed the
|
|
1341
|
+
# run here — after the commits ref, before the reset — which is the safe
|
|
1342
|
+
# outcome reached the loudest possible way. Preservation is observation,
|
|
1343
|
+
# so it degrades into the decision below; `safe_reset` is the repair
|
|
1344
|
+
# write and still raises.
|
|
1345
|
+
# Keep the failure detail (commit-tree/update-ref stderr, or the errno):
|
|
1346
|
+
# if the reset that may follow destroys work, this is the only breadcrumb
|
|
1347
|
+
# explaining why the safety-net snapshot couldn't be captured.
|
|
1348
|
+
self.journal.append(
|
|
1349
|
+
"attempt-worktree-preserve-failed", story_key=task.story_key, error=str(exc)
|
|
1350
|
+
)
|
|
1351
|
+
# Latch the partial marker before deciding pause-vs-reset: on the
|
|
1352
|
+
# re-drive path below the reset still runs, so `preserve_ref` may name
|
|
1353
|
+
# the commits branch parked just above and the defer notice must offer
|
|
1354
|
+
# it as the committed half rather than as the whole attempt (#338). Set
|
|
1355
|
+
# unconditionally — snapshot_worktree can raise before it can tell
|
|
1356
|
+
# whether the tree was even dirty, so "could not capture" is the only
|
|
1357
|
+
# honest state. Harmless when nothing was parked: the notice
|
|
1358
|
+
# short-circuits on the ref first.
|
|
1359
|
+
task.preserve_partial = True
|
|
1360
|
+
if not allow_pause:
|
|
1361
|
+
return # re-drive: never pause — proceed to the (human-directed) reset
|
|
1362
|
+
# Refuse the reset rather than destroy what the snapshot failed to save
|
|
1363
|
+
# (#340) — but only when something unparked is actually at stake, so a
|
|
1364
|
+
# git fault over a harmless reset can't halt an unattended run.
|
|
1365
|
+
if force_include or self._reset_would_destroy(task):
|
|
1366
|
+
self.pause_for_manual_recovery(task, baseline, snapshot_failed=True)
|
|
1367
|
+
return
|
|
1368
|
+
if parked:
|
|
1369
|
+
# Last writer wins over preserve_attempt_commits' branch on purpose:
|
|
1370
|
+
# the snapshot is commit-tree'd parented at the attempt's HEAD
|
|
1371
|
+
# (verify.snapshot_worktree), so it already contains the commits that
|
|
1372
|
+
# branch points at — one ref recovers the whole attempt. That holds on
|
|
1373
|
+
# this success path only; the `except` above records the case where the
|
|
1374
|
+
# snapshot failed and the commits branch is all that survived.
|
|
1375
|
+
task.preserve_ref = parked
|
|
1376
|
+
self.journal.append("attempt-worktree-preserved", story_key=task.story_key, ref=parked)
|
|
1377
|
+
|
|
1378
|
+
def _reset_would_destroy(self, task: StoryTask) -> bool:
|
|
1379
|
+
"""True when the pending `safe_reset` would still erase uncommitted work —
|
|
1380
|
+
the decision input for refusing a rollback whose snapshot failed (#340).
|
|
1381
|
+
|
|
1382
|
+
Probes `verify.attempt_dirty` against *HEAD* rather than the attempt
|
|
1383
|
+
baseline. That reports exactly the tracked edits and run-created untracked
|
|
1384
|
+
files `safe_rollback` is about to drop, and ignores commits above baseline,
|
|
1385
|
+
which `preserve_attempt_commits` has already parked — or paused on — by the
|
|
1386
|
+
time this runs. So a capture failure over a tree whose content was all
|
|
1387
|
+
committed reads as nothing-to-lose and the reset proceeds: a snapshot fault
|
|
1388
|
+
must not halt an unattended run when the reset itself is harmless (#123).
|
|
1389
|
+
|
|
1390
|
+
No ``exclude`` is passed because this only runs on the plain-rollback path,
|
|
1391
|
+
where `rollback_or_pause`'s ``protected`` is empty anyway. Fails safe: an
|
|
1392
|
+
un-determinable probe reads as work-at-risk, mirroring the dirty check's
|
|
1393
|
+
own git-fault doctrine (#156).
|
|
1394
|
+
|
|
1395
|
+
Catches ``OSError`` for the same reason the caller does: spawn faults
|
|
1396
|
+
arrive as ``GitSpawnError`` since #343, but this runs immediately after a
|
|
1397
|
+
snapshot fault — often an ENOSPC out of ``snapshot_worktree``'s
|
|
1398
|
+
``TemporaryDirectory`` — and a filesystem this broken can fail the probe
|
|
1399
|
+
in non-spawn ways too. Guarding only `GitError` would undo the broadening
|
|
1400
|
+
one frame up and crash the rollback anyway."""
|
|
1401
|
+
workspace = self._workspace_get()
|
|
1402
|
+
try:
|
|
1403
|
+
head = verify.rev_parse_head(workspace.root)
|
|
1404
|
+
return verify.attempt_dirty(workspace.root, head, task.baseline_untracked)
|
|
1405
|
+
except (verify.GitError, OSError):
|
|
1406
|
+
return True
|
|
1407
|
+
|
|
1408
|
+
def pause_for_manual_recovery(
|
|
1409
|
+
self,
|
|
1410
|
+
task: StoryTask,
|
|
1411
|
+
baseline: str,
|
|
1412
|
+
*,
|
|
1413
|
+
preserve_failed: bool = False,
|
|
1414
|
+
snapshot_failed: bool = False,
|
|
1415
|
+
restored_spec: str | None = None,
|
|
1416
|
+
) -> None:
|
|
1417
|
+
"""Leave the tree untouched, surface bold manual-recovery instructions, and
|
|
1418
|
+
pause the run. Always raises RunPaused. Four notice shapes: (a, default)
|
|
1419
|
+
the OFF path for a stopped/abandoned in-place attempt with no commits of
|
|
1420
|
+
its own — plain manual-rollback steps; (b, ``preserve_failed``) rollback is
|
|
1421
|
+
ON/resolved but the attempt's commits above baseline could not be parked on
|
|
1422
|
+
a recovery ref, so an automatic ``reset --hard`` would silently discard
|
|
1423
|
+
them — a distinct notice that names the at-risk commits and never tells the
|
|
1424
|
+
operator to blindly reset; (c) the OFF path but the attempt COMMITTED work
|
|
1425
|
+
above its baseline (#100: a completed session whose run died before the
|
|
1426
|
+
orchestrator folded the result) — instructing a bare ``reset --hard`` there
|
|
1427
|
+
would discard finished, possibly already-pushed commits, so this notice
|
|
1428
|
+
tells the operator to save and check integration state first; (d,
|
|
1429
|
+
``snapshot_failed``) the uncommitted-work snapshot could not be captured and
|
|
1430
|
+
the reset would have destroyed it (#340) — names the at-risk *working tree*
|
|
1431
|
+
rather than commits, and offers a git-free rescue because the fault that
|
|
1432
|
+
broke the snapshot may still be breaking git.
|
|
1433
|
+
|
|
1434
|
+
The two flags are mutually exclusive by construction: they are raised from
|
|
1435
|
+
different call sites, and `preserve_attempt_commits` pauses before
|
|
1436
|
+
`preserve_attempt_worktree` ever runs. The initial ``cause=resolved`` unwind
|
|
1437
|
+
never reaches here: it auto-recovers regardless of
|
|
1438
|
+
``scm.rollback_on_failure``. A later latched re-drive may use the
|
|
1439
|
+
snapshot-failed shape when exact Git-invisible owned-spec bytes could not
|
|
1440
|
+
be parked before recovery overwrites them."""
|
|
1441
|
+
workspace = self._workspace_get()
|
|
1442
|
+
short = baseline[:12] or "<baseline_commit>"
|
|
1443
|
+
# Name the tree every instruction targets. Usually the main checkout,
|
|
1444
|
+
# but a preserve-failure pause can fire while a unit worktree is
|
|
1445
|
+
# mounted — a bare "current HEAD" / `git reset --hard` there reads as
|
|
1446
|
+
# the operator's own checkout (whose HEAD is typically *at* the
|
|
1447
|
+
# baseline, making the quoted commit range empty) and invites a
|
|
1448
|
+
# destructive reset of a tree the attempt never touched (#161).
|
|
1449
|
+
root = workspace.root
|
|
1450
|
+
restored_note = (
|
|
1451
|
+
f"Before pausing, froid-loop restored the byte-exact pre-launch operator "
|
|
1452
|
+
f"input at `{restored_spec}` because the failed child had put that tracked "
|
|
1453
|
+
"file back at its Git baseline. That restored edit is uncommitted; save it "
|
|
1454
|
+
"alongside any other work before resetting.\n"
|
|
1455
|
+
if restored_spec
|
|
1456
|
+
else ""
|
|
1457
|
+
)
|
|
1458
|
+
commits: list[str] = []
|
|
1459
|
+
if baseline:
|
|
1460
|
+
# Advisory probe: a git fault here must not block the pause itself —
|
|
1461
|
+
# including an untranslated spawn-level OSError, which is *likelier* on
|
|
1462
|
+
# the snapshot_failed path than anywhere else (the EMFILE/ENOMEM that
|
|
1463
|
+
# broke the capture is still in force when we come to write the notice).
|
|
1464
|
+
# Degrading to "no commits" only costs notice shape (c); crashing here
|
|
1465
|
+
# would lose the pause the caller already decided to take.
|
|
1466
|
+
try:
|
|
1467
|
+
commits = verify.commits_above(root, baseline)
|
|
1468
|
+
except (verify.GitError, OSError):
|
|
1469
|
+
commits = []
|
|
1470
|
+
if preserve_failed:
|
|
1471
|
+
notice = (
|
|
1472
|
+
"**ACTION REQUIRED — commits could not be auto-preserved**\n"
|
|
1473
|
+
f"Story **{task.story_key}**'s attempt committed work above its "
|
|
1474
|
+
"baseline, but a recovery ref for those commits could not be created, "
|
|
1475
|
+
"so the automatic rollback was refused rather than `reset --hard` "
|
|
1476
|
+
"past (and discard) them. **Your commits are intact at the current "
|
|
1477
|
+
f"HEAD of `{root}`.**\n"
|
|
1478
|
+
f' 1. **Save them first** — e.g. `git -C "{root}" branch my-rescue '
|
|
1479
|
+
f"HEAD` (the commits are `{short}..HEAD` there).\n"
|
|
1480
|
+
" 2. Only once they are safe, discard the attempt if you want to: "
|
|
1481
|
+
f'`git -C "{root}" reset --hard {short}`, then review/remove leftover '
|
|
1482
|
+
"untracked files.\n"
|
|
1483
|
+
f"Then run `froid-loop resume {self.state.run_id}`."
|
|
1484
|
+
)
|
|
1485
|
+
elif snapshot_failed:
|
|
1486
|
+
# Name the committed half when it survived, so the operator is not left
|
|
1487
|
+
# assuming the whole attempt is at risk.
|
|
1488
|
+
parked = (
|
|
1489
|
+
f"The attempt's *committed* work is already parked at `{task.preserve_ref}`.\n"
|
|
1490
|
+
if task.preserve_ref
|
|
1491
|
+
else ""
|
|
1492
|
+
)
|
|
1493
|
+
notice = (
|
|
1494
|
+
"**ACTION REQUIRED — uncommitted work could not be auto-preserved**\n"
|
|
1495
|
+
f"Story **{task.story_key}**'s attempt left uncommitted changes, but the "
|
|
1496
|
+
"recovery snapshot could not be captured, so the automatic rollback was "
|
|
1497
|
+
"refused rather than `reset --hard` past (and permanently destroy) them. "
|
|
1498
|
+
"Unlike committed work, an uncommitted edit a reset discards is NOT "
|
|
1499
|
+
f"recoverable from the reflog. **Your working tree at `{root}` is "
|
|
1500
|
+
"untouched.**\n"
|
|
1501
|
+
" 1. **Save what you want to keep** — copy the files out, or "
|
|
1502
|
+
f'`git -C "{root}" diff > rescue.patch` plus any new untracked files.\n'
|
|
1503
|
+
" 2. Check the cause — the journal's `attempt-worktree-preserve-failed` "
|
|
1504
|
+
"entry carries git's own error; a full disk is the most common one.\n"
|
|
1505
|
+
" 3. Only once your work is safe: "
|
|
1506
|
+
f'`git -C "{root}" reset --hard {short}`, then review/remove leftover '
|
|
1507
|
+
"untracked files.\n"
|
|
1508
|
+
f"{parked}"
|
|
1509
|
+
f"Then run `froid-loop resume {self.state.run_id}`."
|
|
1510
|
+
)
|
|
1511
|
+
elif commits:
|
|
1512
|
+
notice = (
|
|
1513
|
+
"**ACTION REQUIRED — manual recovery needed (committed work present)**\n"
|
|
1514
|
+
f"Story **{task.story_key}**'s attempt was stopped with auto-rollback "
|
|
1515
|
+
"OFF, and it **committed work above its baseline**. **Your commits "
|
|
1516
|
+
f"are intact at the current HEAD of `{root}`.** They may already be "
|
|
1517
|
+
"integrated or pushed to a remote — do NOT reset before checking.\n"
|
|
1518
|
+
f"{restored_note}"
|
|
1519
|
+
f' 1. **Save them first** — e.g. `git -C "{root}" branch my-rescue '
|
|
1520
|
+
f"HEAD` (the commits are `{short}..HEAD` there).\n"
|
|
1521
|
+
" 2. Check whether they are already integrated (merged, pushed to "
|
|
1522
|
+
"a remote, referenced by open PRs) before discarding anything.\n"
|
|
1523
|
+
" 3. Only if you decide to discard the attempt: "
|
|
1524
|
+
f'`git -C "{root}" reset --hard {short}`, then review/remove leftover '
|
|
1525
|
+
"untracked files.\n"
|
|
1526
|
+
f"Then run `froid-loop resume {self.state.run_id}`."
|
|
1527
|
+
)
|
|
1528
|
+
else:
|
|
1529
|
+
why = (
|
|
1530
|
+
f"Story **{task.story_key}**'s attempt was stopped and auto-rollback "
|
|
1531
|
+
f"is OFF, so the working tree at `{root}` was left for you to inspect.\n"
|
|
1532
|
+
f"{restored_note}"
|
|
1533
|
+
)
|
|
1534
|
+
notice = (
|
|
1535
|
+
"**ACTION REQUIRED — manual rollback needed**\n"
|
|
1536
|
+
f"{why}"
|
|
1537
|
+
"To discard this attempt yourself:\n"
|
|
1538
|
+
" 1. **BACK UP any untracked files you want to keep** — the reset "
|
|
1539
|
+
"below deletes uncommitted work.\n"
|
|
1540
|
+
f' 2. `git -C "{root}" reset --hard {short}` then review/remove '
|
|
1541
|
+
"leftover untracked files.\n"
|
|
1542
|
+
" 3. **Restore the files you backed up in step 1.**\n"
|
|
1543
|
+
f"Then run `froid-loop resume {self.state.run_id}`. To let the "
|
|
1544
|
+
"orchestrator do a safe automatic rollback next time, enable "
|
|
1545
|
+
"`[scm] rollback_on_failure` (it discards the attempt's uncommitted "
|
|
1546
|
+
"work but never deletes pre-existing untracked files)."
|
|
1547
|
+
)
|
|
1548
|
+
self.journal.append(
|
|
1549
|
+
"rollback-manual-required",
|
|
1550
|
+
story_key=task.story_key,
|
|
1551
|
+
baseline=baseline,
|
|
1552
|
+
commits=len(commits),
|
|
1553
|
+
)
|
|
1554
|
+
gates.notify(
|
|
1555
|
+
self.policy,
|
|
1556
|
+
self.run_dir,
|
|
1557
|
+
f"ACTION REQUIRED: manual rollback for {task.story_key}",
|
|
1558
|
+
notice,
|
|
1559
|
+
)
|
|
1560
|
+
self._save()
|
|
1561
|
+
self._pause(notice, task.story_key)
|