spexcode 0.5.3 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/spec-cli/hooks/dispatch.sh +10 -0
- package/spec-cli/src/cli.ts +10 -9
- package/spec-cli/src/contract-filter.ts +73 -29
- package/spec-cli/src/guide.ts +2 -1
- package/spec-cli/src/harness.ts +426 -120
- package/spec-cli/src/host-resources.ts +20 -28
- package/spec-cli/src/index.ts +2 -1
- package/spec-cli/src/init.ts +26 -7
- package/spec-cli/src/layout.ts +3 -11
- package/spec-cli/src/materialize.ts +118 -49
- package/spec-cli/src/process-identity.ts +144 -19
- package/spec-cli/src/runtime-ownership.ts +5 -16
- package/spec-cli/src/sessions.ts +123 -17
- package/spec-cli/templates/hooks/pre-commit +3 -9
- package/spec-cli/templates/spec/project/.plugins/core/session-fail/spec.md +3 -1
- package/spec-cli/templates/spec/project/.plugins/core/spec.md +2 -2
|
@@ -1,10 +1,36 @@
|
|
|
1
|
+
import { randomUUID } from 'node:crypto'
|
|
1
2
|
import { execFileSync } from 'node:child_process'
|
|
2
|
-
import { readFileSync } from 'node:fs'
|
|
3
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs'
|
|
3
4
|
import { platform } from 'node:os'
|
|
4
|
-
import { join } from 'node:path'
|
|
5
|
+
import { dirname, join } from 'node:path'
|
|
5
6
|
|
|
6
7
|
export type ProcessIdentity = { pid: number; startToken: string }
|
|
7
|
-
|
|
8
|
+
|
|
9
|
+
export type ProcessAdapter = Readonly<{
|
|
10
|
+
platform: NodeJS.Platform
|
|
11
|
+
startToken(pid: number): string | null
|
|
12
|
+
processGroupId(pid: number): number | null
|
|
13
|
+
linuxSessionId(pid: number): number | null
|
|
14
|
+
}>
|
|
15
|
+
|
|
16
|
+
export type VerifiedDetachedRuntime = Readonly<ProcessIdentity & {
|
|
17
|
+
receiptVersion: 4
|
|
18
|
+
processGroupId: number
|
|
19
|
+
linuxSessionId?: number
|
|
20
|
+
}>
|
|
21
|
+
|
|
22
|
+
export type DetachedRuntimeVerification =
|
|
23
|
+
| { ok: true; identity: VerifiedDetachedRuntime }
|
|
24
|
+
| { ok: false; reason: string }
|
|
25
|
+
|
|
26
|
+
type DetachedLaunchReceiptV4 = {
|
|
27
|
+
version: 4
|
|
28
|
+
kind: 'spexcode-detached-runtime'
|
|
29
|
+
pid: number
|
|
30
|
+
startToken: string
|
|
31
|
+
processGroupId: number
|
|
32
|
+
linuxSessionId?: number
|
|
33
|
+
}
|
|
8
34
|
|
|
9
35
|
export function parseProcStat(text: string): { ppid: number; processGroupId: number; sessionId: number; ticks: number; startToken: string; rssPages: number } {
|
|
10
36
|
const end = text.lastIndexOf(')')
|
|
@@ -21,22 +47,6 @@ export function parseProcStat(text: string): { ppid: number; processGroupId: num
|
|
|
21
47
|
}
|
|
22
48
|
}
|
|
23
49
|
|
|
24
|
-
export function processTopology(pid: number, procRoot = '/proc'): ProcessTopology | null {
|
|
25
|
-
if (platform() === 'linux' || procRoot !== '/proc') {
|
|
26
|
-
try {
|
|
27
|
-
const stat = parseProcStat(readFileSync(join(procRoot, String(pid), 'stat'), 'utf8'))
|
|
28
|
-
return { pid, startToken: stat.startToken, processGroupId: stat.processGroupId, sessionId: stat.sessionId }
|
|
29
|
-
} catch { return null }
|
|
30
|
-
}
|
|
31
|
-
try {
|
|
32
|
-
const [processGroupId, sessionId] = execFileSync('ps', ['-o', 'pgid=', '-o', 'sess=', '-p', String(pid)], { encoding: 'utf8' }).trim().split(/\s+/).map(Number)
|
|
33
|
-
const startToken = processStartToken(pid, procRoot)
|
|
34
|
-
return startToken && Number.isFinite(processGroupId) && Number.isFinite(sessionId)
|
|
35
|
-
? { pid, startToken, processGroupId, sessionId }
|
|
36
|
-
: null
|
|
37
|
-
} catch { return null }
|
|
38
|
-
}
|
|
39
|
-
|
|
40
50
|
export function processStartToken(pid: number, procRoot = '/proc'): string | null {
|
|
41
51
|
if (platform() === 'linux' || procRoot !== '/proc') {
|
|
42
52
|
try { return parseProcStat(readFileSync(join(procRoot, String(pid), 'stat'), 'utf8')).startToken }
|
|
@@ -47,3 +57,118 @@ export function processStartToken(pid: number, procRoot = '/proc'): string | nul
|
|
|
47
57
|
return started || null
|
|
48
58
|
} catch { return null }
|
|
49
59
|
}
|
|
60
|
+
|
|
61
|
+
const procField = (pid: number, field: 'processGroupId' | 'sessionId'): number | null => {
|
|
62
|
+
try {
|
|
63
|
+
const stat = parseProcStat(readFileSync(join('/proc', String(pid), 'stat'), 'utf8'))
|
|
64
|
+
return stat[field]
|
|
65
|
+
} catch { return null }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const psProcessGroupId = (pid: number): number | null => {
|
|
69
|
+
try {
|
|
70
|
+
const value = Number(execFileSync('ps', ['-o', 'pgid=', '-p', String(pid)], { encoding: 'utf8' }).trim())
|
|
71
|
+
return Number.isInteger(value) && value > 0 ? value : null
|
|
72
|
+
} catch { return null }
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
export const hostProcessAdapter: ProcessAdapter = Object.freeze({
|
|
76
|
+
platform: platform(),
|
|
77
|
+
startToken: (pid) => processStartToken(pid),
|
|
78
|
+
processGroupId: (pid) => platform() === 'linux' ? procField(pid, 'processGroupId') : psProcessGroupId(pid),
|
|
79
|
+
linuxSessionId: (pid) => platform() === 'linux' ? procField(pid, 'sessionId') : null,
|
|
80
|
+
})
|
|
81
|
+
|
|
82
|
+
const exactKeys = (value: Record<string, unknown>, expected: string[]) => {
|
|
83
|
+
const actual = Object.keys(value).sort()
|
|
84
|
+
return actual.length === expected.length && actual.every((key, index) => key === expected[index])
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const parseDetachedLaunchReceipt = (text: string, hostPlatform: NodeJS.Platform): DetachedLaunchReceiptV4 | null => {
|
|
88
|
+
let value: unknown
|
|
89
|
+
try { value = JSON.parse(text) } catch { return null }
|
|
90
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) return null
|
|
91
|
+
const receipt = value as Record<string, unknown>
|
|
92
|
+
const keys = ['kind', 'pid', 'processGroupId', 'startToken', 'version', ...(hostPlatform === 'linux' ? ['linuxSessionId'] : [])].sort()
|
|
93
|
+
if (!exactKeys(receipt, keys) || receipt.version !== 4 || receipt.kind !== 'spexcode-detached-runtime' ||
|
|
94
|
+
!Number.isInteger(receipt.pid) || Number(receipt.pid) <= 0 || typeof receipt.startToken !== 'string' || !receipt.startToken ||
|
|
95
|
+
!Number.isInteger(receipt.processGroupId) || Number(receipt.processGroupId) <= 0 ||
|
|
96
|
+
(hostPlatform === 'linux' && (!Number.isInteger(receipt.linuxSessionId) || Number(receipt.linuxSessionId) <= 0))) return null
|
|
97
|
+
return receipt as DetachedLaunchReceiptV4
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const observeDetachedRuntime = (pid: number, adapter: ProcessAdapter): DetachedRuntimeVerification => {
|
|
101
|
+
if (adapter.platform === 'win32') return { ok: false, reason: 'detached runtime identity is unsupported on win32' }
|
|
102
|
+
const startBefore = adapter.startToken(pid)
|
|
103
|
+
if (!startBefore) return { ok: false, reason: `PID ${pid} has no readable process-start identity` }
|
|
104
|
+
const processGroupId = adapter.processGroupId(pid)
|
|
105
|
+
if (processGroupId !== pid) return { ok: false, reason: `PID ${pid} is not its own process-group leader (pgrp=${processGroupId ?? 'unknown'})` }
|
|
106
|
+
let linuxSessionId: number | undefined
|
|
107
|
+
if (adapter.platform === 'linux') {
|
|
108
|
+
const sessionId = adapter.linuxSessionId(pid)
|
|
109
|
+
if (sessionId !== pid) return { ok: false, reason: `Linux PID ${pid} is not its own session leader (session=${sessionId ?? 'unknown'})` }
|
|
110
|
+
linuxSessionId = sessionId
|
|
111
|
+
}
|
|
112
|
+
const startAfter = adapter.startToken(pid)
|
|
113
|
+
if (!startAfter || startAfter !== startBefore)
|
|
114
|
+
return { ok: false, reason: `PID ${pid} process-start identity changed during detached-boundary verification` }
|
|
115
|
+
return {
|
|
116
|
+
ok: true,
|
|
117
|
+
identity: Object.freeze({
|
|
118
|
+
pid,
|
|
119
|
+
startToken: startAfter,
|
|
120
|
+
receiptVersion: 4,
|
|
121
|
+
processGroupId,
|
|
122
|
+
...(linuxSessionId === undefined ? {} : { linuxSessionId }),
|
|
123
|
+
}),
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
export function verifyDetachedRuntime(pid: number, receiptFile: string, adapter: ProcessAdapter = hostProcessAdapter): DetachedRuntimeVerification {
|
|
128
|
+
let text: string
|
|
129
|
+
try { text = readFileSync(receiptFile, 'utf8') }
|
|
130
|
+
catch { return { ok: false, reason: 'detached launch receipt is missing or unreadable' } }
|
|
131
|
+
const receipt = parseDetachedLaunchReceipt(text, adapter.platform)
|
|
132
|
+
if (!receipt) return { ok: false, reason: 'detached launch receipt has the wrong version or shape' }
|
|
133
|
+
if (receipt.pid !== pid) return { ok: false, reason: `detached launch receipt names PID ${receipt.pid}, not ${pid}` }
|
|
134
|
+
if (receipt.processGroupId !== pid)
|
|
135
|
+
return { ok: false, reason: `detached launch receipt has wrong process group ${receipt.processGroupId} for PID ${pid}` }
|
|
136
|
+
if (adapter.platform === 'linux' && receipt.linuxSessionId !== pid)
|
|
137
|
+
return { ok: false, reason: `detached launch receipt has wrong Linux session ${receipt.linuxSessionId ?? 'missing'} for PID ${pid}` }
|
|
138
|
+
const live = observeDetachedRuntime(pid, adapter)
|
|
139
|
+
if (!live.ok) return live
|
|
140
|
+
if (live.identity.startToken !== receipt.startToken)
|
|
141
|
+
return { ok: false, reason: `PID ${pid} process-start identity does not match its detached launch receipt` }
|
|
142
|
+
return live
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
export function writeDetachedRuntimeReceipt(pid: number, receiptFile: string, adapter: ProcessAdapter = hostProcessAdapter): VerifiedDetachedRuntime {
|
|
146
|
+
const observed = observeDetachedRuntime(pid, adapter)
|
|
147
|
+
if (!observed.ok) throw new Error(`cannot prove detached shared runtime: ${observed.reason}`)
|
|
148
|
+
const receipt: DetachedLaunchReceiptV4 = {
|
|
149
|
+
version: 4,
|
|
150
|
+
kind: 'spexcode-detached-runtime',
|
|
151
|
+
pid,
|
|
152
|
+
startToken: observed.identity.startToken,
|
|
153
|
+
processGroupId: observed.identity.processGroupId,
|
|
154
|
+
...(observed.identity.linuxSessionId === undefined ? {} : { linuxSessionId: observed.identity.linuxSessionId }),
|
|
155
|
+
}
|
|
156
|
+
mkdirSync(dirname(receiptFile), { recursive: true })
|
|
157
|
+
const tmp = `${receiptFile}.${process.pid}.${randomUUID()}.tmp`
|
|
158
|
+
let published = false
|
|
159
|
+
try {
|
|
160
|
+
writeFileSync(tmp, `${JSON.stringify(receipt)}\n`, { mode: 0o600 })
|
|
161
|
+
renameSync(tmp, receiptFile)
|
|
162
|
+
published = true
|
|
163
|
+
const verified = verifyDetachedRuntime(pid, receiptFile, adapter)
|
|
164
|
+
if (!verified.ok) throw new Error(`cannot verify detached shared runtime receipt: ${verified.reason}`)
|
|
165
|
+
return verified.identity
|
|
166
|
+
} catch (error) {
|
|
167
|
+
rmSync(tmp, { force: true })
|
|
168
|
+
if (published) rmSync(receiptFile, { force: true })
|
|
169
|
+
throw error
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
export const detachedRuntimeGenerationToken = (identity: VerifiedDetachedRuntime) =>
|
|
174
|
+
`detached-v4|${identity.pid}|${identity.startToken}|${identity.processGroupId}|${identity.linuxSessionId ?? '-'}`
|
|
@@ -4,7 +4,7 @@ import { closeSync, mkdirSync, openSync, readdirSync, renameSync, rmSync, writeF
|
|
|
4
4
|
import { dirname, join } from 'node:path'
|
|
5
5
|
import { repoRoot } from './git.js'
|
|
6
6
|
import { readJsonConfig, runtimeRoot } from './layout.js'
|
|
7
|
-
import { processStartToken,
|
|
7
|
+
import { processStartToken, writeDetachedRuntimeReceipt, type ProcessIdentity } from './process-identity.js'
|
|
8
8
|
|
|
9
9
|
export type BackendInstanceRecord = {
|
|
10
10
|
version: 1
|
|
@@ -51,20 +51,11 @@ export function unregisterBackendInstance(instanceId: string, pid = process.pid)
|
|
|
51
51
|
} catch { /* not ours / already removed */ }
|
|
52
52
|
}
|
|
53
53
|
|
|
54
|
-
export function writeIsolationStamp(pid: number, file: string): void {
|
|
55
|
-
const topology = processTopology(pid)
|
|
56
|
-
if (!topology) throw new Error(`cannot identify shared runtime PID ${pid}`)
|
|
57
|
-
if (topology.processGroupId !== pid || topology.sessionId !== pid)
|
|
58
|
-
throw new Error(`shared runtime PID ${pid} is not detached from its launch session (pgrp=${topology.processGroupId}, session=${topology.sessionId})`)
|
|
59
|
-
mkdirSync(dirname(file), { recursive: true })
|
|
60
|
-
writeFileSync(file, `detached-v3 ${pid} ${topology.startToken} ${topology.processGroupId} ${topology.sessionId}\n`, { mode: 0o600 })
|
|
61
|
-
}
|
|
62
|
-
|
|
63
54
|
export function spawnDetachedRuntime(opts: {
|
|
64
55
|
cwd: string
|
|
65
56
|
logFile: string
|
|
66
57
|
pidFile: string
|
|
67
|
-
|
|
58
|
+
receiptFile: string
|
|
68
59
|
command: string
|
|
69
60
|
args: string[]
|
|
70
61
|
env?: NodeJS.ProcessEnv
|
|
@@ -85,15 +76,13 @@ export function spawnDetachedRuntime(opts: {
|
|
|
85
76
|
if (!child.pid) throw new Error(`could not spawn detached shared runtime: ${opts.command}`)
|
|
86
77
|
child.unref()
|
|
87
78
|
try {
|
|
88
|
-
|
|
89
|
-
const startToken = processStartToken(child.pid)
|
|
90
|
-
if (!startToken) throw new Error(`cannot identify detached shared runtime PID ${child.pid}`)
|
|
79
|
+
const identity = writeDetachedRuntimeReceipt(child.pid, opts.receiptFile)
|
|
91
80
|
writeFileSync(opts.pidFile, `${child.pid}\n`, { mode: 0o600 })
|
|
92
|
-
return { pid:
|
|
81
|
+
return { pid: identity.pid, startToken: identity.startToken }
|
|
93
82
|
} catch (error) {
|
|
94
83
|
try { child.kill('SIGTERM') } catch { /* exact just-spawned child already exited */ }
|
|
95
84
|
rmSync(opts.pidFile, { force: true })
|
|
96
|
-
rmSync(opts.
|
|
85
|
+
rmSync(opts.receiptFile, { force: true })
|
|
97
86
|
throw error
|
|
98
87
|
}
|
|
99
88
|
}
|
package/spec-cli/src/sessions.ts
CHANGED
|
@@ -7,7 +7,7 @@ import { fileURLToPath } from 'node:url'
|
|
|
7
7
|
import { seedWorktreeHostState } from './worktree-sources.js'
|
|
8
8
|
import { git, gitA, gitTry, repoRoot, mergeBaseDiff, mergeConflicts, type ReviewDiffFile } from './git.js'
|
|
9
9
|
import { loadConfig, loadSpecs, type ConfigPreset, type SpecLite } from './specs.js'
|
|
10
|
-
import { adapterLoadedReferenceState, defaultHarness, sessionIdentityEnvVars, defaultLauncher, harnessById, procSnapshot, resolveLauncher, rendezvousListening, stampRvSock, type Harness, type HarnessLaunchReadinessFence, type DispatchResult, type PaneProbe, type ProcTable } from './harness.js'
|
|
10
|
+
import { adapterLoadedReferenceState, defaultHarness, sessionIdentityEnvVars, defaultLauncher, harnessById, procSnapshot, resolveLauncher, rendezvousListening, stampRvSock, type Harness, type HarnessLaunchReadinessFence, type TurnFailure, type FailureSubscription, type DispatchResult, type PaneProbe, type ProcTable } from './harness.js'
|
|
11
11
|
import { materialize } from './materialize.js'
|
|
12
12
|
import { mainBranch, gitCommonDir, readConfig, runtimeRoot, treeSlotDir, sessionStoreDir, sessionRecordPath, sessionArtifactPath, listSessionIds, rawLaunchReadinessOriginal, readAliasedRawRecord, readRecordEntry, readAliasedRecordEntry, readPublicRecordEntry, envSessionId, isSessionLifecycle, isSessionProposal, type PublicRecordEntry, type RawRecord, type SessionLifecycle, type SessionProposal } from './layout.js'
|
|
13
13
|
import { recordSent, recordStatus, lastHumanSendVia } from './session-timeline.js'
|
|
@@ -1651,6 +1651,106 @@ export function superviseQueue(intervalMs = 3000): void {
|
|
|
1651
1651
|
void tick()
|
|
1652
1652
|
}
|
|
1653
1653
|
|
|
1654
|
+
type TurnFailureObserverState = {
|
|
1655
|
+
fingerprint: string
|
|
1656
|
+
subscription: FailureSubscription | null
|
|
1657
|
+
startedAt: number
|
|
1658
|
+
failures: number
|
|
1659
|
+
retryAt: number
|
|
1660
|
+
lastReason: string | null
|
|
1661
|
+
}
|
|
1662
|
+
const turnFailureObservers = new Map<string, TurnFailureObserverState>()
|
|
1663
|
+
let supervisingTurnFailures = false
|
|
1664
|
+
const TURN_FAILURE_OBSERVER_STABLE_MS = 5000
|
|
1665
|
+
|
|
1666
|
+
export function turnFailureNote(harness: string, failure: TurnFailure): string {
|
|
1667
|
+
const message = failure.message.replace(/\s+/g, ' ').trim().slice(0, 500) || 'turn failed'
|
|
1668
|
+
const at = failure.completedAt == null ? '' : ` at ${new Date(failure.completedAt * 1000).toISOString()}`
|
|
1669
|
+
return `${harness} turn failed${at}: ${message}`
|
|
1670
|
+
}
|
|
1671
|
+
|
|
1672
|
+
export function turnFailureRetryDelay(failures: number): number {
|
|
1673
|
+
return Math.min(30_000, 1000 * 2 ** Math.max(0, Math.min(failures - 1, 5)))
|
|
1674
|
+
}
|
|
1675
|
+
|
|
1676
|
+
function deferTurnFailureObserver(id: string, harness: string, state: TurnFailureObserverState, reason: string): void {
|
|
1677
|
+
state.subscription = null
|
|
1678
|
+
state.failures++
|
|
1679
|
+
const delay = turnFailureRetryDelay(state.failures)
|
|
1680
|
+
state.retryAt = Date.now() + delay
|
|
1681
|
+
if (state.lastReason !== reason)
|
|
1682
|
+
console.warn(`[spex ${harness}] turn failure observer for ${id} disconnected (${reason}); retrying in ${delay}ms`)
|
|
1683
|
+
state.lastReason = reason
|
|
1684
|
+
}
|
|
1685
|
+
|
|
1686
|
+
// Reconcile one adapter-owned native failure subscription per live governed session. Product code knows only
|
|
1687
|
+
// the optional interface capability; Codex owns WebSocket/thread semantics and Claude keeps using StopFailure.
|
|
1688
|
+
export function reconcileTurnFailureObservers(): void {
|
|
1689
|
+
const wanted = new Map<string, { rec: SessRec; harness: Harness; fingerprint: string }>()
|
|
1690
|
+
for (const id of listSessionIds()) {
|
|
1691
|
+
let rec: SessRec | null = null
|
|
1692
|
+
try { rec = readRecord(id) } catch { continue }
|
|
1693
|
+
if (!rec?.governed || rec.stopped || rec.archived || !rec.harnessSessionId) continue
|
|
1694
|
+
const harness = harnessById(rec.harness || defaultHarness.id)
|
|
1695
|
+
if (!harness.observeTurnFailures) continue
|
|
1696
|
+
wanted.set(id, { rec, harness, fingerprint: `${harness.id}:${rec.harnessSessionId}:${runtimeRoot()}` })
|
|
1697
|
+
}
|
|
1698
|
+
for (const [id, state] of turnFailureObservers) {
|
|
1699
|
+
if (wanted.get(id)?.fingerprint === state.fingerprint) continue
|
|
1700
|
+
turnFailureObservers.delete(id)
|
|
1701
|
+
state.subscription?.close()
|
|
1702
|
+
}
|
|
1703
|
+
for (const [id, target] of wanted) {
|
|
1704
|
+
const now = Date.now()
|
|
1705
|
+
let state = turnFailureObservers.get(id)
|
|
1706
|
+
if (state?.subscription) {
|
|
1707
|
+
if (state.failures > 0 && now - state.startedAt >= TURN_FAILURE_OBSERVER_STABLE_MS) {
|
|
1708
|
+
state.failures = 0
|
|
1709
|
+
state.retryAt = 0
|
|
1710
|
+
state.lastReason = null
|
|
1711
|
+
}
|
|
1712
|
+
continue
|
|
1713
|
+
}
|
|
1714
|
+
if (state && now < state.retryAt) continue
|
|
1715
|
+
state ??= { fingerprint: target.fingerprint, subscription: null, startedAt: 0, failures: 0, retryAt: 0, lastReason: null }
|
|
1716
|
+
state.startedAt = now
|
|
1717
|
+
turnFailureObservers.set(id, state)
|
|
1718
|
+
try {
|
|
1719
|
+
const subscription = target.harness.observeTurnFailures!({
|
|
1720
|
+
session: id,
|
|
1721
|
+
worktreePath: target.rec.worktreePath,
|
|
1722
|
+
harnessSessionId: target.rec.harnessSessionId,
|
|
1723
|
+
runtimeDir: runtimeRoot(),
|
|
1724
|
+
launchCmd: target.rec.launchCmd,
|
|
1725
|
+
}, (failure) => {
|
|
1726
|
+
if (turnFailureObservers.get(id)?.fingerprint !== target.fingerprint) return
|
|
1727
|
+
try { markTurnFailure(id, turnFailureNote(target.harness.id, failure)) }
|
|
1728
|
+
catch (error) { console.error(`[spex ${target.harness.id}] could not record native turn failure for ${id}: ${error instanceof Error ? error.message : String(error)}`) }
|
|
1729
|
+
})
|
|
1730
|
+
state.subscription = subscription
|
|
1731
|
+
void subscription.closed.then((reason) => {
|
|
1732
|
+
if (turnFailureObservers.get(id) !== state) return
|
|
1733
|
+
if (reason) deferTurnFailureObserver(id, target.harness.id, state, reason)
|
|
1734
|
+
else turnFailureObservers.delete(id)
|
|
1735
|
+
})
|
|
1736
|
+
} catch (error) {
|
|
1737
|
+
deferTurnFailureObserver(id, target.harness.id, state, error instanceof Error ? error.message : String(error))
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
}
|
|
1741
|
+
|
|
1742
|
+
export function superviseTurnFailures(intervalMs = 1000): void {
|
|
1743
|
+
if (supervisingTurnFailures) return
|
|
1744
|
+
supervisingTurnFailures = true
|
|
1745
|
+
const tick = () => {
|
|
1746
|
+
try { reconcileTurnFailureObservers() }
|
|
1747
|
+
catch (error) { console.error(`spex: turn failure reconciliation failed: ${error instanceof Error ? error.message : String(error)}`) }
|
|
1748
|
+
const timer = setTimeout(tick, intervalMs)
|
|
1749
|
+
timer.unref?.()
|
|
1750
|
+
}
|
|
1751
|
+
tick()
|
|
1752
|
+
}
|
|
1753
|
+
|
|
1654
1754
|
// @@@ assertProjectMatch - a WRITE is PROJECT-BOUND, but routing is by URL. A mutating verb's intent is
|
|
1655
1755
|
// "act on the project my cwd is in", yet the resolved base is a pure URL carrying no project identity —
|
|
1656
1756
|
// the backend it answers acts on ITS OWN mainRoot, so a stale inherited SPEXCODE_API_URL (pointing at
|
|
@@ -2105,19 +2205,23 @@ export function markState(status: Lifecycle, opts: { proposal?: Proposal; note?:
|
|
|
2105
2205
|
}
|
|
2106
2206
|
export const markDone = (proposal: Proposal = 'nothing', sessionId?: string, note?: string) => markState('awaiting', { proposal, note, sessionId })
|
|
2107
2207
|
export const markError = (sessionId?: string) => markState('error', { sessionId })
|
|
2108
|
-
// @@@
|
|
2109
|
-
//
|
|
2110
|
-
//
|
|
2111
|
-
export function
|
|
2112
|
-
if (
|
|
2208
|
+
// @@@ harness turn failure - native adapter failures are external runtime facts that must become visible on
|
|
2209
|
+
// the durable board. Compare-and-set only an undeclared active record, so a declaration that landed before a
|
|
2210
|
+
// late process close or app-server completion remains authoritative.
|
|
2211
|
+
export function markTurnFailure(sessionId: string | undefined, note: string): boolean {
|
|
2212
|
+
if (!sessionId) return false
|
|
2113
2213
|
return runSessionOperationSync({ op: 'lifecycle-transition', sessionId }, () => withRecordLockSync(sessionId, () => {
|
|
2114
2214
|
const rec = readLiveRecord(sessionId)
|
|
2115
|
-
if (!rec || rec.status !== 'active') return false
|
|
2116
|
-
|
|
2117
|
-
writeRecord({ ...rec, status: 'error', proposal: null, note: `${harness} turn exited with ${outcome}` })
|
|
2215
|
+
if (!rec || rec.status !== 'active' || rec.stopped || rec.archived) return false
|
|
2216
|
+
writeRecord({ ...rec, status: 'error', proposal: null, note })
|
|
2118
2217
|
return true
|
|
2119
2218
|
}))
|
|
2120
2219
|
}
|
|
2220
|
+
export function markHeadlessTurnFailure(sessionId: string, harness: string, exitCode: string): boolean {
|
|
2221
|
+
if (exitCode === '0') return false
|
|
2222
|
+
const outcome = /^\d+$/.test(exitCode) ? `exit code ${exitCode}` : `signal ${exitCode}`
|
|
2223
|
+
return markTurnFailure(sessionId, `${harness} turn exited with ${outcome}`)
|
|
2224
|
+
}
|
|
2121
2225
|
export function markHarnessSessionId(sessionId: string | undefined, harnessSessionId: string | undefined): boolean {
|
|
2122
2226
|
const id = sessionId || ownSessionId()
|
|
2123
2227
|
if (!id || !harnessSessionId) return false
|
|
@@ -2437,10 +2541,11 @@ async function assertSessionLeafOwned(id: string, rec: SessRec): Promise<LeafIde
|
|
|
2437
2541
|
return { pid, startToken, ownerNeedle }
|
|
2438
2542
|
}
|
|
2439
2543
|
|
|
2440
|
-
async function stopAgentProcess(id: string, rec: SessRec | null, requireCold = false): Promise<void> {
|
|
2544
|
+
async function stopAgentProcess(id: string, rec: SessRec | null, requireCold = false, coldReceipt?: unknown): Promise<void> {
|
|
2441
2545
|
// The caller resolves one readable owner before entering this seam. An absent/corrupt record never reaches
|
|
2442
2546
|
// tmux, signals, or adapter cleanup: a bare session id is an address, not ownership authority.
|
|
2443
|
-
const assertOwned = () => assertSessionStopSafe(id, rec ? { ...rec, harness: rec.harness } : null
|
|
2547
|
+
const assertOwned = () => assertSessionStopSafe(id, rec ? { ...rec, harness: rec.harness } : null,
|
|
2548
|
+
{ ...(requireCold && coldReceipt !== undefined ? { coldReceipt } : {}) })
|
|
2444
2549
|
await assertOwned()
|
|
2445
2550
|
if (!rec) throw new ResourceConflict(`refusing to stop ${id}: no readable session owner`)
|
|
2446
2551
|
const harness = harnessById(rec.harness || defaultHarness.id)
|
|
@@ -2453,7 +2558,7 @@ async function stopAgentProcess(id: string, rec: SessRec | null, requireCold = f
|
|
|
2453
2558
|
launchedAt.delete(id)
|
|
2454
2559
|
await harness.cleanupRuntime(rec)
|
|
2455
2560
|
if (requireCold) {
|
|
2456
|
-
const cold = await harness.coldRuntime?.(rec)
|
|
2561
|
+
const cold = await harness.coldRuntime?.(rec, coldReceipt)
|
|
2457
2562
|
if (cold && !cold.ok) throw new ResourceConflict(`refusing to archive ${id}: ${cold.reason}`)
|
|
2458
2563
|
}
|
|
2459
2564
|
}
|
|
@@ -2521,7 +2626,7 @@ async function archiveSessionUnlocked(id: string, on = true): Promise<boolean> {
|
|
|
2521
2626
|
if (rootAbsent) return true
|
|
2522
2627
|
const pre = await h.coldPreflight?.({ ...wt.rec, archived: false, stopped: true })
|
|
2523
2628
|
if (!pre || pre.ok) {
|
|
2524
|
-
const cold = await h.coldRuntime?.({ ...wt.rec, archived: false, stopped: true })
|
|
2629
|
+
const cold = await h.coldRuntime?.({ ...wt.rec, archived: false, stopped: true }, pre?.ok ? pre.receipt : undefined)
|
|
2525
2630
|
if (!cold || cold.ok) return true
|
|
2526
2631
|
}
|
|
2527
2632
|
}
|
|
@@ -2541,8 +2646,8 @@ async function archiveSessionUnlocked(id: string, on = true): Promise<boolean> {
|
|
|
2541
2646
|
: liveness({ ...wt.rec, archived: false, stopped: false }, snap)
|
|
2542
2647
|
if (lv === 'unknown' || lv === 'starting')
|
|
2543
2648
|
throw new ResourceConflict(`refusing to archive ${id}: session liveness is ${lv}; exact leaf ownership is unproven`)
|
|
2544
|
-
// The adapter guard runs BEFORE any tmux/process signal. Active/unknown
|
|
2545
|
-
// refuse here
|
|
2649
|
+
// The adapter guard runs BEFORE any tmux/process signal. Active/unknown native turns and ambiguous descendant
|
|
2650
|
+
// ownership refuse here; a verified adapter receipt carries an exact subtree through to coldRuntime's commit.
|
|
2546
2651
|
const preflight = await h.coldPreflight?.({ ...wt.rec, archived: false, stopped: lv === 'offline' })
|
|
2547
2652
|
if (preflight && !preflight.ok) throw new ResourceConflict(`refusing to archive ${id}: ${preflight.reason}`)
|
|
2548
2653
|
// Even a proven-offline leaf can leave a stale rendezvous/socket or adapter artifact. Reuse the same exact
|
|
@@ -2552,7 +2657,8 @@ async function archiveSessionUnlocked(id: string, on = true): Promise<boolean> {
|
|
|
2552
2657
|
let coldAttempted = false
|
|
2553
2658
|
try {
|
|
2554
2659
|
coldAttempted = true
|
|
2555
|
-
await stopAgentProcess(id, { ...wt.rec, archived: false, stopped: lv === 'offline' }, true
|
|
2660
|
+
await stopAgentProcess(id, { ...wt.rec, archived: false, stopped: lv === 'offline' }, true,
|
|
2661
|
+
preflight?.ok ? preflight.receipt : undefined)
|
|
2556
2662
|
coldCommitted = true
|
|
2557
2663
|
const latest = readRecord(id)
|
|
2558
2664
|
if (!latest) throw new ResourceConflict(`refusing to archive ${id}: session record disappeared before filing`)
|
|
@@ -2566,7 +2672,7 @@ async function archiveSessionUnlocked(id: string, on = true): Promise<boolean> {
|
|
|
2566
2672
|
writeRecord({ ...latest, archived: true, stopped: true, coldProof: coldProofFor(latest) })
|
|
2567
2673
|
} catch (error) {
|
|
2568
2674
|
if (coldCommitted) {
|
|
2569
|
-
const restored = await h.restoreRuntime?.(wt.rec)
|
|
2675
|
+
const restored = await h.restoreRuntime?.(wt.rec, preflight?.ok ? preflight.receipt : undefined)
|
|
2570
2676
|
if (restored && !restored.ok) {
|
|
2571
2677
|
const current = readRecord(id)
|
|
2572
2678
|
if (current) writeRecord({ ...current, archived: false, stopped: true, coldProof: null, adapterRecovery: `restore-runtime:${restored.reason}` })
|
|
@@ -58,18 +58,12 @@ fi
|
|
|
58
58
|
# stamp already passed above. Escape hatch for seeding / eager topology: SPEXCODE_ALLOW_MAIN=1 git commit …
|
|
59
59
|
#
|
|
60
60
|
# The trunk is resolved the SAME way the rest of SpexCode resolves it — `spex internal trunk` = layout.ts
|
|
61
|
-
# mainBranch() (
|
|
62
|
-
# whatever the repo's trunk is actually named
|
|
63
|
-
#
|
|
64
|
-
# auto-detect of the main checkout's current branch, then 'main' — still protecting a non-`main` trunk.
|
|
61
|
+
# mainBranch() (the stable mainBranch project fact stamped at adoption; conventional default 'main') — so
|
|
62
|
+
# the guard protects whatever the repo's trunk is actually named. When the CLI isn't resolvable (advisory
|
|
63
|
+
# mode), only the convention is available: shell does not guess identity from a mutable checkout.
|
|
65
64
|
branch=$(git symbolic-ref --short HEAD 2>/dev/null || echo)
|
|
66
65
|
git_dir=$(git rev-parse --git-dir 2>/dev/null)
|
|
67
66
|
trunk=$(spex_cli internal trunk 2>/dev/null | head -1)
|
|
68
|
-
# Fallback auto-detect: the main checkout's current branch. Strip the hook's exported GIT_DIR/
|
|
69
|
-
# GIT_INDEX_FILE first — otherwise they pin discovery to the COMMITTING worktree and `-C "$main_root"`
|
|
70
|
-
# is ignored, so it would read THIS branch and think every node branch is the trunk. (`spex internal
|
|
71
|
-
# trunk` above is immune: layout.ts mainBranch() resolves through git.ts's env-stripping git().)
|
|
72
|
-
[ -n "$trunk" ] || trunk=$(env -u GIT_DIR -u GIT_INDEX_FILE git -C "$main_root" symbolic-ref --short HEAD 2>/dev/null)
|
|
73
67
|
[ -n "$trunk" ] || trunk=main
|
|
74
68
|
if [ "$branch" = "$trunk" ] && [ ! -f "$git_dir/MERGE_HEAD" ] && [ -z "${SPEXCODE_ALLOW_MAIN:-}" ]; then
|
|
75
69
|
echo "✗ SpexCode: direct commits on $trunk (the trunk) are blocked." >&2
|
|
@@ -7,7 +7,9 @@ events:
|
|
|
7
7
|
- StopFailure
|
|
8
8
|
order: 10
|
|
9
9
|
block: false
|
|
10
|
+
code:
|
|
11
|
+
- .spec/project/.plugins/core/session-fail/fail.sh
|
|
10
12
|
---
|
|
11
13
|
When a turn ends not because the agent declared but because the API itself failed, this hook structurally marks the session `error`. A failed turn is a real outcome the board must show, and without this signal the session would freeze under whatever state it last held — reading as "active" or "awaiting" long after it actually died.
|
|
12
14
|
|
|
13
|
-
It is non-blocking
|
|
15
|
+
It is non-blocking on the failure event: the failure already happened, so the only job is to report it truthfully. As a board-lifecycle hook it acts only on a GOVERNED session — resolved in the global store from the payload's `session_id` — and writes via `spex internal session-fail --session <id>`. That machine entry reaches the same live-active compare-and-set as Codex's native failed completion and a headless turn's non-zero exit (harness-adapter): only an undeclared, non-stopped `active` record becomes `error`. A declaration, explicit stop, or archive that landed first remains authoritative; a late native failure never rewrites it. This one writer keeps the [[stop-gate]] family's invariant intact for every harness while each adapter retains only its native failure signal.
|
|
@@ -18,6 +18,6 @@ The body is the contract; update it with code when intent changes.
|
|
|
18
18
|
2. COMMIT BEFORE YOU DECLARE. Commit the spec and the code it justifies before declaring done or proposing merge.
|
|
19
19
|
Independent intent gets its own sibling node; do not ride it on an assigned node.
|
|
20
20
|
3. THE BODY IS A LIVING CURRENT-STATE DOCUMENT. Rewrite present intent in place; never add a `## vN` changelog.
|
|
21
|
-
4. KEEP THE LOSS SIGNAL HONEST. `spex spec lint` is the blocking correctness gate;
|
|
22
|
-
reports measurement gaps. Re-run changed eval scenarios through the real product, commit the verified tree, then
|
|
21
|
+
4. KEEP THE LOSS SIGNAL HONEST. Before declaring, run both: `spex spec lint` is the blocking correctness gate;
|
|
22
|
+
`spex eval lint --changed` reports measurement gaps. Re-run changed eval scenarios through the real product, commit the verified tree, then
|
|
23
23
|
file with `spex eval add`; the reading's `codeSha` must name that commit, and evidence must fit the behavior.
|