spexcode 0.5.8 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +5 -3
- package/spec-cli/bin/spex.mjs +1 -13
- package/spec-cli/hooks/compat/mark-active-0.5.2-eef1.fixture +53 -0
- package/spec-cli/hooks/compat/mark-active-sed-v0.fixture +46 -0
- package/spec-cli/hooks/dispatch.sh +15 -19
- package/spec-cli/src/anchors.ts +185 -56
- package/spec-cli/src/attach.ts +7 -17
- package/spec-cli/src/cli.ts +122 -225
- package/spec-cli/src/client.ts +114 -132
- package/spec-cli/src/cockpit.ts +43 -0
- package/spec-cli/src/codex-runtime-generations.ts +556 -0
- package/spec-cli/src/commit-surgery.ts +0 -14
- package/spec-cli/src/contract-filter.ts +0 -14
- package/spec-cli/src/delivery-queue.ts +107 -0
- package/spec-cli/src/doctor.ts +0 -16
- package/spec-cli/src/gateway-auth.ts +0 -13
- package/spec-cli/src/gateway.ts +0 -22
- package/spec-cli/src/git.ts +151 -48
- package/spec-cli/src/graph.ts +28 -7
- package/spec-cli/src/graphCache.ts +253 -72
- package/spec-cli/src/graphDelta.ts +0 -13
- package/spec-cli/src/graphStream.ts +70 -37
- package/spec-cli/src/guide.ts +29 -6
- package/spec-cli/src/harness.ts +232 -248
- package/spec-cli/src/help.ts +27 -26
- package/spec-cli/src/host-resources.ts +11 -2
- package/spec-cli/src/index.ts +71 -95
- package/spec-cli/src/issues-cli.ts +276 -0
- package/spec-cli/src/issues.ts +15 -197
- package/spec-cli/src/layout.ts +78 -2
- package/spec-cli/src/lint.ts +5 -3
- package/spec-cli/src/localIssues.ts +22 -185
- package/spec-cli/src/loop-in.ts +78 -0
- package/spec-cli/src/materialize.ts +0 -42
- package/spec-cli/src/mentions.ts +49 -193
- package/spec-cli/src/opencode-headless.ts +7 -7
- package/spec-cli/src/opencode.ts +2 -5
- package/spec-cli/src/pi-headless.ts +5 -5
- package/spec-cli/src/pty-bridge.ts +4 -5
- package/spec-cli/src/reviewSnapshot.ts +1 -0
- package/spec-cli/src/reviews.ts +6 -1
- package/spec-cli/src/root-lru.ts +54 -0
- package/spec-cli/src/runtime-guard.ts +0 -9
- package/spec-cli/src/session-cursors.ts +91 -0
- package/spec-cli/src/session-follow.ts +182 -0
- package/spec-cli/src/session-timeline.ts +55 -127
- package/spec-cli/src/sessions.ts +235 -1021
- package/spec-cli/src/shim-runtime.ts +9 -16
- package/spec-cli/src/specs.ts +33 -10
- package/spec-cli/src/uninstall.ts +0 -9
- package/spec-cli/src/uploads.ts +240 -18
- package/spec-cli/src/worktree-sources.ts +0 -16
- package/spec-cli/templates/spec/project/.plugins/commands/supervisor/spec.md +1 -1
- package/spec-cli/templates/spec/project/.plugins/core/comment-altitude/spec.md +28 -0
- package/spec-cli/templates/spec/project/.plugins/core/mark-active/mark-active.sh +7 -3
- package/spec-cli/templates/spec/project/.plugins/core/mark-active/spec.md +3 -1
- package/spec-cli/templates/spec/project/.plugins/core/stop-gate/stop-gate.sh +2 -2
- package/spec-cli/templates/spec/project/.plugins/skills/distill/spec.md +5 -0
- package/spec-cli/templates/spexcode.json +12 -0
- package/spec-dashboard/dist/assets/{App-u2P7KdSg.js → App-b8Nh0sgk.js} +2 -2
- package/spec-dashboard/dist/assets/{Dashboard-B8wp5_61.js → Dashboard-CvAjfRC2.js} +7 -7
- package/spec-dashboard/dist/assets/{EvalsPage-Bq1Tkb8y.js → EvalsPage-Bz-nMKoS.js} +1 -1
- package/spec-dashboard/dist/assets/IssuesPage-CAP64YWE.js +1 -0
- package/spec-dashboard/dist/assets/MobileApp-D9L1Va8Z.js +2 -0
- package/spec-dashboard/dist/assets/{Modal-bAkq9IIT.js → Modal-Drscez-d.js} +1 -1
- package/spec-dashboard/dist/assets/{PageScroll-px_rUZVJ.js → PageScroll-qW6uOJL8.js} +1 -1
- package/spec-dashboard/dist/assets/{ProjectsPage-8uGqYM12.js → ProjectsPage-CjybFBmR.js} +1 -1
- package/spec-dashboard/dist/assets/{SessionInterface-CswwbewF.js → SessionInterface-Dl9v0JFM.js} +13 -13
- package/spec-dashboard/dist/assets/SessionWindow-iOk0yHoU.js +1 -0
- package/spec-dashboard/dist/assets/{Settings-bpAbfnmS.js → Settings-BZ1lGRJs.js} +1 -1
- package/spec-dashboard/dist/assets/Thread-D_kcDnfd.js +13 -0
- package/spec-dashboard/dist/assets/{data-CQFbQEMH.js → data-Bwd3kAVL.js} +1 -1
- package/spec-dashboard/dist/assets/{index-CixSnz1H.css → index-DAbQBBK_.css} +1 -1
- package/spec-dashboard/dist/assets/{index-Di1ch5dd.js → index-paP-z_Vd.js} +6 -6
- package/spec-dashboard/dist/assets/{TimelineChat-Ckmb1Ez2.js → launch-B-bYdWmh.js} +25 -25
- package/spec-dashboard/dist/index.html +2 -2
- package/spec-dashboard/src/session.js +1 -28
- package/spec-eval/src/cache.ts +0 -14
- package/spec-eval/src/cli.ts +26 -5
- package/spec-eval/src/evaltab.ts +67 -108
- package/spec-eval/src/filing.ts +0 -18
- package/spec-eval/src/freshness.ts +78 -32
- package/spec-eval/src/humanok.ts +0 -15
- package/spec-eval/src/scenariofresh.ts +8 -80
- package/spec-eval/src/scenarios.ts +16 -67
- package/spec-eval/src/sessioneval.ts +157 -78
- package/spec-eval/src/sidecar.ts +1 -70
- package/spec-eval/src/timeline.ts +0 -23
- package/spec-forge/src/cache.ts +34 -12
- package/spec-forge/src/cli.ts +0 -6
- package/spec-forge/src/drivers/github.ts +1 -12
- package/spec-forge/src/drivers/gitlab.ts +4 -19
- package/spec-forge/src/drivers.ts +0 -18
- package/spec-forge/src/links.ts +0 -4
- package/spec-forge/src/needs-eval.ts +0 -1
- package/spec-forge/src/port.ts +0 -12
- package/spec-forge/src/resident.ts +4 -20
- package/spec-cli/src/maintenance-wrapper.ts +0 -276
- package/spec-cli/src/session-maintenance.ts +0 -751
- package/spec-dashboard/dist/assets/IssuesPage-BlkPSkmv.js +0 -1
- package/spec-dashboard/dist/assets/MobileApp-B1GxRZXK.js +0 -2
- package/spec-dashboard/dist/assets/SessionWindow-IspcLjFA.js +0 -1
- package/spec-dashboard/dist/assets/Thread-BpL3N3kw.js +0 -13
- /package/spec-dashboard/dist/assets/{TimelineChat-Cp579UoJ.css → launch-Cp579UoJ.css} +0 -0
|
@@ -1,751 +0,0 @@
|
|
|
1
|
-
import { AsyncLocalStorage } from 'node:async_hooks'
|
|
2
|
-
import { execFileSync } from 'node:child_process'
|
|
3
|
-
import { createHash, randomBytes as cryptoRandomBytes, timingSafeEqual } from 'node:crypto'
|
|
4
|
-
import { closeSync, existsSync, fsyncSync, linkSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'
|
|
5
|
-
import { platform } from 'node:os'
|
|
6
|
-
import { join } from 'node:path'
|
|
7
|
-
import { runtimeRoot as projectRuntimeRoot } from './layout.js'
|
|
8
|
-
import { parseProcStat, processStartToken } from './process-identity.js'
|
|
9
|
-
|
|
10
|
-
export type ProcessIdentity = { pid: number; startToken: string }
|
|
11
|
-
export type LeaseOwner = ProcessIdentity & { instanceId: string }
|
|
12
|
-
export type ProcessReading = ProcessIdentity | null | 'ambiguous'
|
|
13
|
-
export type Capability =
|
|
14
|
-
| { op: 'stop'; sessionId: string }
|
|
15
|
-
| { op: 'resume'; sessionId: string; force: boolean }
|
|
16
|
-
export type Authorization = { token: string; epoch: number }
|
|
17
|
-
export type Operation =
|
|
18
|
-
| { op: 'create' }
|
|
19
|
-
| { op: 'fallback-create' }
|
|
20
|
-
| { op: 'lifecycle-transition'; sessionId: string }
|
|
21
|
-
| { op: 'hook-state'; sessionId: string }
|
|
22
|
-
| { op: 'send'; sessionId: string }
|
|
23
|
-
| { op: 'raw-key-input'; sessionId: string }
|
|
24
|
-
| { op: 'terminal-input'; sessionId: string }
|
|
25
|
-
| { op: 'interrupt'; sessionId: string }
|
|
26
|
-
| { op: 'rename'; sessionId: string }
|
|
27
|
-
| { op: 'sort'; sessionId: string }
|
|
28
|
-
| ({ op: 'stop'; sessionId: string } & Partial<{ authorization: Authorization }>)
|
|
29
|
-
| ({ op: 'resume'; sessionId: string; force: boolean } & Partial<{ authorization: Authorization }>)
|
|
30
|
-
| { op: 'archive'; sessionId: string }
|
|
31
|
-
| { op: 'close'; sessionId: string }
|
|
32
|
-
| { op: 'quarantine'; sessionId: string }
|
|
33
|
-
| { op: 'merge-dispatch'; sessionId: string }
|
|
34
|
-
| { op: 'queue-drain' }
|
|
35
|
-
| { op: 'attach'; sessionId: string }
|
|
36
|
-
| { op: 'shared-spawn'; sessionId: string; delegate?: string }
|
|
37
|
-
|
|
38
|
-
type CapabilityState = 'unused' | 'inflight' | 'committed' | 'indeterminate'
|
|
39
|
-
type CapabilityEntry = { capability: Capability; state: CapabilityState; requestId?: string }
|
|
40
|
-
type Ticket = {
|
|
41
|
-
id: string
|
|
42
|
-
epoch: number
|
|
43
|
-
operation: Operation['op']
|
|
44
|
-
sessionId?: string
|
|
45
|
-
force?: boolean
|
|
46
|
-
owner: ProcessIdentity
|
|
47
|
-
deadline: number
|
|
48
|
-
mode?: 'ordinary' | 'maintenance'
|
|
49
|
-
parentTicketId?: string
|
|
50
|
-
}
|
|
51
|
-
type Delegate = {
|
|
52
|
-
tokenHash: string
|
|
53
|
-
parentTicketId: string
|
|
54
|
-
epoch: number
|
|
55
|
-
operation: 'shared-spawn'
|
|
56
|
-
sessionId: string
|
|
57
|
-
state: 'unused' | 'running' | 'completed'
|
|
58
|
-
}
|
|
59
|
-
type DurableState = {
|
|
60
|
-
version: 1
|
|
61
|
-
state: 'open' | 'draining' | 'active'
|
|
62
|
-
epoch: number
|
|
63
|
-
tokenHash: string | null
|
|
64
|
-
owner: LeaseOwner | null
|
|
65
|
-
heartbeatDeadline: number | null
|
|
66
|
-
capabilities: CapabilityEntry[]
|
|
67
|
-
tickets: Ticket[]
|
|
68
|
-
delegates: Delegate[]
|
|
69
|
-
}
|
|
70
|
-
export type MaintenanceTicket = {
|
|
71
|
-
id: string
|
|
72
|
-
epoch: number
|
|
73
|
-
delegateSharedSpawn(sessionId: string): string
|
|
74
|
-
}
|
|
75
|
-
export type MaintenanceLease = {
|
|
76
|
-
token: string
|
|
77
|
-
epoch: number
|
|
78
|
-
state: 'draining' | 'active'
|
|
79
|
-
owner: LeaseOwner
|
|
80
|
-
capabilities: readonly Capability[]
|
|
81
|
-
}
|
|
82
|
-
export type MaintenanceState = {
|
|
83
|
-
state: DurableState['state']
|
|
84
|
-
epoch: number
|
|
85
|
-
owner: LeaseOwner | null
|
|
86
|
-
heartbeatDeadline: number | null
|
|
87
|
-
tickets: readonly {
|
|
88
|
-
operation: Operation['op']
|
|
89
|
-
sessionId?: string
|
|
90
|
-
force?: boolean
|
|
91
|
-
owner: ProcessIdentity
|
|
92
|
-
deadline: number
|
|
93
|
-
}[]
|
|
94
|
-
capabilities: readonly CapabilityEntry[]
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
type CoordinatorInput = {
|
|
98
|
-
runtimeRoot: string
|
|
99
|
-
now: () => number
|
|
100
|
-
randomBytes: (size: number) => Buffer
|
|
101
|
-
processIdentity: (pid: number) => ProcessReading
|
|
102
|
-
selfIdentity: () => ProcessIdentity
|
|
103
|
-
ticketReportMs: number
|
|
104
|
-
onEvent?: (event: Record<string, unknown>) => void
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
const OPERATIONS = new Set<Operation['op']>([
|
|
108
|
-
'create', 'fallback-create', 'lifecycle-transition', 'hook-state', 'send', 'raw-key-input',
|
|
109
|
-
'terminal-input', 'interrupt', 'rename', 'sort', 'stop', 'resume', 'archive', 'close', 'quarantine',
|
|
110
|
-
'merge-dispatch', 'queue-drain', 'attach', 'shared-spawn',
|
|
111
|
-
])
|
|
112
|
-
const MIN_TTL_MS = 5_000
|
|
113
|
-
const MAX_TTL_MS = 300_000
|
|
114
|
-
const MAX_WAIT_MS = 120_000
|
|
115
|
-
const LOCK_WAIT_MS = 5_000
|
|
116
|
-
const syncPause = (ms: number) => Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms)
|
|
117
|
-
const context = new AsyncLocalStorage<{ ticketId: string }>()
|
|
118
|
-
|
|
119
|
-
export class SessionMaintenanceError extends Error {
|
|
120
|
-
readonly code: string
|
|
121
|
-
readonly state?: DurableState['state']
|
|
122
|
-
readonly epoch?: number
|
|
123
|
-
readonly operation?: Operation['op']
|
|
124
|
-
readonly sessionId?: string
|
|
125
|
-
|
|
126
|
-
constructor(code: string, message: string, details: Partial<Pick<SessionMaintenanceError, 'state' | 'epoch' | 'operation' | 'sessionId'>> = {}) {
|
|
127
|
-
super(message)
|
|
128
|
-
this.name = 'SessionMaintenanceError'
|
|
129
|
-
this.code = code
|
|
130
|
-
Object.assign(this, details)
|
|
131
|
-
}
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
const fail = (code: string, message: string, details?: Partial<Pick<SessionMaintenanceError, 'state' | 'epoch' | 'operation' | 'sessionId'>>): never => {
|
|
135
|
-
throw new SessionMaintenanceError(code, message, details)
|
|
136
|
-
}
|
|
137
|
-
const emptyState = (): DurableState => ({
|
|
138
|
-
version: 1, state: 'open', epoch: 0, tokenHash: null, owner: null, heartbeatDeadline: null,
|
|
139
|
-
capabilities: [], tickets: [], delegates: [],
|
|
140
|
-
})
|
|
141
|
-
const copy = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T
|
|
142
|
-
const hashToken = (token: string): string => createHash('sha256').update(token).digest('hex')
|
|
143
|
-
const sameIdentity = (a: ProcessIdentity | null | undefined, b: ProcessIdentity | null | undefined): boolean =>
|
|
144
|
-
!!a && !!b && a.pid === b.pid && a.startToken === b.startToken
|
|
145
|
-
const operationSession = (operation: Operation): string | undefined => 'sessionId' in operation ? operation.sessionId : undefined
|
|
146
|
-
const operationForce = (operation: Operation): boolean | undefined => operation.op === 'resume' ? operation.force : undefined
|
|
147
|
-
|
|
148
|
-
function canonicalCapabilities(input: readonly Capability[]): Capability[] {
|
|
149
|
-
if (!Array.isArray(input)) fail('maintenance_invalid', 'maintenance capabilities must be an array')
|
|
150
|
-
const seen = new Set<string>()
|
|
151
|
-
return input.map((raw) => {
|
|
152
|
-
if (!raw || typeof raw !== 'object' || typeof raw.sessionId !== 'string' || !raw.sessionId.trim())
|
|
153
|
-
fail('maintenance_invalid', 'maintenance capability needs an exact session id')
|
|
154
|
-
const capability: Capability = raw.op === 'stop' && Object.keys(raw).every((key) => key === 'op' || key === 'sessionId')
|
|
155
|
-
? { op: 'stop', sessionId: raw.sessionId }
|
|
156
|
-
: raw.op === 'resume' && typeof raw.force === 'boolean'
|
|
157
|
-
&& Object.keys(raw).every((key) => key === 'op' || key === 'sessionId' || key === 'force')
|
|
158
|
-
? { op: 'resume', sessionId: raw.sessionId, force: raw.force }
|
|
159
|
-
: fail('maintenance_invalid', 'maintenance capability must be exact stop or resume authority')
|
|
160
|
-
const key = `${capability.op}\0${capability.sessionId}`
|
|
161
|
-
if (seen.has(key)) fail('maintenance_invalid', `duplicate maintenance capability for ${capability.op} ${capability.sessionId}`)
|
|
162
|
-
seen.add(key)
|
|
163
|
-
return capability
|
|
164
|
-
})
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
function normalizeState(raw: unknown): DurableState {
|
|
168
|
-
if (!raw || typeof raw !== 'object') fail('maintenance_state_invalid', 'session maintenance state is unreadable')
|
|
169
|
-
const value = raw as Partial<DurableState>
|
|
170
|
-
if (value.version !== 1 || !['open', 'draining', 'active'].includes(String(value.state)) || !Number.isSafeInteger(value.epoch) || Number(value.epoch) < 0)
|
|
171
|
-
fail('maintenance_state_invalid', 'session maintenance state has an invalid version, state, or epoch')
|
|
172
|
-
return {
|
|
173
|
-
version: 1,
|
|
174
|
-
state: value.state as DurableState['state'],
|
|
175
|
-
epoch: Number(value.epoch),
|
|
176
|
-
tokenHash: typeof value.tokenHash === 'string' ? value.tokenHash : null,
|
|
177
|
-
owner: value.owner && typeof value.owner.instanceId === 'string' && value.owner.instanceId
|
|
178
|
-
&& Number.isInteger(value.owner.pid) && typeof value.owner.startToken === 'string'
|
|
179
|
-
? copy(value.owner as LeaseOwner)
|
|
180
|
-
: null,
|
|
181
|
-
heartbeatDeadline: typeof value.heartbeatDeadline === 'number' ? value.heartbeatDeadline : null,
|
|
182
|
-
capabilities: Array.isArray(value.capabilities) ? copy(value.capabilities) : [],
|
|
183
|
-
tickets: Array.isArray(value.tickets) ? copy(value.tickets) : [],
|
|
184
|
-
delegates: Array.isArray(value.delegates) ? copy(value.delegates) : [],
|
|
185
|
-
}
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
function presentedTokenMatches(token: string, expectedHash: string | null): boolean {
|
|
189
|
-
const actual = createHash('sha256').update(token).digest()
|
|
190
|
-
const valid = typeof expectedHash === 'string' && /^[0-9a-f]{64}$/i.test(expectedHash)
|
|
191
|
-
const expected = valid ? Buffer.from(expectedHash!, 'hex') : Buffer.alloc(actual.length)
|
|
192
|
-
return timingSafeEqual(actual, expected) && valid
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
function parentPid(pid: number): number | null {
|
|
196
|
-
if (platform() === 'linux') {
|
|
197
|
-
try { return parseProcStat(readFileSync(`/proc/${pid}/stat`, 'utf8')).ppid } catch { return null }
|
|
198
|
-
}
|
|
199
|
-
try {
|
|
200
|
-
const value = Number(execFileSync('ps', ['-o', 'ppid=', '-p', String(pid)], { encoding: 'utf8' }).trim())
|
|
201
|
-
return Number.isInteger(value) && value > 0 ? value : null
|
|
202
|
-
} catch { return null }
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
function isDescendantOf(pid: number, owner: ProcessIdentity, readIdentity: (pid: number) => ProcessReading): boolean {
|
|
206
|
-
const ownerReading = readIdentity(owner.pid)
|
|
207
|
-
if (ownerReading === 'ambiguous' || !sameIdentity(ownerReading, owner)) return false
|
|
208
|
-
let current = pid
|
|
209
|
-
for (let depth = 0; depth < 64 && current > 1; depth++) {
|
|
210
|
-
if (current === owner.pid) return true
|
|
211
|
-
const parent = parentPid(current)
|
|
212
|
-
if (!parent || parent === current) return false
|
|
213
|
-
current = parent
|
|
214
|
-
}
|
|
215
|
-
return false
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
export function createSessionMaintenance(input: CoordinatorInput) {
|
|
219
|
-
const statePath = join(input.runtimeRoot, 'session-maintenance.json')
|
|
220
|
-
const lockPath = join(input.runtimeRoot, '.session-maintenance.lock')
|
|
221
|
-
const lockPrefix = '.session-maintenance.lock.reap-'
|
|
222
|
-
mkdirSync(input.runtimeRoot, { recursive: true })
|
|
223
|
-
|
|
224
|
-
const identityState = (owner: ProcessIdentity): 'live' | 'dead' | 'reused' | 'ambiguous' => {
|
|
225
|
-
const reading = input.processIdentity(owner.pid)
|
|
226
|
-
if (reading === 'ambiguous') return 'ambiguous'
|
|
227
|
-
if (reading === null) return 'dead'
|
|
228
|
-
return sameIdentity(reading, owner) ? 'live' : 'reused'
|
|
229
|
-
}
|
|
230
|
-
|
|
231
|
-
type LockRecord = { version: 1; nonce: string; owner: ProcessIdentity }
|
|
232
|
-
const readLock = (path = lockPath): LockRecord | null => {
|
|
233
|
-
try {
|
|
234
|
-
const raw = JSON.parse(readFileSync(path, 'utf8')) as Partial<LockRecord>
|
|
235
|
-
return raw.version === 1 && typeof raw.nonce === 'string' && raw.nonce
|
|
236
|
-
&& raw.owner && Number.isInteger(raw.owner.pid) && typeof raw.owner.startToken === 'string'
|
|
237
|
-
? raw as LockRecord
|
|
238
|
-
: null
|
|
239
|
-
} catch { return null }
|
|
240
|
-
}
|
|
241
|
-
const writeLinkedRecord = (path: string, record: LockRecord): boolean => {
|
|
242
|
-
const temp = join(input.runtimeRoot, `.session-maintenance.owner-${process.pid}-${record.nonce}.tmp`)
|
|
243
|
-
let fd: number | null = null
|
|
244
|
-
try {
|
|
245
|
-
fd = openSync(temp, 'wx', 0o600)
|
|
246
|
-
writeFileSync(fd, JSON.stringify(record))
|
|
247
|
-
fsyncSync(fd)
|
|
248
|
-
closeSync(fd); fd = null
|
|
249
|
-
linkSync(temp, path)
|
|
250
|
-
return true
|
|
251
|
-
} catch (error) {
|
|
252
|
-
if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error
|
|
253
|
-
return false
|
|
254
|
-
} finally {
|
|
255
|
-
if (fd !== null) closeSync(fd)
|
|
256
|
-
try { unlinkSync(temp) } catch { /* linked or already absent */ }
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
const reapMarkers = (): string[] => {
|
|
260
|
-
try { return readdirSync(input.runtimeRoot).filter((name) => name.startsWith(lockPrefix)) }
|
|
261
|
-
catch { return [] }
|
|
262
|
-
}
|
|
263
|
-
const sameLockRecord = (left: LockRecord | null, right: LockRecord): boolean =>
|
|
264
|
-
!!left && left.nonce === right.nonce && sameIdentity(left.owner, right.owner)
|
|
265
|
-
const reclaimDeadRecord = (path: string, expected: LockRecord, depth = 0): boolean => {
|
|
266
|
-
if (depth >= 64) fail('maintenance_conflict', 'session maintenance lock claim depth exceeded')
|
|
267
|
-
if (!sameLockRecord(readLock(path), expected) || !['dead', 'reused'].includes(identityState(expected.owner))) return false
|
|
268
|
-
const claimPath = `${path}.claim`
|
|
269
|
-
const self = input.selfIdentity()
|
|
270
|
-
const claim: LockRecord = { version: 1, nonce: input.randomBytes(16).toString('hex'), owner: self }
|
|
271
|
-
if (!writeLinkedRecord(claimPath, claim)) {
|
|
272
|
-
const existing = readLock(claimPath)
|
|
273
|
-
if (!existing || !['dead', 'reused'].includes(identityState(existing.owner))) return false
|
|
274
|
-
if (!reclaimDeadRecord(claimPath, existing, depth + 1)) return false
|
|
275
|
-
return reclaimDeadRecord(path, expected, depth + 1)
|
|
276
|
-
}
|
|
277
|
-
try {
|
|
278
|
-
const confirmed = readLock(path)
|
|
279
|
-
if (!sameLockRecord(confirmed, expected) || !['dead', 'reused'].includes(identityState(expected.owner))) return false
|
|
280
|
-
unlinkSync(path)
|
|
281
|
-
return true
|
|
282
|
-
} finally {
|
|
283
|
-
if (sameLockRecord(readLock(claimPath), claim)) unlinkSync(claimPath)
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
const clearDeadMarkers = (): boolean => {
|
|
287
|
-
for (const name of reapMarkers()) {
|
|
288
|
-
const path = join(input.runtimeRoot, name)
|
|
289
|
-
const marker = readLock(path)
|
|
290
|
-
if (marker && ['dead', 'reused'].includes(identityState(marker.owner))) reclaimDeadRecord(path, marker)
|
|
291
|
-
}
|
|
292
|
-
return reapMarkers().length === 0
|
|
293
|
-
}
|
|
294
|
-
|
|
295
|
-
const acquireLock = (): (() => void) => {
|
|
296
|
-
const deadline = Date.now() + LOCK_WAIT_MS
|
|
297
|
-
const self = input.selfIdentity()
|
|
298
|
-
const nonce = input.randomBytes(16).toString('hex')
|
|
299
|
-
const own: LockRecord = { version: 1, nonce, owner: self }
|
|
300
|
-
for (;;) {
|
|
301
|
-
if (clearDeadMarkers() && writeLinkedRecord(lockPath, own)) {
|
|
302
|
-
return () => {
|
|
303
|
-
const current = readLock()
|
|
304
|
-
if (!current || current.nonce !== nonce || !sameIdentity(current.owner, self))
|
|
305
|
-
fail('maintenance_conflict', 'session maintenance lock ownership changed before release')
|
|
306
|
-
unlinkSync(lockPath)
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
|
|
310
|
-
const current = readLock()
|
|
311
|
-
if (!current) {
|
|
312
|
-
// Legacy mkdir-before-owner residue is the old crash window this protocol removes. It has no
|
|
313
|
-
// complete identity that can authorize a state mutation, so remove only an empty directory.
|
|
314
|
-
try { if (readdirSync(lockPath).length === 0) rmSync(lockPath, { recursive: true }) } catch { /* file or live legacy owner */ }
|
|
315
|
-
} else if (['dead', 'reused'].includes(identityState(current.owner))) {
|
|
316
|
-
const markerPath = join(input.runtimeRoot, `${lockPrefix}${current.nonce}`)
|
|
317
|
-
const marker: LockRecord = { version: 1, nonce: input.randomBytes(16).toString('hex'), owner: self }
|
|
318
|
-
if (writeLinkedRecord(markerPath, marker)) {
|
|
319
|
-
try {
|
|
320
|
-
const confirmed = readLock()
|
|
321
|
-
if (confirmed?.nonce === current.nonce && sameIdentity(confirmed.owner, current.owner)
|
|
322
|
-
&& ['dead', 'reused'].includes(identityState(confirmed.owner))) unlinkSync(lockPath)
|
|
323
|
-
} finally {
|
|
324
|
-
const claim = readLock(markerPath)
|
|
325
|
-
if (claim?.nonce === marker.nonce && sameIdentity(claim.owner, self)) unlinkSync(markerPath)
|
|
326
|
-
}
|
|
327
|
-
continue
|
|
328
|
-
}
|
|
329
|
-
}
|
|
330
|
-
if (Date.now() >= deadline) fail('maintenance_conflict', 'session maintenance state lock is held by a live or ambiguous owner')
|
|
331
|
-
syncPause(5)
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
const load = (): { state: DurableState; exists: boolean } => {
|
|
336
|
-
if (!existsSync(statePath)) return { state: emptyState(), exists: false }
|
|
337
|
-
try { return { state: normalizeState(JSON.parse(readFileSync(statePath, 'utf8'))), exists: true } }
|
|
338
|
-
catch (error) {
|
|
339
|
-
if (error instanceof SessionMaintenanceError) throw error
|
|
340
|
-
return fail('maintenance_state_invalid', `session maintenance state is unreadable: ${(error as Error).message}`)
|
|
341
|
-
}
|
|
342
|
-
}
|
|
343
|
-
const persist = (state: DurableState): void => {
|
|
344
|
-
const temp = `${statePath}.${process.pid}.${Date.now()}.tmp`
|
|
345
|
-
writeFileSync(temp, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 })
|
|
346
|
-
renameSync(temp, statePath)
|
|
347
|
-
}
|
|
348
|
-
|
|
349
|
-
const recover = (state: DurableState, protectedTicketId?: string): boolean => {
|
|
350
|
-
let dirty = false
|
|
351
|
-
const retained: Ticket[] = []
|
|
352
|
-
for (const ticket of state.tickets) {
|
|
353
|
-
const ownerState = ticket.id === protectedTicketId ? 'live' : identityState(ticket.owner)
|
|
354
|
-
if (ownerState === 'live' || ownerState === 'ambiguous') {
|
|
355
|
-
retained.push(ticket)
|
|
356
|
-
continue
|
|
357
|
-
}
|
|
358
|
-
dirty = true
|
|
359
|
-
if (ticket.mode === 'maintenance') {
|
|
360
|
-
const entry = state.capabilities.find(({ capability }) => capability.op === ticket.operation
|
|
361
|
-
&& capability.sessionId === ticket.sessionId
|
|
362
|
-
&& (capability.op !== 'resume' || capability.force === ticket.force))
|
|
363
|
-
if (entry?.state === 'inflight' && entry.requestId === ticket.id) {
|
|
364
|
-
entry.state = 'indeterminate'
|
|
365
|
-
delete entry.requestId
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
for (const delegate of state.delegates) if (delegate.parentTicketId === ticket.id && delegate.state !== 'completed') delegate.state = 'completed'
|
|
369
|
-
}
|
|
370
|
-
if (retained.length !== state.tickets.length) state.tickets = retained
|
|
371
|
-
|
|
372
|
-
if (state.state !== 'open' && state.tokenHash && state.owner) {
|
|
373
|
-
const ownerState = identityState(state.owner)
|
|
374
|
-
if (ownerState === 'dead' || ownerState === 'reused' || input.now() > (state.heartbeatDeadline ?? -Infinity)) {
|
|
375
|
-
state.state = 'draining'
|
|
376
|
-
state.tokenHash = null
|
|
377
|
-
state.owner = null
|
|
378
|
-
state.heartbeatDeadline = null
|
|
379
|
-
for (const entry of state.capabilities) if (entry.state === 'inflight') {
|
|
380
|
-
entry.state = 'indeterminate'
|
|
381
|
-
delete entry.requestId
|
|
382
|
-
}
|
|
383
|
-
dirty = true
|
|
384
|
-
}
|
|
385
|
-
}
|
|
386
|
-
if (state.state === 'draining' && state.tickets.length === 0) {
|
|
387
|
-
if (state.tokenHash && state.owner && input.now() <= (state.heartbeatDeadline ?? -Infinity)) state.state = 'active'
|
|
388
|
-
else {
|
|
389
|
-
state.state = 'open'
|
|
390
|
-
state.tokenHash = null
|
|
391
|
-
state.owner = null
|
|
392
|
-
state.heartbeatDeadline = null
|
|
393
|
-
state.capabilities = []
|
|
394
|
-
state.delegates = []
|
|
395
|
-
}
|
|
396
|
-
dirty = true
|
|
397
|
-
}
|
|
398
|
-
return dirty
|
|
399
|
-
}
|
|
400
|
-
|
|
401
|
-
const locked = <T>(body: (state: DurableState, dirty: () => void) => T, beforeRecover?: (state: DurableState) => string | undefined): T => {
|
|
402
|
-
const release = acquireLock()
|
|
403
|
-
try {
|
|
404
|
-
const loaded = load()
|
|
405
|
-
let changed = !loaded.exists
|
|
406
|
-
const protectedTicketId = beforeRecover?.(loaded.state)
|
|
407
|
-
if (recover(loaded.state, protectedTicketId)) changed = true
|
|
408
|
-
const value = body(loaded.state, () => { changed = true })
|
|
409
|
-
if (changed) persist(loaded.state)
|
|
410
|
-
return value
|
|
411
|
-
} finally { release() }
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
locked((_state, dirty) => { if (!existsSync(statePath)) dirty() })
|
|
415
|
-
|
|
416
|
-
const verifyLease = (state: DurableState, authorization: Authorization): void => {
|
|
417
|
-
if (state.state !== 'active' || authorization.epoch !== state.epoch)
|
|
418
|
-
fail('maintenance_conflict', 'maintenance lease epoch is not active', { state: state.state, epoch: state.epoch })
|
|
419
|
-
if (!presentedTokenMatches(authorization.token, state.tokenHash))
|
|
420
|
-
fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch })
|
|
421
|
-
}
|
|
422
|
-
const capabilityFor = (state: DurableState, operation: Operation) => state.capabilities.find(({ capability }) =>
|
|
423
|
-
capability.op === operation.op && capability.sessionId === operationSession(operation)
|
|
424
|
-
&& (capability.op !== 'resume' || capability.force === operationForce(operation)))
|
|
425
|
-
|
|
426
|
-
const inheritedOrdinaryParent = (state: DurableState): Ticket | null => {
|
|
427
|
-
const local = context.getStore()?.ticketId
|
|
428
|
-
const inherited = process.env.SPEXCODE_MAINTENANCE_PARENT_TICKET?.trim()
|
|
429
|
-
const id = local || inherited
|
|
430
|
-
if (!id) return null
|
|
431
|
-
const ticket = state.tickets.find((candidate) => candidate.id === id && candidate.mode !== 'maintenance')
|
|
432
|
-
if (!ticket) return null
|
|
433
|
-
if (local) return ticket
|
|
434
|
-
return isDescendantOf(process.pid, ticket.owner, input.processIdentity) ? ticket : null
|
|
435
|
-
}
|
|
436
|
-
|
|
437
|
-
const sharedSpawnAuthority = (state: DurableState, operation: Extract<Operation, { op: 'shared-spawn' }>): { delegate: Delegate; parentTicket: Ticket } => {
|
|
438
|
-
const raw = 'delegate' in operation ? operation.delegate : undefined
|
|
439
|
-
const delegate = typeof raw === 'string' ? state.delegates.find((candidate) => candidate.epoch === state.epoch
|
|
440
|
-
&& candidate.operation === 'shared-spawn' && candidate.sessionId === operation.sessionId
|
|
441
|
-
&& presentedTokenMatches(raw, candidate.tokenHash)) : undefined
|
|
442
|
-
const parentTicket = delegate && state.tickets.find((candidate) => candidate.id === delegate.parentTicketId
|
|
443
|
-
&& candidate.epoch === state.epoch && candidate.operation === 'resume'
|
|
444
|
-
&& candidate.sessionId === operation.sessionId && candidate.mode === 'maintenance')
|
|
445
|
-
if (!delegate || delegate.state !== 'unused' || !parentTicket || identityState(parentTicket.owner) !== 'live')
|
|
446
|
-
return fail('maintenance_delegate_invalid', 'shared-runtime delegate is forged, stale, completed, mismatched, replayed, or detached from its live resume owner', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
447
|
-
return { delegate, parentTicket }
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
const beginTicket = (state: DurableState, operation: Operation, owner: ProcessIdentity, markDirty: () => void): Ticket => {
|
|
451
|
-
if (!OPERATIONS.has(operation.op)) fail('maintenance_invalid', `unknown maintenance operation ${(operation as { op?: string }).op ?? ''}`)
|
|
452
|
-
const parent = inheritedOrdinaryParent(state)
|
|
453
|
-
const ticketId = input.randomBytes(16).toString('hex')
|
|
454
|
-
let mode: Ticket['mode'] = 'ordinary'
|
|
455
|
-
let parentTicketId: string | undefined
|
|
456
|
-
|
|
457
|
-
if (operation.op === 'shared-spawn') {
|
|
458
|
-
if (state.state === 'open') {
|
|
459
|
-
if ('delegate' in operation) fail('maintenance_delegate_invalid', 'shared-runtime delegate channel is not valid outside active maintenance', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
460
|
-
} else if (state.state !== 'active') {
|
|
461
|
-
fail('maintenance_active', `session maintenance is ${state.state}; shared-spawn was not admitted`, { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
462
|
-
} else {
|
|
463
|
-
const { delegate, parentTicket } = sharedSpawnAuthority(state, operation)
|
|
464
|
-
delegate.state = 'running'
|
|
465
|
-
mode = 'maintenance'
|
|
466
|
-
parentTicketId = parentTicket.id
|
|
467
|
-
}
|
|
468
|
-
} else if ((operation.op === 'stop' || operation.op === 'resume') && operation.authorization) {
|
|
469
|
-
verifyLease(state, operation.authorization)
|
|
470
|
-
const entry = capabilityFor(state, operation)
|
|
471
|
-
if (!entry) return fail('maintenance_capability_missing', 'operation is not in the exact maintenance capability plan', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
472
|
-
if (entry.state !== 'unused') fail('maintenance_capability_used', 'maintenance capability is already inflight, committed, or indeterminate', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
473
|
-
entry.state = 'inflight'
|
|
474
|
-
entry.requestId = ticketId
|
|
475
|
-
mode = 'maintenance'
|
|
476
|
-
} else if (state.state !== 'open' && (operation.op === 'stop' || operation.op === 'resume')) {
|
|
477
|
-
if (state.state === 'active')
|
|
478
|
-
return fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
479
|
-
fail('maintenance_active', `session maintenance is ${state.state}; ${operation.op} was not admitted`, { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operation.sessionId })
|
|
480
|
-
} else if (state.state === 'active' && 'authorization' in operation && operation.authorization) {
|
|
481
|
-
verifyLease(state, operation.authorization)
|
|
482
|
-
fail('maintenance_capability_missing', 'operation is not in the exact maintenance capability plan', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
483
|
-
} else if (state.state !== 'open' && parent) parentTicketId = parent.id
|
|
484
|
-
else if (state.state !== 'open') {
|
|
485
|
-
if ('delegate' in operation) fail('maintenance_delegate_invalid', 'shared-runtime delegate is not valid for this operation', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
486
|
-
fail('maintenance_active', `session maintenance is ${state.state}; ${operation.op} was not admitted`, { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
const ticket: Ticket = {
|
|
490
|
-
id: ticketId, epoch: state.epoch, operation: operation.op,
|
|
491
|
-
...(operationSession(operation) ? { sessionId: operationSession(operation) } : {}),
|
|
492
|
-
...(operationForce(operation) !== undefined ? { force: operationForce(operation) } : {}),
|
|
493
|
-
owner, deadline: input.now() + input.ticketReportMs, mode,
|
|
494
|
-
...(parentTicketId ? { parentTicketId } : {}),
|
|
495
|
-
}
|
|
496
|
-
state.tickets.push(ticket)
|
|
497
|
-
markDirty()
|
|
498
|
-
return ticket
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
const beginOperationTicket = (operation: Operation, owner: ProcessIdentity): Ticket => locked(
|
|
502
|
-
(state, dirty) => beginTicket(state, operation, owner, dirty),
|
|
503
|
-
(state) => {
|
|
504
|
-
if (operation.op !== 'shared-spawn' || state.state !== 'active') return undefined
|
|
505
|
-
// Protect the proven-live parent from recovery, then re-read it in beginTicket before delegate mutation.
|
|
506
|
-
return sharedSpawnAuthority(state, operation).parentTicket.id
|
|
507
|
-
},
|
|
508
|
-
)
|
|
509
|
-
|
|
510
|
-
const finishTicket = (ticketId: string, outcome: 'committed' | 'retryable' | 'indeterminate'): void => locked((state, dirty) => {
|
|
511
|
-
const index = state.tickets.findIndex((ticket) => ticket.id === ticketId)
|
|
512
|
-
if (index < 0) return
|
|
513
|
-
const [ticket] = state.tickets.splice(index, 1)
|
|
514
|
-
if (ticket.mode === 'maintenance' && (ticket.operation === 'stop' || ticket.operation === 'resume')) {
|
|
515
|
-
const entry = state.capabilities.find(({ capability }) => capability.op === ticket.operation
|
|
516
|
-
&& capability.sessionId === ticket.sessionId && (capability.op !== 'resume' || capability.force === ticket.force))
|
|
517
|
-
if (entry?.state === 'inflight' && entry.requestId === ticket.id) {
|
|
518
|
-
entry.state = outcome === 'retryable' ? 'unused' : outcome
|
|
519
|
-
delete entry.requestId
|
|
520
|
-
}
|
|
521
|
-
}
|
|
522
|
-
if (ticket.operation === 'shared-spawn' && ticket.parentTicketId) {
|
|
523
|
-
const delegate = state.delegates.find((candidate) => candidate.parentTicketId === ticket.parentTicketId
|
|
524
|
-
&& candidate.sessionId === ticket.sessionId && candidate.state === 'running')
|
|
525
|
-
if (delegate) delegate.state = 'completed'
|
|
526
|
-
}
|
|
527
|
-
dirty()
|
|
528
|
-
})
|
|
529
|
-
|
|
530
|
-
const delegateForTicket = (ticketId: string, sessionId: string): string => locked((state, dirty) => {
|
|
531
|
-
const ticket = state.tickets.find((candidate) => candidate.id === ticketId && candidate.operation === 'resume'
|
|
532
|
-
&& candidate.mode === 'maintenance' && candidate.sessionId === sessionId && candidate.epoch === state.epoch)
|
|
533
|
-
if (!ticket || state.state !== 'active') return fail('maintenance_delegate_invalid', 'resume ticket is not live for this session')
|
|
534
|
-
if (state.delegates.some((delegate) => delegate.parentTicketId === ticket.id))
|
|
535
|
-
fail('maintenance_delegate_invalid', 'resume ticket already delegated its shared spawn')
|
|
536
|
-
const token = input.randomBytes(32).toString('hex')
|
|
537
|
-
state.delegates.push({ tokenHash: hashToken(token), parentTicketId: ticket.id, epoch: state.epoch, operation: 'shared-spawn', sessionId, state: 'unused' })
|
|
538
|
-
dirty()
|
|
539
|
-
return token
|
|
540
|
-
})
|
|
541
|
-
|
|
542
|
-
const readState = (): MaintenanceState => locked((state) => ({
|
|
543
|
-
state: state.state,
|
|
544
|
-
epoch: state.epoch,
|
|
545
|
-
owner: copy(state.owner),
|
|
546
|
-
heartbeatDeadline: state.heartbeatDeadline,
|
|
547
|
-
tickets: copy(state.tickets.map((ticket) => ({
|
|
548
|
-
operation: ticket.operation,
|
|
549
|
-
...(ticket.sessionId ? { sessionId: ticket.sessionId } : {}),
|
|
550
|
-
...(ticket.force !== undefined ? { force: ticket.force } : {}),
|
|
551
|
-
owner: ticket.owner,
|
|
552
|
-
deadline: ticket.deadline,
|
|
553
|
-
}))),
|
|
554
|
-
capabilities: copy(state.capabilities),
|
|
555
|
-
}))
|
|
556
|
-
|
|
557
|
-
const acquireLease = async ({ capabilities: rawCapabilities, owner, ttlMs, waitMs }: { capabilities: readonly Capability[]; owner: LeaseOwner; ttlMs: number; waitMs: number }): Promise<MaintenanceLease> => {
|
|
558
|
-
if (!Number.isFinite(ttlMs) || ttlMs < MIN_TTL_MS || ttlMs > MAX_TTL_MS || !Number.isFinite(waitMs) || waitMs < 0 || waitMs > MAX_WAIT_MS || waitMs > ttlMs)
|
|
559
|
-
fail('maintenance_invalid', 'maintenance ttl/wait is outside the finite bounds')
|
|
560
|
-
if (identityState(owner) !== 'live') fail('maintenance_invalid', 'maintenance lease owner identity is not exact and live')
|
|
561
|
-
const capabilities = canonicalCapabilities(rawCapabilities)
|
|
562
|
-
const token = input.randomBytes(32).toString('hex')
|
|
563
|
-
const started = locked((state, dirty) => {
|
|
564
|
-
if (state.state !== 'open') fail('maintenance_conflict', `session maintenance is already ${state.state}`, { state: state.state, epoch: state.epoch })
|
|
565
|
-
state.state = 'draining'
|
|
566
|
-
state.epoch += 1
|
|
567
|
-
state.tokenHash = hashToken(token)
|
|
568
|
-
state.owner = copy(owner)
|
|
569
|
-
state.heartbeatDeadline = input.now() + ttlMs
|
|
570
|
-
state.capabilities = capabilities.map((capability) => ({ capability, state: 'unused' }))
|
|
571
|
-
state.delegates = []
|
|
572
|
-
if (state.tickets.length === 0) state.state = 'active'
|
|
573
|
-
dirty()
|
|
574
|
-
return {
|
|
575
|
-
token,
|
|
576
|
-
epoch: state.epoch,
|
|
577
|
-
state: state.state as MaintenanceLease['state'],
|
|
578
|
-
owner: copy(owner),
|
|
579
|
-
capabilities: copy(capabilities),
|
|
580
|
-
}
|
|
581
|
-
})
|
|
582
|
-
input.onEvent?.({ type: 'maintenance-acquire', state: started.state, epoch: started.epoch })
|
|
583
|
-
if (started.state === 'active' || waitMs === 0) return started
|
|
584
|
-
const deadline = input.now() + waitMs
|
|
585
|
-
for (;;) {
|
|
586
|
-
const current = readState()
|
|
587
|
-
if (current.epoch !== started.epoch || current.state === 'open') fail('maintenance_conflict', 'maintenance acquisition lost its epoch')
|
|
588
|
-
if (current.state === 'active') return { ...started, state: 'active' }
|
|
589
|
-
if (input.now() >= deadline) return started
|
|
590
|
-
await new Promise((resolve) => setTimeout(resolve, 10))
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
|
|
594
|
-
const heartbeatLease = async ({ token, epoch, ttlMs }: { token: string; epoch: number; ttlMs: number }): Promise<void> => {
|
|
595
|
-
if (!Number.isFinite(ttlMs) || ttlMs < MIN_TTL_MS || ttlMs > MAX_TTL_MS) fail('maintenance_invalid', 'maintenance heartbeat ttl is outside the finite bounds')
|
|
596
|
-
locked((state, dirty) => {
|
|
597
|
-
if (state.epoch !== epoch || (state.state !== 'active' && state.state !== 'draining')) fail('maintenance_conflict', 'maintenance heartbeat names a stale epoch', { state: state.state, epoch: state.epoch })
|
|
598
|
-
if (!presentedTokenMatches(token, state.tokenHash)) fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch })
|
|
599
|
-
state.heartbeatDeadline = input.now() + ttlMs
|
|
600
|
-
dirty()
|
|
601
|
-
})
|
|
602
|
-
input.onEvent?.({ type: 'maintenance-heartbeat', epoch })
|
|
603
|
-
}
|
|
604
|
-
|
|
605
|
-
const releaseLease = async ({ token, epoch }: { token: string; epoch: number }): Promise<void> => {
|
|
606
|
-
locked((state, dirty) => {
|
|
607
|
-
if (state.epoch !== epoch || state.state === 'open') fail('maintenance_conflict', 'maintenance release names a stale epoch', { state: state.state, epoch: state.epoch })
|
|
608
|
-
if (!presentedTokenMatches(token, state.tokenHash)) fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch })
|
|
609
|
-
if (state.tickets.length) fail('maintenance_tickets_live', 'maintenance cannot release while exact live or ambiguous tickets remain', { state: state.state, epoch: state.epoch })
|
|
610
|
-
state.state = 'open'
|
|
611
|
-
state.tokenHash = null
|
|
612
|
-
state.owner = null
|
|
613
|
-
state.heartbeatDeadline = null
|
|
614
|
-
state.capabilities = []
|
|
615
|
-
state.delegates = []
|
|
616
|
-
dirty()
|
|
617
|
-
})
|
|
618
|
-
input.onEvent?.({ type: 'maintenance-release', epoch })
|
|
619
|
-
}
|
|
620
|
-
|
|
621
|
-
const runOperation = async <T>(operation: Operation, body: (ticket: MaintenanceTicket) => Promise<T> | T): Promise<T> => {
|
|
622
|
-
const owner = input.selfIdentity()
|
|
623
|
-
const ticket = beginOperationTicket(operation, owner)
|
|
624
|
-
let outcome: 'committed' | 'retryable' | 'indeterminate' = 'indeterminate'
|
|
625
|
-
try {
|
|
626
|
-
const result = await context.run({ ticketId: ticket.id }, () => body({
|
|
627
|
-
id: ticket.id,
|
|
628
|
-
epoch: ticket.epoch,
|
|
629
|
-
delegateSharedSpawn: (sessionId: string) => delegateForTicket(ticket.id, sessionId),
|
|
630
|
-
}))
|
|
631
|
-
outcome = (operation.op === 'stop' || operation.op === 'resume')
|
|
632
|
-
&& (result === false || (!!result && typeof result === 'object' && 'ok' in result && (result as { ok?: unknown }).ok === false))
|
|
633
|
-
? 'retryable'
|
|
634
|
-
: 'committed'
|
|
635
|
-
return result
|
|
636
|
-
} finally {
|
|
637
|
-
finishTicket(ticket.id, outcome)
|
|
638
|
-
}
|
|
639
|
-
}
|
|
640
|
-
|
|
641
|
-
const runOperationSync = <T>(operation: Operation, body: (ticket: MaintenanceTicket) => T): T => {
|
|
642
|
-
const owner = input.selfIdentity()
|
|
643
|
-
const ticket = beginOperationTicket(operation, owner)
|
|
644
|
-
let outcome: 'committed' | 'retryable' | 'indeterminate' = 'indeterminate'
|
|
645
|
-
try {
|
|
646
|
-
const result = context.run({ ticketId: ticket.id }, () => body({
|
|
647
|
-
id: ticket.id,
|
|
648
|
-
epoch: ticket.epoch,
|
|
649
|
-
delegateSharedSpawn: (sessionId: string) => delegateForTicket(ticket.id, sessionId),
|
|
650
|
-
}))
|
|
651
|
-
outcome = (operation.op === 'stop' || operation.op === 'resume')
|
|
652
|
-
&& (result === false || (!!result && typeof result === 'object' && 'ok' in result && (result as { ok?: unknown }).ok === false))
|
|
653
|
-
? 'retryable'
|
|
654
|
-
: 'committed'
|
|
655
|
-
return result
|
|
656
|
-
} finally {
|
|
657
|
-
finishTicket(ticket.id, outcome)
|
|
658
|
-
}
|
|
659
|
-
}
|
|
660
|
-
|
|
661
|
-
const authorizeHttpOperation = async ({ authenticated, projectMatches, headers, operation }: { authenticated: boolean; projectMatches: boolean; headers: Record<string, string>; operation: Operation }): Promise<Authorization | undefined> => {
|
|
662
|
-
if (!authenticated) fail('unauthorized', 'authentication is required before maintenance admission')
|
|
663
|
-
if (!projectMatches) fail('project_mismatch', 'request project does not match the maintenance project')
|
|
664
|
-
const token = Object.entries(headers).find(([name]) => name.toLowerCase() === 'x-spexcode-session-maintenance')?.[1] ?? ''
|
|
665
|
-
return locked((state) => {
|
|
666
|
-
if (!token) {
|
|
667
|
-
if (state.state === 'active') fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
668
|
-
return undefined
|
|
669
|
-
}
|
|
670
|
-
if (state.state !== 'active') fail('maintenance_conflict', 'maintenance lease is not active', { state: state.state, epoch: state.epoch })
|
|
671
|
-
if (!presentedTokenMatches(token, state.tokenHash)) fail('maintenance_token_invalid', 'maintenance bearer is missing, wrong, or stale', { state: state.state, epoch: state.epoch })
|
|
672
|
-
const entry = capabilityFor(state, operation)
|
|
673
|
-
if (!entry) return fail('maintenance_capability_missing', 'operation is not in the exact maintenance capability plan', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
674
|
-
if (entry.state !== 'unused') fail('maintenance_capability_used', 'maintenance capability is already inflight, committed, or indeterminate', { state: state.state, epoch: state.epoch, operation: operation.op, sessionId: operationSession(operation) })
|
|
675
|
-
return { token, epoch: state.epoch }
|
|
676
|
-
})
|
|
677
|
-
}
|
|
678
|
-
|
|
679
|
-
const beginExternalOperation = (operation: Operation, owner: ProcessIdentity): string => {
|
|
680
|
-
if (identityState(owner) !== 'live') fail('maintenance_invalid', 'external operation owner is not exact and live')
|
|
681
|
-
return beginOperationTicket(operation, owner).id
|
|
682
|
-
}
|
|
683
|
-
const finishExternalOperation = (ticketId: string, owner: ProcessIdentity): void => locked((state, dirty) => {
|
|
684
|
-
const index = state.tickets.findIndex((ticket) => ticket.id === ticketId && sameIdentity(ticket.owner, owner))
|
|
685
|
-
if (index < 0) return
|
|
686
|
-
state.tickets.splice(index, 1)
|
|
687
|
-
dirty()
|
|
688
|
-
})
|
|
689
|
-
|
|
690
|
-
return {
|
|
691
|
-
headerName: 'X-SpexCode-Session-Maintenance',
|
|
692
|
-
runOperation, runOperationSync, acquireLease, heartbeatLease, releaseLease, readState, authorizeHttpOperation,
|
|
693
|
-
beginExternalOperation, finishExternalOperation,
|
|
694
|
-
}
|
|
695
|
-
}
|
|
696
|
-
|
|
697
|
-
function productionProcessIdentity(pid: number): ProcessReading {
|
|
698
|
-
try { process.kill(pid, 0) } catch (error) {
|
|
699
|
-
if ((error as NodeJS.ErrnoException).code === 'ESRCH') return null
|
|
700
|
-
return 'ambiguous'
|
|
701
|
-
}
|
|
702
|
-
const startToken = processStartToken(pid)
|
|
703
|
-
return startToken ? { pid, startToken } : 'ambiguous'
|
|
704
|
-
}
|
|
705
|
-
|
|
706
|
-
export function exactProcessIdentity(pid: number): ProcessIdentity | null {
|
|
707
|
-
const reading = productionProcessIdentity(pid)
|
|
708
|
-
return reading && reading !== 'ambiguous' ? reading : null
|
|
709
|
-
}
|
|
710
|
-
|
|
711
|
-
const coordinators = new Map<string, ReturnType<typeof createSessionMaintenance>>()
|
|
712
|
-
export function sessionMaintenance(root = projectRuntimeRoot()): ReturnType<typeof createSessionMaintenance> {
|
|
713
|
-
let coordinator = coordinators.get(root)
|
|
714
|
-
if (coordinator) return coordinator
|
|
715
|
-
const detectedStart = processStartToken(process.pid)
|
|
716
|
-
if (!detectedStart) throw new SessionMaintenanceError('maintenance_identity_unknown', `cannot read exact process identity for ${process.pid}`)
|
|
717
|
-
const selfStart: string = detectedStart
|
|
718
|
-
coordinator = createSessionMaintenance({
|
|
719
|
-
runtimeRoot: root,
|
|
720
|
-
now: () => Date.now(),
|
|
721
|
-
randomBytes: cryptoRandomBytes,
|
|
722
|
-
processIdentity: productionProcessIdentity,
|
|
723
|
-
selfIdentity: () => ({ pid: process.pid, startToken: selfStart }),
|
|
724
|
-
ticketReportMs: 30_000,
|
|
725
|
-
})
|
|
726
|
-
coordinators.set(root, coordinator)
|
|
727
|
-
return coordinator
|
|
728
|
-
}
|
|
729
|
-
|
|
730
|
-
export const runSessionOperation = <T>(operation: Operation, body: (ticket: MaintenanceTicket) => Promise<T> | T): Promise<T> =>
|
|
731
|
-
sessionMaintenance().runOperation(operation, body)
|
|
732
|
-
export const runSessionOperationSync = <T>(operation: Operation, body: (ticket: MaintenanceTicket) => T): T =>
|
|
733
|
-
sessionMaintenance().runOperationSync(operation, body)
|
|
734
|
-
|
|
735
|
-
export function maintenanceBrokerDescriptors(): { request: number; response: number; turn: number } | null {
|
|
736
|
-
const values = (process.env.SPEXCODE_MAINTENANCE_BROKER_FDS || '').split(',').map(Number)
|
|
737
|
-
if (values.length !== 3 || values.some((fd) => !Number.isInteger(fd) || fd < 3) || new Set(values).size !== 3) return null
|
|
738
|
-
return { request: values[0], response: values[1], turn: values[2] }
|
|
739
|
-
}
|
|
740
|
-
|
|
741
|
-
export function maintenanceErrorPayload(error: unknown): Record<string, unknown> | null {
|
|
742
|
-
if (!(error instanceof SessionMaintenanceError)) return null
|
|
743
|
-
return {
|
|
744
|
-
error: error.message,
|
|
745
|
-
code: error.code,
|
|
746
|
-
...(error.state ? { state: error.state } : {}),
|
|
747
|
-
...(error.epoch !== undefined ? { epoch: error.epoch } : {}),
|
|
748
|
-
...(error.operation ? { operation: error.operation } : {}),
|
|
749
|
-
...(error.sessionId ? { sessionId: error.sessionId } : {}),
|
|
750
|
-
}
|
|
751
|
-
}
|