ultra-builder-pro-cli 0.20.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.ultra-template/docs/research/README.md +26 -5
- package/.ultra-template/specs/discovery.md +1 -1
- package/.ultra-template/tasks/tasks.json +10 -3
- package/.ultra-template/templates/task-context.md +19 -0
- package/CHANGELOG.md +106 -0
- package/README.md +215 -41
- package/adapters/_shared/codex-assets.cjs +69 -11
- package/adapters/_shared/interaction-contract.cjs +36 -3
- package/adapters/_shared/runtime-assets.cjs +41 -1
- package/adapters/claude.js +7 -5
- package/adapters/codex.js +1 -1
- package/adapters/kimi.js +2 -8
- package/adapters/opencode.js +43 -30
- package/bin/orchestrator.js +64 -24
- package/docs/AGENT-CONTEXT.md +89 -49
- package/docs/ARCHITECTURE.md +112 -45
- package/docs/ARTIFACT-AUTHORITY.md +274 -0
- package/docs/COMMIT-HASH-BACKFILL.md +33 -90
- package/docs/DECISIONS.md +14 -6
- package/docs/LEGACY-CLI-CROSSWALK.md +16 -6
- package/docs/PLUGIN-ISOLATION-CONTRACT.md +6 -4
- package/docs/ROADMAP.md +20 -6
- package/docs/RUNTIME-COMPAT-MATRIX.md +29 -16
- package/docs/STATE-DB-ACCESS-POLICY.md +43 -13
- package/docs/WORKFLOW-LIFECYCLE.md +105 -50
- package/hooks/README.md +10 -8
- package/hooks/active_task_context.py +105 -18
- package/hooks/context_spine.py +16 -4
- package/hooks/health_check.py +5 -4
- package/hooks/pre_stop_check.py +5 -2
- package/hooks/runtime_paths.py +396 -0
- package/hooks/subagent_tracker.py +5 -2
- package/hooks/workflow_checkpoint.py +16 -3
- package/hooks/workflow_context.py +11 -3
- package/hooks/workflow_resume.py +7 -4
- package/mcp-server/breadcrumb.cjs +20 -2
- package/mcp-server/lib/archive-journal.cjs +1026 -111
- package/mcp-server/lib/archive-mutation-worker.py +365 -0
- package/mcp-server/lib/artifact-registry.cjs +1127 -0
- package/mcp-server/lib/baseline-workflow.cjs +69 -28
- package/mcp-server/lib/change-packet.cjs +818 -0
- package/mcp-server/lib/change-workflow.cjs +932 -176
- package/mcp-server/lib/context-paths.cjs +142 -0
- package/mcp-server/lib/context-spine.cjs +511 -45
- package/mcp-server/lib/decision-dialogue.cjs +345 -3
- package/mcp-server/lib/delivery-transaction.cjs +471 -0
- package/mcp-server/lib/doctor-backup-worker.cjs +328 -0
- package/mcp-server/lib/doctor.cjs +401 -11
- package/mcp-server/lib/git-bootstrap.cjs +458 -47
- package/mcp-server/lib/init-project.cjs +240 -95
- package/mcp-server/lib/plan-publication-worker.cjs +830 -0
- package/mcp-server/lib/plan-store.cjs +700 -5
- package/mcp-server/lib/project-breadcrumb.cjs +23 -4
- package/mcp-server/lib/projector.cjs +211 -42
- package/mcp-server/lib/runtime-paths.cjs +1687 -0
- package/mcp-server/lib/safe-project-file.cjs +385 -0
- package/mcp-server/lib/spec-learning.cjs +28 -12
- package/mcp-server/lib/state-authority.cjs +19 -3
- package/mcp-server/lib/state-db.cjs +263 -9
- package/mcp-server/lib/state-ops.cjs +254 -50
- package/mcp-server/lib/task-ledger.cjs +1342 -0
- package/mcp-server/lib/telemetry.cjs +12 -3
- package/mcp-server/lib/workflow-state.cjs +610 -58
- package/mcp-server/server.cjs +423 -42
- package/orchestrator/daemon.cjs +2 -10
- package/orchestrator/parallel-orchestrator.cjs +314 -50
- package/orchestrator/planner/plan-builder.cjs +1 -1
- package/orchestrator/recovery.cjs +125 -7
- package/orchestrator/session-close-journal-worker.cjs +5 -0
- package/orchestrator/session-close-journal.cjs +1014 -0
- package/orchestrator/session-runner.cjs +679 -69
- package/orchestrator/worktree-manager.cjs +186 -35
- package/package.json +2 -3
- package/skills/cc-collab/SKILL.md +4 -1
- package/skills/codex-collab/SKILL.md +5 -2
- package/skills/ultra-change/SKILL.md +34 -8
- package/skills/ultra-deliver/SKILL.md +26 -11
- package/skills/ultra-dev/SKILL.md +52 -10
- package/skills/ultra-doctor/SKILL.md +16 -3
- package/skills/ultra-init/SKILL.md +22 -11
- package/skills/ultra-plan/SKILL.md +58 -18
- package/skills/ultra-research/SKILL.md +37 -35
- package/skills/ultra-review/SKILL.md +14 -8
- package/skills/ultra-status/SKILL.md +18 -9
- package/skills/ultra-test/SKILL.md +11 -5
- package/skills/ultra-think/SKILL.md +6 -2
- package/skills/ultra-think/references/decision-dialogue.md +17 -6
- package/skills/ultra-verify/SKILL.md +4 -1
- package/spec/cli-protocol.md +21 -7
- package/spec/mcp-tools.yaml +629 -44
- package/spec/schemas/change-delta.v1.schema.json +153 -0
- package/spec/schemas/context-file.v4.5.schema.json +1 -1
- package/spec/schemas/documentation-reconciliation.v1.schema.json +106 -0
- package/spec/schemas/mcp-tools.schema.json +1 -1
- package/spec/schemas/state-db.sql +51 -14
- package/spec/schemas/task-ledger.v1.schema.json +217 -0
- package/spec/schemas/tasks.v4.5.schema.json +1 -1
- package/ultra-tools/cli.cjs +3 -2
- package/ultra-tools/commands/db.cjs +76 -25
- package/ultra-tools/commands/migrate.cjs +1000 -82
- package/ultra-tools/commands/session.cjs +34 -8
- package/ultra-tools/commands/status.cjs +32 -6
- package/ultra-tools/commands/system.cjs +129 -22
- package/.ultra-template/tasks/contexts/TEMPLATE.md +0 -19
- package/commands/learn.md +0 -10
- package/orchestrator/skill-miner.cjs +0 -133
- package/output-styles/coding-vibes.md +0 -67
- package/output-styles/structural-thinking.md +0 -61
- package/skills/learn/SKILL.md +0 -58
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
'use strict';
|
|
2
2
|
|
|
3
|
-
// State-operations layer over .ultra/state.db.
|
|
3
|
+
// State-operations layer over .ultra/.runtime/state.db.
|
|
4
4
|
//
|
|
5
5
|
// Every mutation in the Ultra Builder Pro runtime goes through this module.
|
|
6
6
|
// Callers (MCP server tool handlers, the migration CLI, the orchestrator)
|
|
@@ -57,6 +57,14 @@ const TASK_CONTRACT_PATCH_FIELDS = new Set([
|
|
|
57
57
|
const SESSION_PATCHABLE = Object.freeze([
|
|
58
58
|
'pid', 'status', 'lease_expires_at', 'heartbeat_at', 'worktree_path', 'artifact_dir',
|
|
59
59
|
]);
|
|
60
|
+
const SESSION_STATUSES = new Set(['running', 'completed', 'crashed', 'orphan']);
|
|
61
|
+
const SESSION_TERMINAL_STATUSES = new Set(['completed', 'crashed']);
|
|
62
|
+
const SESSION_STATUS_TRANSITIONS = Object.freeze({
|
|
63
|
+
running: new Set(['completed', 'crashed', 'orphan']),
|
|
64
|
+
orphan: new Set(['completed', 'crashed']),
|
|
65
|
+
completed: new Set(),
|
|
66
|
+
crashed: new Set(),
|
|
67
|
+
});
|
|
60
68
|
|
|
61
69
|
class StateOpsError extends Error {
|
|
62
70
|
constructor(code, message, { retriable = false, details } = {}) {
|
|
@@ -170,9 +178,44 @@ function normalizeContextRefs(value) {
|
|
|
170
178
|
if (!ref || !reason) {
|
|
171
179
|
throw new StateOpsError('VALIDATION_ERROR', `context_refs[${index}] requires ref and reason`);
|
|
172
180
|
}
|
|
173
|
-
|
|
174
|
-
|
|
181
|
+
const kind = String(item.kind || 'source').trim();
|
|
182
|
+
if (!['spec', 'source', 'test', 'docs', 'external'].includes(kind)) {
|
|
183
|
+
throw new StateOpsError('VALIDATION_ERROR', `context_refs[${index}].kind is unsupported`);
|
|
184
|
+
}
|
|
185
|
+
const expectedDigest = item.expected_digest ?? item.digest ?? null;
|
|
186
|
+
if (expectedDigest !== null && !/^[0-9a-f]{64}$/.test(String(expectedDigest))) {
|
|
187
|
+
throw new StateOpsError(
|
|
188
|
+
'VALIDATION_ERROR', `context_refs[${index}].expected_digest must be sha256`,
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
const freshnessPolicy = String(
|
|
192
|
+
item.freshness_policy || (expectedDigest ? 'digest' : 'existence'),
|
|
193
|
+
).trim();
|
|
194
|
+
if (!['digest', 'existence', 'advisory'].includes(freshnessPolicy)) {
|
|
195
|
+
throw new StateOpsError(
|
|
196
|
+
'VALIDATION_ERROR',
|
|
197
|
+
`context_refs[${index}].freshness_policy must be digest, existence, or advisory`,
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
const normalized = {
|
|
201
|
+
ref,
|
|
202
|
+
reason,
|
|
203
|
+
kind,
|
|
204
|
+
required: item.required !== false,
|
|
205
|
+
freshness_policy: freshnessPolicy,
|
|
175
206
|
};
|
|
207
|
+
if (expectedDigest !== null) normalized.expected_digest = String(expectedDigest);
|
|
208
|
+
if (item.anchor !== undefined) {
|
|
209
|
+
const anchor = String(item.anchor).trim();
|
|
210
|
+
if (!anchor) throw new StateOpsError('VALIDATION_ERROR', `context_refs[${index}].anchor cannot be empty`);
|
|
211
|
+
normalized.anchor = anchor;
|
|
212
|
+
}
|
|
213
|
+
if (item.scope !== undefined) {
|
|
214
|
+
const scope = String(item.scope).trim();
|
|
215
|
+
if (!scope) throw new StateOpsError('VALIDATION_ERROR', `context_refs[${index}].scope cannot be empty`);
|
|
216
|
+
normalized.scope = scope;
|
|
217
|
+
}
|
|
218
|
+
return normalized;
|
|
176
219
|
});
|
|
177
220
|
}
|
|
178
221
|
|
|
@@ -475,7 +518,12 @@ function patchTask(db, id, patch = {}) {
|
|
|
475
518
|
}
|
|
476
519
|
if (nextStatus !== null) {
|
|
477
520
|
const allowed = STATUS_TRANSITIONS[current.status] || new Set();
|
|
478
|
-
|
|
521
|
+
const recoveringUncommittedCompletion = current.status === 'completed'
|
|
522
|
+
&& nextStatus === 'blocked'
|
|
523
|
+
&& !current.completion_commit
|
|
524
|
+
&& Boolean(current.session_id);
|
|
525
|
+
if (!allowed.has(nextStatus) && nextStatus !== current.status
|
|
526
|
+
&& !recoveringUncommittedCompletion) {
|
|
479
527
|
throw new StateOpsError(
|
|
480
528
|
'ILLEGAL_STATUS_TRANSITION',
|
|
481
529
|
`cannot transition task ${id} from ${current.status} to ${nextStatus}`,
|
|
@@ -609,10 +657,32 @@ function createSession(db, { sid, task_id, runtime, pid = null, worktree_path, a
|
|
|
609
657
|
if (!readTask(db, task_id)) {
|
|
610
658
|
throw new StateOpsError('TASK_NOT_FOUND', `task ${task_id} does not exist`);
|
|
611
659
|
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
660
|
+
try {
|
|
661
|
+
db.prepare(
|
|
662
|
+
`INSERT INTO sessions (sid, task_id, runtime, pid, worktree_path, artifact_dir, status, lease_expires_at)
|
|
663
|
+
VALUES (?, ?, ?, ?, ?, ?, 'running', ?)`,
|
|
664
|
+
).run(sid, task_id, runtime, pid, worktree_path, artifact_dir, lease);
|
|
665
|
+
} catch (error) {
|
|
666
|
+
if (String(error?.code || '').startsWith('SQLITE_CONSTRAINT')) {
|
|
667
|
+
const conflict = db.prepare(
|
|
668
|
+
"SELECT sid, status FROM sessions WHERE task_id = ? AND status = 'running' LIMIT 1",
|
|
669
|
+
).get(task_id);
|
|
670
|
+
if (conflict) {
|
|
671
|
+
throw new StateOpsError(
|
|
672
|
+
'ADMISSION_DENIED',
|
|
673
|
+
`active session exists for task ${task_id} (${conflict.sid})`,
|
|
674
|
+
{
|
|
675
|
+
details: {
|
|
676
|
+
task_id,
|
|
677
|
+
conflict_sid: conflict.sid,
|
|
678
|
+
conflict_status: conflict.status,
|
|
679
|
+
},
|
|
680
|
+
},
|
|
681
|
+
);
|
|
682
|
+
}
|
|
683
|
+
}
|
|
684
|
+
throw error;
|
|
685
|
+
}
|
|
616
686
|
appendEventInTx(db, {
|
|
617
687
|
type: 'session_spawned',
|
|
618
688
|
task_id,
|
|
@@ -628,29 +698,104 @@ function updateSession(db, sid, patch = {}) {
|
|
|
628
698
|
return tx(db, () => {
|
|
629
699
|
const cur = db.prepare('SELECT * FROM sessions WHERE sid = ?').get(sid);
|
|
630
700
|
if (!cur) throw new StateOpsError('SESSION_NOT_FOUND', `no session ${sid}`);
|
|
701
|
+
const requestedStatus = patch.status;
|
|
702
|
+
if (requestedStatus !== undefined && !SESSION_STATUSES.has(requestedStatus)) {
|
|
703
|
+
throw new StateOpsError(
|
|
704
|
+
'VALIDATION_ERROR',
|
|
705
|
+
`unsupported session status: ${String(requestedStatus)}`,
|
|
706
|
+
);
|
|
707
|
+
}
|
|
708
|
+
if (requestedStatus !== undefined && requestedStatus !== cur.status) {
|
|
709
|
+
if (SESSION_TERMINAL_STATUSES.has(cur.status)) {
|
|
710
|
+
throw new StateOpsError(
|
|
711
|
+
'SESSION_TERMINAL_CONFLICT',
|
|
712
|
+
`session ${sid} is already ${cur.status} and cannot transition to ${requestedStatus}`,
|
|
713
|
+
{
|
|
714
|
+
details: {
|
|
715
|
+
sid,
|
|
716
|
+
current_status: cur.status,
|
|
717
|
+
requested_status: requestedStatus,
|
|
718
|
+
},
|
|
719
|
+
},
|
|
720
|
+
);
|
|
721
|
+
}
|
|
722
|
+
if (!SESSION_STATUS_TRANSITIONS[cur.status]?.has(requestedStatus)) {
|
|
723
|
+
throw new StateOpsError(
|
|
724
|
+
'SESSION_STATUS_TRANSITION_INVALID',
|
|
725
|
+
`session ${sid} cannot transition from ${cur.status} to ${requestedStatus}`,
|
|
726
|
+
{
|
|
727
|
+
details: {
|
|
728
|
+
sid,
|
|
729
|
+
current_status: cur.status,
|
|
730
|
+
requested_status: requestedStatus,
|
|
731
|
+
},
|
|
732
|
+
},
|
|
733
|
+
);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
const normalizedPatch = (
|
|
737
|
+
SESSION_TERMINAL_STATUSES.has(requestedStatus)
|
|
738
|
+
)
|
|
739
|
+
? { ...patch, pid: null }
|
|
740
|
+
: patch;
|
|
631
741
|
const sets = [];
|
|
632
742
|
const params = [];
|
|
633
|
-
for (const key of Object.keys(
|
|
743
|
+
for (const key of Object.keys(normalizedPatch)) {
|
|
634
744
|
if (!SESSION_PATCHABLE.includes(key)) {
|
|
635
745
|
throw new StateOpsError('VALIDATION_ERROR', `session field ${key} is not patchable`);
|
|
636
746
|
}
|
|
637
747
|
sets.push(`${key} = ?`);
|
|
638
|
-
params.push(
|
|
748
|
+
params.push(normalizedPatch[key]);
|
|
639
749
|
}
|
|
640
750
|
if (sets.length === 0) return cur;
|
|
641
|
-
params.push(sid);
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
751
|
+
params.push(sid, cur.status);
|
|
752
|
+
const result = db.prepare(
|
|
753
|
+
`UPDATE sessions SET ${sets.join(', ')} WHERE sid = ? AND status = ?`,
|
|
754
|
+
).run(...params);
|
|
755
|
+
if (result.changes !== 1) {
|
|
756
|
+
const latest = db.prepare('SELECT status FROM sessions WHERE sid = ?').get(sid);
|
|
757
|
+
if (latest && SESSION_TERMINAL_STATUSES.has(latest.status)
|
|
758
|
+
&& requestedStatus !== undefined && latest.status !== requestedStatus) {
|
|
759
|
+
throw new StateOpsError(
|
|
760
|
+
'SESSION_TERMINAL_CONFLICT',
|
|
761
|
+
`session ${sid} is already ${latest.status} and cannot transition to ${requestedStatus}`,
|
|
762
|
+
{
|
|
763
|
+
details: {
|
|
764
|
+
sid,
|
|
765
|
+
current_status: latest.status,
|
|
766
|
+
requested_status: requestedStatus,
|
|
767
|
+
},
|
|
768
|
+
},
|
|
769
|
+
);
|
|
770
|
+
}
|
|
771
|
+
throw new StateOpsError(
|
|
772
|
+
'SESSION_STATUS_CONFLICT',
|
|
773
|
+
`session ${sid} changed while applying an update`,
|
|
774
|
+
{
|
|
775
|
+
retriable: true,
|
|
776
|
+
details: {
|
|
777
|
+
sid,
|
|
778
|
+
expected_status: cur.status,
|
|
779
|
+
actual_status: latest?.status || null,
|
|
780
|
+
},
|
|
781
|
+
},
|
|
782
|
+
);
|
|
783
|
+
}
|
|
784
|
+
if (requestedStatus && requestedStatus !== cur.status) {
|
|
785
|
+
const eventType = requestedStatus === 'completed' ? 'session_closed'
|
|
786
|
+
: requestedStatus === 'crashed' ? 'session_crashed'
|
|
787
|
+
: requestedStatus === 'orphan' ? 'session_orphaned'
|
|
647
788
|
: 'session_closed';
|
|
648
789
|
appendEventInTx(db, {
|
|
649
790
|
type: eventType,
|
|
650
791
|
task_id: cur.task_id,
|
|
651
792
|
session_id: sid,
|
|
652
793
|
runtime: cur.runtime,
|
|
653
|
-
payload: {
|
|
794
|
+
payload: {
|
|
795
|
+
from: cur.status,
|
|
796
|
+
to: requestedStatus,
|
|
797
|
+
worker_pid: cur.pid,
|
|
798
|
+
},
|
|
654
799
|
});
|
|
655
800
|
}
|
|
656
801
|
return db.prepare('SELECT * FROM sessions WHERE sid = ?').get(sid);
|
|
@@ -782,11 +927,15 @@ function admissionCheck(db, task_id, { freshnessSeconds = 120 } = {}) {
|
|
|
782
927
|
};
|
|
783
928
|
}
|
|
784
929
|
|
|
785
|
-
function reapOrphanSessions(db, {
|
|
930
|
+
function reapOrphanSessions(db, {
|
|
931
|
+
graceSeconds = 300,
|
|
932
|
+
exclude_session_ids = [],
|
|
933
|
+
} = {}) {
|
|
786
934
|
const cutoff = new Date(Date.now() - graceSeconds * 1000).toISOString();
|
|
935
|
+
const excluded = new Set(exclude_session_ids);
|
|
787
936
|
const candidates = db.prepare(
|
|
788
937
|
"SELECT * FROM sessions WHERE status = 'running' AND (lease_expires_at < ? OR heartbeat_at < ?)",
|
|
789
|
-
).all(cutoff, cutoff);
|
|
938
|
+
).all(cutoff, cutoff).filter((session) => !excluded.has(session.sid));
|
|
790
939
|
const reaped = [];
|
|
791
940
|
for (const s of candidates) {
|
|
792
941
|
updateSession(db, s.sid, { status: 'orphan' });
|
|
@@ -810,49 +959,103 @@ function readCircuitBreakerRow(db, task_id) {
|
|
|
810
959
|
return db.prepare('SELECT * FROM circuit_breaker WHERE task_id = ?').get(task_id) || null;
|
|
811
960
|
}
|
|
812
961
|
|
|
813
|
-
function
|
|
962
|
+
function recordTaskFailureInTx(db, task_id, {
|
|
814
963
|
reason = 'unknown',
|
|
815
964
|
session_id = null,
|
|
816
965
|
fail_threshold = DEFAULT_FAIL_THRESHOLD,
|
|
817
966
|
} = {}) {
|
|
818
967
|
if (!task_id) throw new StateOpsError('VALIDATION_ERROR', 'task_id required');
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
}
|
|
968
|
+
const now = nowIso();
|
|
969
|
+
const existing = readCircuitBreakerRow(db, task_id);
|
|
970
|
+
const wasTripped = !!(existing && existing.tripped_at);
|
|
971
|
+
const newCount = (existing ? existing.failure_count : 0) + 1;
|
|
972
|
+
const crossesThreshold = !wasTripped && newCount >= fail_threshold;
|
|
973
|
+
const trippedAt = wasTripped
|
|
974
|
+
? existing.tripped_at
|
|
975
|
+
: (crossesThreshold ? now : null);
|
|
976
|
+
|
|
977
|
+
if (existing) {
|
|
978
|
+
db.prepare(
|
|
979
|
+
'UPDATE circuit_breaker SET failure_count = ?, tripped_at = ?, last_failure_at = ?, last_failure_reason = ? WHERE task_id = ?',
|
|
980
|
+
).run(newCount, trippedAt, now, reason, task_id);
|
|
981
|
+
} else {
|
|
982
|
+
db.prepare(
|
|
983
|
+
'INSERT INTO circuit_breaker (task_id, failure_count, tripped_at, last_failure_at, last_failure_reason) VALUES (?, ?, ?, ?, ?)',
|
|
984
|
+
).run(task_id, newCount, trippedAt, now, reason);
|
|
985
|
+
}
|
|
838
986
|
|
|
987
|
+
appendEventInTx(db, {
|
|
988
|
+
type: 'task_failure',
|
|
989
|
+
task_id,
|
|
990
|
+
session_id,
|
|
991
|
+
payload: { reason, failure_count: newCount },
|
|
992
|
+
});
|
|
993
|
+
|
|
994
|
+
if (crossesThreshold) {
|
|
839
995
|
appendEventInTx(db, {
|
|
840
|
-
type: '
|
|
996
|
+
type: 'task_circuit_broken',
|
|
841
997
|
task_id,
|
|
842
998
|
session_id,
|
|
843
|
-
payload: {
|
|
999
|
+
payload: { failure_count: newCount, threshold: fail_threshold },
|
|
844
1000
|
});
|
|
1001
|
+
}
|
|
845
1002
|
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
type: 'task_circuit_broken',
|
|
849
|
-
task_id,
|
|
850
|
-
session_id,
|
|
851
|
-
payload: { failure_count: newCount, threshold: fail_threshold },
|
|
852
|
-
});
|
|
853
|
-
}
|
|
1003
|
+
return { failure_count: newCount, tripped: crossesThreshold || wasTripped };
|
|
1004
|
+
}
|
|
854
1005
|
|
|
855
|
-
|
|
1006
|
+
function recordTaskFailure(db, task_id, options = {}) {
|
|
1007
|
+
return tx(db, () => recordTaskFailureInTx(db, task_id, options));
|
|
1008
|
+
}
|
|
1009
|
+
|
|
1010
|
+
function crashOrphanSession(db, sid, {
|
|
1011
|
+
reason = 'session_crashed_on_boot',
|
|
1012
|
+
fail_threshold = DEFAULT_FAIL_THRESHOLD,
|
|
1013
|
+
} = {}) {
|
|
1014
|
+
if (!sid) throw new StateOpsError('VALIDATION_ERROR', 'sid required');
|
|
1015
|
+
return tx(db, () => {
|
|
1016
|
+
const session = db.prepare('SELECT * FROM sessions WHERE sid = ?').get(sid);
|
|
1017
|
+
if (!session) {
|
|
1018
|
+
throw new StateOpsError('SESSION_NOT_FOUND', `no session ${sid}`);
|
|
1019
|
+
}
|
|
1020
|
+
if (session.status !== 'orphan') {
|
|
1021
|
+
return {
|
|
1022
|
+
changed: false,
|
|
1023
|
+
status: session.status,
|
|
1024
|
+
failure_recorded: false,
|
|
1025
|
+
};
|
|
1026
|
+
}
|
|
1027
|
+
const result = db.prepare(
|
|
1028
|
+
"UPDATE sessions SET status = 'crashed', pid = NULL WHERE sid = ? AND status = 'orphan'",
|
|
1029
|
+
).run(sid);
|
|
1030
|
+
if (result.changes !== 1) {
|
|
1031
|
+
return {
|
|
1032
|
+
changed: false,
|
|
1033
|
+
status: db.prepare('SELECT status FROM sessions WHERE sid = ?').get(sid)?.status || null,
|
|
1034
|
+
failure_recorded: false,
|
|
1035
|
+
};
|
|
1036
|
+
}
|
|
1037
|
+
appendEventInTx(db, {
|
|
1038
|
+
type: 'session_crashed',
|
|
1039
|
+
task_id: session.task_id,
|
|
1040
|
+
session_id: sid,
|
|
1041
|
+
runtime: session.runtime,
|
|
1042
|
+
payload: {
|
|
1043
|
+
from: 'orphan',
|
|
1044
|
+
to: 'crashed',
|
|
1045
|
+
worker_pid: session.pid,
|
|
1046
|
+
},
|
|
1047
|
+
});
|
|
1048
|
+
const failure = recordTaskFailureInTx(db, session.task_id, {
|
|
1049
|
+
reason,
|
|
1050
|
+
session_id: sid,
|
|
1051
|
+
fail_threshold,
|
|
1052
|
+
});
|
|
1053
|
+
return {
|
|
1054
|
+
changed: true,
|
|
1055
|
+
status: 'crashed',
|
|
1056
|
+
failure_recorded: true,
|
|
1057
|
+
failure,
|
|
1058
|
+
};
|
|
856
1059
|
});
|
|
857
1060
|
}
|
|
858
1061
|
|
|
@@ -1004,6 +1207,7 @@ module.exports = {
|
|
|
1004
1207
|
heartbeatSession,
|
|
1005
1208
|
admissionCheck,
|
|
1006
1209
|
reapOrphanSessions,
|
|
1210
|
+
crashOrphanSession,
|
|
1007
1211
|
// circuit breaker
|
|
1008
1212
|
recordTaskFailure,
|
|
1009
1213
|
resetCircuitBreaker,
|