shennian 0.2.16 → 0.2.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/commands/upgrade.js +4 -1
- package/dist/src/index.js +42 -22
- package/dist/src/upgrade/engine.d.ts +16 -0
- package/dist/src/upgrade/engine.js +57 -2
- package/package.json +1 -1
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
import chalk from 'chalk';
|
|
4
4
|
import { execSync, spawn } from 'node:child_process';
|
|
5
5
|
import os from 'node:os';
|
|
6
|
-
import { getCurrentVersion, fetchLatestVersion, compareVersions, performUpgrade, checkForUpdate, } from '../upgrade/engine.js';
|
|
6
|
+
import { getCurrentVersion, fetchLatestVersion, compareVersions, performUpgrade, checkForUpdate, recordUpgradeFailure, } from '../upgrade/engine.js';
|
|
7
7
|
const SPINNER = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏'];
|
|
8
8
|
export function restartCurrentDaemonAfterUpgrade(deps = {}) {
|
|
9
9
|
const platform = deps.platform ?? os.platform();
|
|
@@ -120,6 +120,7 @@ export function registerUpgradeCommand(program) {
|
|
|
120
120
|
restartCurrentDaemonAfterUpgrade();
|
|
121
121
|
}
|
|
122
122
|
else {
|
|
123
|
+
recordUpgradeFailure(latest, result.error);
|
|
123
124
|
console.log();
|
|
124
125
|
if (result.rolledBack) {
|
|
125
126
|
console.error(chalk.red(`✗ Upgrade failed (rolled back to ${current})`));
|
|
@@ -174,6 +175,8 @@ export async function handleUpgradeStart(client, reqId, targetVersion) {
|
|
|
174
175
|
restartCurrentDaemonAfterUpgrade();
|
|
175
176
|
}
|
|
176
177
|
else {
|
|
178
|
+
const failure = recordUpgradeFailure(latest, result.error);
|
|
179
|
+
console.log(`[${new Date().toISOString()}] [upgrade] ${latest} failed, retry after ${new Date(failure.nextRetryAt).toISOString()}: ${result.error}`);
|
|
177
180
|
sendEvent({
|
|
178
181
|
state: result.rolledBack ? 'rolledback' : 'failed',
|
|
179
182
|
machineId: 'self',
|
package/dist/src/index.js
CHANGED
|
@@ -13,7 +13,7 @@ import { registerAgentCommand } from './commands/agent.js';
|
|
|
13
13
|
import { registerUpgradeCommand } from './commands/upgrade.js';
|
|
14
14
|
import { SessionManager } from './session/manager.js';
|
|
15
15
|
import { SERVERS, regionToUrl, urlToRegion } from './region.js';
|
|
16
|
-
import { getCurrentVersion, handleStartupCrashCheck, checkForUpdate } from './upgrade/engine.js';
|
|
16
|
+
import { getCurrentVersion, handleStartupCrashCheck, checkForUpdate, isUpgradeVersionInCooldown, recordUpgradeFailure, } from './upgrade/engine.js';
|
|
17
17
|
import { detectAgents } from './agents/detect.js';
|
|
18
18
|
const cliVersion = getCurrentVersion();
|
|
19
19
|
import { getCachedAgentInfos, resolveAgentInfos } from './agents/model-registry.js';
|
|
@@ -281,25 +281,45 @@ program.parse();
|
|
|
281
281
|
async function scheduleAutoUpgrade(client, policy) {
|
|
282
282
|
if (policy === 'none')
|
|
283
283
|
return;
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
result
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
284
|
+
let upgradeInFlight = false;
|
|
285
|
+
const tick = async () => {
|
|
286
|
+
if (upgradeInFlight)
|
|
287
|
+
return;
|
|
288
|
+
let result;
|
|
289
|
+
try {
|
|
290
|
+
result = await checkForUpdate();
|
|
291
|
+
}
|
|
292
|
+
catch {
|
|
293
|
+
return; // silently skip on network error
|
|
294
|
+
}
|
|
295
|
+
if (!result.hasUpdate)
|
|
296
|
+
return;
|
|
297
|
+
const { changeType, current, latest } = result;
|
|
298
|
+
// Respect the policy
|
|
299
|
+
if (policy === 'patch' && changeType !== 'patch')
|
|
300
|
+
return;
|
|
301
|
+
if (policy === 'minor' && changeType === 'major')
|
|
302
|
+
return;
|
|
303
|
+
if (isUpgradeVersionInCooldown(latest))
|
|
304
|
+
return;
|
|
305
|
+
console.log(`[${new Date().toISOString()}] [upgrade] ${current} → ${latest} (${changeType}), auto-upgrading...`);
|
|
306
|
+
upgradeInFlight = true;
|
|
307
|
+
try {
|
|
308
|
+
const { handleUpgradeStart } = await import('./commands/upgrade.js');
|
|
309
|
+
// Use a dummy reqId since this is self-triggered
|
|
310
|
+
await handleUpgradeStart(client, 'auto-upgrade', latest);
|
|
311
|
+
}
|
|
312
|
+
catch (err) {
|
|
313
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
314
|
+
const failure = recordUpgradeFailure(latest, message);
|
|
315
|
+
console.log(`[${new Date().toISOString()}] [upgrade] ${latest} failed, retry after ${new Date(failure.nextRetryAt).toISOString()}: ${message}`);
|
|
316
|
+
}
|
|
317
|
+
finally {
|
|
318
|
+
upgradeInFlight = false;
|
|
319
|
+
}
|
|
320
|
+
};
|
|
321
|
+
// Delay slightly so we don't hit npm on every cold start during development,
|
|
322
|
+
// then keep polling so a bad published patch can be superseded by a good one.
|
|
323
|
+
setTimeout(() => void tick(), 30_000);
|
|
324
|
+
setInterval(() => void tick(), 30 * 60_000);
|
|
305
325
|
}
|
|
@@ -32,6 +32,17 @@ type UpgradeAttempt = {
|
|
|
32
32
|
attemptCount: number;
|
|
33
33
|
attemptAt: number;
|
|
34
34
|
};
|
|
35
|
+
export type UpgradeFailureRecord = {
|
|
36
|
+
version: string;
|
|
37
|
+
failureCount: number;
|
|
38
|
+
firstFailedAt: number;
|
|
39
|
+
lastFailedAt: number;
|
|
40
|
+
nextRetryAt: number;
|
|
41
|
+
error: string;
|
|
42
|
+
};
|
|
43
|
+
export type UpgradeFailures = {
|
|
44
|
+
versions: Record<string, UpgradeFailureRecord>;
|
|
45
|
+
};
|
|
35
46
|
export declare function getCurrentVersion(): string;
|
|
36
47
|
export declare function findPackageJson(startFilePath?: string): string | null;
|
|
37
48
|
export declare function fetchLatestVersion(): Promise<string>;
|
|
@@ -46,6 +57,11 @@ export declare function compareVersions(current: string, latest: string): 'none'
|
|
|
46
57
|
export declare function readUpgradeAttempt(): UpgradeAttempt | null;
|
|
47
58
|
export declare function writeUpgradeAttempt(attempt: UpgradeAttempt): void;
|
|
48
59
|
export declare function clearUpgradeAttempt(): void;
|
|
60
|
+
export declare function readUpgradeFailures(filePath?: string): UpgradeFailures;
|
|
61
|
+
export declare function getUpgradeRetryDelayMs(failureCount: number): number;
|
|
62
|
+
export declare function isUpgradeVersionInCooldown(version: string, now?: number, failures?: UpgradeFailures): boolean;
|
|
63
|
+
export declare function recordUpgradeFailure(version: string, error: string, now?: number): UpgradeFailureRecord;
|
|
64
|
+
export declare function clearUpgradeFailure(version: string): void;
|
|
49
65
|
/**
|
|
50
66
|
* Called on daemon startup. Detects crash loops from a failed upgrade and
|
|
51
67
|
* initiates rollback if crash count exceeds threshold.
|
|
@@ -6,11 +6,16 @@ import { promisify } from 'node:util';
|
|
|
6
6
|
import { fileURLToPath } from 'node:url';
|
|
7
7
|
import { getShennianDir, resolveShennianPath } from '../config/index.js';
|
|
8
8
|
const exec = promisify(execCb);
|
|
9
|
-
const SHENNIAN_DIR = getShennianDir();
|
|
10
9
|
const BACKUP_DIR = resolveShennianPath('backup');
|
|
11
10
|
const UPGRADE_ATTEMPT_FILE = resolveShennianPath('upgrade-attempt.json');
|
|
12
11
|
const MAX_CRASH_COUNT = 3;
|
|
13
12
|
const BACKUP_TTL_DAYS = 7;
|
|
13
|
+
const RETRY_DELAYS_MS = [
|
|
14
|
+
5 * 60_000,
|
|
15
|
+
30 * 60_000,
|
|
16
|
+
2 * 60 * 60_000,
|
|
17
|
+
6 * 60 * 60_000,
|
|
18
|
+
];
|
|
14
19
|
// ─── Version helpers ─────────────────────────────────────────────────────────
|
|
15
20
|
export function getCurrentVersion() {
|
|
16
21
|
try {
|
|
@@ -116,7 +121,7 @@ export function readUpgradeAttempt() {
|
|
|
116
121
|
}
|
|
117
122
|
}
|
|
118
123
|
export function writeUpgradeAttempt(attempt) {
|
|
119
|
-
fs.mkdirSync(
|
|
124
|
+
fs.mkdirSync(getShennianDir(), { recursive: true });
|
|
120
125
|
fs.writeFileSync(UPGRADE_ATTEMPT_FILE, JSON.stringify(attempt, null, 2));
|
|
121
126
|
}
|
|
122
127
|
export function clearUpgradeAttempt() {
|
|
@@ -125,6 +130,54 @@ export function clearUpgradeAttempt() {
|
|
|
125
130
|
}
|
|
126
131
|
catch { /* noop */ }
|
|
127
132
|
}
|
|
133
|
+
function getUpgradeFailuresFile() {
|
|
134
|
+
return resolveShennianPath('upgrade-failures.json');
|
|
135
|
+
}
|
|
136
|
+
export function readUpgradeFailures(filePath = getUpgradeFailuresFile()) {
|
|
137
|
+
try {
|
|
138
|
+
const parsed = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
139
|
+
if (!parsed || typeof parsed !== 'object' || !parsed.versions)
|
|
140
|
+
return { versions: {} };
|
|
141
|
+
return { versions: parsed.versions };
|
|
142
|
+
}
|
|
143
|
+
catch {
|
|
144
|
+
return { versions: {} };
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
function writeUpgradeFailures(failures) {
|
|
148
|
+
fs.mkdirSync(getShennianDir(), { recursive: true });
|
|
149
|
+
fs.writeFileSync(getUpgradeFailuresFile(), JSON.stringify(failures, null, 2));
|
|
150
|
+
}
|
|
151
|
+
export function getUpgradeRetryDelayMs(failureCount) {
|
|
152
|
+
return RETRY_DELAYS_MS[Math.min(Math.max(failureCount - 1, 0), RETRY_DELAYS_MS.length - 1)];
|
|
153
|
+
}
|
|
154
|
+
export function isUpgradeVersionInCooldown(version, now = Date.now(), failures = readUpgradeFailures()) {
|
|
155
|
+
const record = failures.versions[version];
|
|
156
|
+
return Boolean(record && record.nextRetryAt > now);
|
|
157
|
+
}
|
|
158
|
+
export function recordUpgradeFailure(version, error, now = Date.now()) {
|
|
159
|
+
const failures = readUpgradeFailures();
|
|
160
|
+
const previous = failures.versions[version];
|
|
161
|
+
const failureCount = (previous?.failureCount ?? 0) + 1;
|
|
162
|
+
const record = {
|
|
163
|
+
version,
|
|
164
|
+
failureCount,
|
|
165
|
+
firstFailedAt: previous?.firstFailedAt ?? now,
|
|
166
|
+
lastFailedAt: now,
|
|
167
|
+
nextRetryAt: now + getUpgradeRetryDelayMs(failureCount),
|
|
168
|
+
error,
|
|
169
|
+
};
|
|
170
|
+
failures.versions[version] = record;
|
|
171
|
+
writeUpgradeFailures(failures);
|
|
172
|
+
return record;
|
|
173
|
+
}
|
|
174
|
+
export function clearUpgradeFailure(version) {
|
|
175
|
+
const failures = readUpgradeFailures();
|
|
176
|
+
if (!failures.versions[version])
|
|
177
|
+
return;
|
|
178
|
+
delete failures.versions[version];
|
|
179
|
+
writeUpgradeFailures(failures);
|
|
180
|
+
}
|
|
128
181
|
/**
|
|
129
182
|
* Called on daemon startup. Detects crash loops from a failed upgrade and
|
|
130
183
|
* initiates rollback if crash count exceeds threshold.
|
|
@@ -162,6 +215,7 @@ export async function handleStartupCrashCheck() {
|
|
|
162
215
|
export async function performUpgrade(targetVersion, onProgress) {
|
|
163
216
|
const currentVersion = getCurrentVersion();
|
|
164
217
|
if (currentVersion === targetVersion) {
|
|
218
|
+
clearUpgradeFailure(targetVersion);
|
|
165
219
|
return { ok: false, error: `Already on version ${targetVersion}` };
|
|
166
220
|
}
|
|
167
221
|
// Step 1: Verify npm is accessible
|
|
@@ -225,6 +279,7 @@ export async function performUpgrade(targetVersion, onProgress) {
|
|
|
225
279
|
attemptAt: Date.now(),
|
|
226
280
|
});
|
|
227
281
|
cleanOldBackups();
|
|
282
|
+
clearUpgradeFailure(targetVersion);
|
|
228
283
|
// Step 6: Signal restart (caller sends upgrade.status: restarting before killing)
|
|
229
284
|
onProgress({ step: 'restarting', from: currentVersion, to: targetVersion });
|
|
230
285
|
return { ok: true, from: currentVersion, to: targetVersion };
|