runcloud 0.1.101 → 0.1.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/deploy.js +129 -100
- package/dist/diagnose.js +209 -22
- package/dist/sessionDiagnostics.js +192 -0
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/commands/deploy.js
CHANGED
|
@@ -1,125 +1,154 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { join, resolve } from 'node:path';
|
|
4
|
-
import { build } from 'esbuild';
|
|
1
|
+
import { existsSync } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
5
3
|
import { ApiClient, ApiError, friendlyApiError } from '../api.js';
|
|
6
|
-
import { requireCredentials,
|
|
7
|
-
import {
|
|
8
|
-
|
|
4
|
+
import { requireCredentials, resolveProject } from '../config.js';
|
|
5
|
+
import { createSnapshot } from '../archive.js';
|
|
6
|
+
import { parseSseFrames } from '../sse.js';
|
|
7
|
+
import { assertKnownServices, parseServiceFilter, readBackendServices } from '../services-manifest.js';
|
|
8
|
+
const POLL_INTERVAL_MS = 5_000;
|
|
9
9
|
const POLL_TIMEOUT_MS = 20 * 60 * 1000;
|
|
10
|
-
function
|
|
11
|
-
|
|
10
|
+
export async function pollRun(client, runId, opts = {}) {
|
|
11
|
+
const intervalMs = opts.intervalMs ?? POLL_INTERVAL_MS;
|
|
12
|
+
const timeoutMs = opts.timeoutMs ?? POLL_TIMEOUT_MS;
|
|
13
|
+
const deadline = Date.now() + timeoutMs;
|
|
14
|
+
for (;;) {
|
|
15
|
+
const run = await client.get(`/newly_backend/deploy_runs/${runId}`);
|
|
16
|
+
if (run.status !== 'running')
|
|
17
|
+
return run;
|
|
18
|
+
if (Date.now() > deadline)
|
|
19
|
+
throw new Error(`Deploy run ${runId} timed out after ${timeoutMs / 1000}s`);
|
|
20
|
+
await new Promise((r) => setTimeout(r, intervalMs));
|
|
21
|
+
}
|
|
12
22
|
}
|
|
13
|
-
function
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
23
|
+
export async function streamRun(client, runId, print = (s) => process.stdout.write(s)) {
|
|
24
|
+
let buffer = '';
|
|
25
|
+
let started = false;
|
|
26
|
+
try {
|
|
27
|
+
for await (const chunk of client.stream(`/newly_backend/deploy_runs/${runId}/logs/stream`)) {
|
|
28
|
+
started = true;
|
|
29
|
+
buffer = (buffer + chunk).replace(/\r\n/g, '\n');
|
|
30
|
+
const lastSep = buffer.lastIndexOf('\n\n');
|
|
31
|
+
if (lastSep === -1)
|
|
32
|
+
continue;
|
|
33
|
+
const ready = buffer.slice(0, lastSep + 2);
|
|
34
|
+
buffer = buffer.slice(lastSep + 2);
|
|
35
|
+
for (const frame of parseSseFrames(ready)) {
|
|
36
|
+
if (frame.event === 'done') {
|
|
37
|
+
return JSON.parse(frame.data);
|
|
38
|
+
}
|
|
39
|
+
try {
|
|
40
|
+
print(JSON.parse(frame.data));
|
|
41
|
+
}
|
|
42
|
+
catch {
|
|
43
|
+
print(frame.data);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
22
46
|
}
|
|
23
47
|
}
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
48
|
+
catch (e) {
|
|
49
|
+
if (e instanceof ApiError && e.status === 404) {
|
|
50
|
+
return pollRun(client, runId);
|
|
51
|
+
}
|
|
52
|
+
if (started) {
|
|
53
|
+
return pollRun(client, runId);
|
|
54
|
+
}
|
|
55
|
+
throw e;
|
|
31
56
|
}
|
|
32
|
-
return
|
|
33
|
-
}
|
|
34
|
-
export async function bundleService(dir, service) {
|
|
35
|
-
const result = await build({
|
|
36
|
-
entryPoints: [serviceEntry(dir, service)],
|
|
37
|
-
bundle: true,
|
|
38
|
-
write: false,
|
|
39
|
-
format: 'esm',
|
|
40
|
-
platform: 'browser',
|
|
41
|
-
target: 'es2022',
|
|
42
|
-
conditions: ['workerd', 'worker', 'browser'],
|
|
43
|
-
sourcemap: false,
|
|
44
|
-
legalComments: 'none',
|
|
45
|
-
});
|
|
46
|
-
const output = result.outputFiles[0];
|
|
47
|
-
if (!output)
|
|
48
|
-
throw new Error(`Bundling service ${service.name} produced no output`);
|
|
49
|
-
return Buffer.from(output.contents);
|
|
57
|
+
return pollRun(client, runId);
|
|
50
58
|
}
|
|
51
|
-
export
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
const deployment = (await client.get(`/service-deployments/${deploymentId}`));
|
|
56
|
-
if (['ready', 'failed', 'superseded'].includes(deployment.status))
|
|
57
|
-
return deployment;
|
|
58
|
-
if (Date.now() > deadline)
|
|
59
|
-
throw new Error(`Deployment ${deploymentId} timed out`);
|
|
60
|
-
await new Promise((done) => setTimeout(done, intervalMs));
|
|
59
|
+
export function finishOrThrow(run, label) {
|
|
60
|
+
if (run.status === 'succeeded') {
|
|
61
|
+
console.log(`${label} deploy succeeded.`);
|
|
62
|
+
return;
|
|
61
63
|
}
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
console.log(`Building ${service.name}…`);
|
|
65
|
-
const bundle = await bundleService(dir, service);
|
|
66
|
-
const sourceDigest = sha256(bundle);
|
|
67
|
-
const configurationSpec = {
|
|
68
|
-
name: service.name,
|
|
69
|
-
type: service.type,
|
|
70
|
-
public: service.public ?? true,
|
|
71
|
-
bindings: service.bindings ?? {},
|
|
72
|
-
};
|
|
73
|
-
const deployment = (await client.postWithHeaders(`/environment-services/${serviceId}/deployments`, { bundleBase64: bundle.toString('base64'), sourceDigest, configurationSpec }, { 'Idempotency-Key': randomUUID() }));
|
|
74
|
-
const finished = await pollDeployment(client, deployment.id);
|
|
75
|
-
if (finished.status !== 'ready') {
|
|
76
|
-
throw new Error(`Deployment ${finished.id} finished with status ${finished.status}${finished.lastError ? `: ${finished.lastError}` : ''}`);
|
|
64
|
+
if (run.failure_reason === 'no_backend_code') {
|
|
65
|
+
throw new Error('No backend code to deploy (backend/ missing).');
|
|
77
66
|
}
|
|
78
|
-
|
|
79
|
-
return finished;
|
|
67
|
+
throw new Error(`${label} deploy failed (${run.failure_reason ?? 'unknown'}):\n${run.output_tail ?? ''}`);
|
|
80
68
|
}
|
|
81
|
-
export async function
|
|
69
|
+
export async function runDevDeploy(projectId, client, dir, services = []) {
|
|
82
70
|
if (!existsSync(join(dir, 'backend'))) {
|
|
83
|
-
throw new Error('No backend/ in this directory. Run `newly pull <project-id>` first,
|
|
71
|
+
throw new Error('No backend/ in this directory. Run `newly pull <project-id>` first, ' +
|
|
72
|
+
'or cd into a pulled project.');
|
|
84
73
|
}
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
74
|
+
console.log('Packaging local backend…');
|
|
75
|
+
const snapshot = await createSnapshot(dir);
|
|
76
|
+
const kib = snapshot.length < 1024 ? '<1' : Math.round(snapshot.length / 1024).toString();
|
|
77
|
+
console.log(`Uploading snapshot (${kib} KiB) and deploying server-side…`);
|
|
78
|
+
const params = new URLSearchParams({ env: 'dev' });
|
|
79
|
+
for (const service of services)
|
|
80
|
+
params.append('services', service);
|
|
81
|
+
const { run_id } = await client.upload(`/newly_backend/deploy/${projectId}?${params}`, snapshot, 'application/gzip');
|
|
82
|
+
const run = await streamRun(client, run_id);
|
|
83
|
+
finishOrThrow(run, 'dev');
|
|
84
|
+
}
|
|
85
|
+
export async function runProdDeploy(projectId, client, gitSha = 'HEAD', services = []) {
|
|
86
|
+
const label = gitSha === 'HEAD' ? 'repo HEAD' : gitSha;
|
|
87
|
+
console.log(`Deploying prod from the canonical project code (${label})…`);
|
|
88
|
+
const { run_id } = await client.post(`/newly_backend/deploy/${projectId}`, {
|
|
89
|
+
env: 'prod',
|
|
90
|
+
git_sha: gitSha,
|
|
91
|
+
services,
|
|
92
|
+
});
|
|
93
|
+
const run = await streamRun(client, run_id);
|
|
94
|
+
finishOrThrow(run, 'prod');
|
|
95
|
+
}
|
|
96
|
+
export async function ensureBackendProvisioned(client, projectId, env, opts = {}) {
|
|
97
|
+
const print = opts.print ?? console.log;
|
|
98
|
+
const status = await client.get(`/newly_backend/status/${projectId}`);
|
|
99
|
+
if (status[env])
|
|
100
|
+
return;
|
|
101
|
+
print(`Provisioning ${env} backend…`);
|
|
102
|
+
if (env === 'dev') {
|
|
103
|
+
await client.post(`/newly_backend/provision_dev/${projectId}`, {
|
|
104
|
+
dev_region: opts.region ?? 'auto',
|
|
105
|
+
});
|
|
99
106
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
deployments.push(await deployService(client, dir, service, serviceIds.get(service.name)));
|
|
107
|
+
else {
|
|
108
|
+
await client.post(`/newly_backend/provision_prod/${projectId}`);
|
|
103
109
|
}
|
|
104
|
-
|
|
110
|
+
print(`Provisioned ${env} backend.`);
|
|
105
111
|
}
|
|
106
112
|
export function registerDeploy(program) {
|
|
107
113
|
program
|
|
108
114
|
.command('deploy')
|
|
109
|
-
.description('
|
|
110
|
-
.
|
|
115
|
+
.description('Provision if needed, then deploy the backend server-side')
|
|
116
|
+
.option('--env <env>', 'dev or prod', 'dev')
|
|
117
|
+
.option('--region <region>', "Newly server region hint for first-time dev provisioning (defaults to 'auto')", 'auto')
|
|
111
118
|
.option('--project <id>', 'project id (defaults to the linked directory)')
|
|
112
|
-
.
|
|
119
|
+
.option('--service <names>', 'comma-separated service names from backend/newly.services.json')
|
|
120
|
+
.option('--git-sha <sha>', 'prod git revision to deploy from canonical project code', 'HEAD')
|
|
121
|
+
.option('--yes', 'skip the prod confirmation prompt', false)
|
|
122
|
+
.action(async (opts) => {
|
|
113
123
|
try {
|
|
114
|
-
|
|
124
|
+
if (opts.env !== 'dev' && opts.env !== 'prod')
|
|
125
|
+
throw new Error('--env must be dev or prod');
|
|
115
126
|
const creds = requireCredentials();
|
|
116
|
-
const projectId = resolveProject(
|
|
117
|
-
const
|
|
118
|
-
|
|
119
|
-
|
|
127
|
+
const projectId = resolveProject(process.cwd(), opts.project);
|
|
128
|
+
const client = new ApiClient(creds.apiUrl, creds.token);
|
|
129
|
+
const services = parseServiceFilter(opts.service);
|
|
130
|
+
assertKnownServices(readBackendServices(process.cwd()), services);
|
|
131
|
+
const stackInfo = await client.get(`/newly_backend/stack/${projectId}`);
|
|
132
|
+
if (!stackInfo.cli_deployable) {
|
|
133
|
+
throw new Error('This project does not support CLI deploys yet. Use the Newly web app to deploy it.');
|
|
134
|
+
}
|
|
135
|
+
if (opts.env === 'dev' && !existsSync(join(process.cwd(), 'backend'))) {
|
|
136
|
+
throw new Error('No backend/ in this directory. Run `newly pull <project-id>` first, ' +
|
|
137
|
+
'or cd into a pulled project.');
|
|
138
|
+
}
|
|
139
|
+
await ensureBackendProvisioned(client, projectId, opts.env, {
|
|
140
|
+
region: opts.region,
|
|
141
|
+
print: console.log,
|
|
142
|
+
});
|
|
143
|
+
if (opts.env === 'prod') {
|
|
144
|
+
await runProdDeploy(projectId, client, opts.gitSha, services);
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
await runDevDeploy(projectId, client, process.cwd(), services);
|
|
148
|
+
}
|
|
120
149
|
}
|
|
121
|
-
catch (
|
|
122
|
-
console.error(
|
|
150
|
+
catch (e) {
|
|
151
|
+
console.error(e instanceof ApiError ? friendlyApiError(e) : String(e instanceof Error ? e.message : e));
|
|
123
152
|
process.exitCode = 1;
|
|
124
153
|
}
|
|
125
154
|
});
|
package/dist/diagnose.js
CHANGED
|
@@ -1,13 +1,43 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import { GoogleAuth, Impersonated } from 'google-auth-library';
|
|
4
|
+
import { renderSessionDiagnosticTerminal, writeSessionDiagnosticBundle, } from './sessionDiagnostics.js';
|
|
5
|
+
const execFileAsync = promisify(execFile);
|
|
1
6
|
const DEFAULT_URLS = {
|
|
2
7
|
dev: {
|
|
3
8
|
apiOps: 'https://cp-api-ops-dev-97683904813.us-east4.run.app',
|
|
4
9
|
worker: 'https://cp-worker-dev-97683904813.us-east4.run.app',
|
|
5
10
|
},
|
|
6
11
|
prod: {
|
|
7
|
-
apiOps: 'https://cp-api-ops-prod-
|
|
12
|
+
apiOps: 'https://cp-api-ops-prod-wqoplvhyxa-uk.a.run.app',
|
|
8
13
|
worker: 'https://cp-worker-prod-350096142301.us-east4.run.app',
|
|
9
14
|
},
|
|
10
15
|
};
|
|
16
|
+
const DEFAULT_SERVICE_ACCOUNTS = {
|
|
17
|
+
dev: 'cp-diagnose-ops-dev@newly-v3-dev.iam.gserviceaccount.com',
|
|
18
|
+
prod: 'cp-diagnose-ops-prod@newly-v3-prod.iam.gserviceaccount.com',
|
|
19
|
+
};
|
|
20
|
+
export function resolveSessionDiagnoseTarget(env) {
|
|
21
|
+
if (env === 'dev' || env === 'prod') {
|
|
22
|
+
const url = stripSlash(process.env[`DIAGNOSE_CP_API_OPS_URL_${env.toUpperCase()}`] ??
|
|
23
|
+
process.env.DIAGNOSE_CP_API_OPS_URL ??
|
|
24
|
+
DEFAULT_URLS[env].apiOps);
|
|
25
|
+
return {
|
|
26
|
+
url,
|
|
27
|
+
audience: process.env.DIAGNOSE_OIDC_AUDIENCE ?? url,
|
|
28
|
+
serviceAccount: process.env.DIAGNOSE_OPS_IMPERSONATE_SERVICE_ACCOUNT ?? DEFAULT_SERVICE_ACCOUNTS[env],
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
const match = /^pr-(\d+)$/.exec(env);
|
|
32
|
+
if (!match)
|
|
33
|
+
throw new Error(`--env must be dev, prod, or pr-<number> (got "${env}")`);
|
|
34
|
+
const url = `https://cp-api-pr-${match[1]}-97683904813.us-east4.run.app`;
|
|
35
|
+
return {
|
|
36
|
+
url,
|
|
37
|
+
audience: process.env.DIAGNOSE_OIDC_AUDIENCE ?? url,
|
|
38
|
+
serviceAccount: process.env.DIAGNOSE_OPS_IMPERSONATE_SERVICE_ACCOUNT ?? DEFAULT_SERVICE_ACCOUNTS.dev,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
11
41
|
function stripSlash(url) {
|
|
12
42
|
return url.replace(/\/+$/, '');
|
|
13
43
|
}
|
|
@@ -16,6 +46,7 @@ export function resolveTargets(env) {
|
|
|
16
46
|
throw new Error(`--env must be dev or prod (got "${env}")`);
|
|
17
47
|
}
|
|
18
48
|
const upper = env.toUpperCase();
|
|
49
|
+
const serviceAccount = process.env.DIAGNOSE_OPS_IMPERSONATE_SERVICE_ACCOUNT ?? DEFAULT_SERVICE_ACCOUNTS[env];
|
|
19
50
|
const apiOps = process.env[`DIAGNOSE_CP_API_OPS_URL_${upper}`] ??
|
|
20
51
|
process.env.DIAGNOSE_CP_API_OPS_URL ??
|
|
21
52
|
DEFAULT_URLS[env].apiOps;
|
|
@@ -23,8 +54,8 @@ export function resolveTargets(env) {
|
|
|
23
54
|
process.env.DIAGNOSE_CP_WORKER_URL ??
|
|
24
55
|
DEFAULT_URLS[env].worker;
|
|
25
56
|
return [
|
|
26
|
-
{ service: 'cp-api', url: stripSlash(apiOps) },
|
|
27
|
-
{ service: 'cp-worker', url: stripSlash(worker) },
|
|
57
|
+
{ service: 'cp-api', url: stripSlash(apiOps), serviceAccount },
|
|
58
|
+
{ service: 'cp-worker', url: stripSlash(worker), serviceAccount },
|
|
28
59
|
];
|
|
29
60
|
}
|
|
30
61
|
function extractAuthorization(headers) {
|
|
@@ -34,30 +65,123 @@ function extractAuthorization(headers) {
|
|
|
34
65
|
const h = headers;
|
|
35
66
|
return h?.Authorization ?? h?.authorization;
|
|
36
67
|
}
|
|
37
|
-
|
|
38
|
-
const
|
|
68
|
+
async function gcloudIdentityToken(audience, serviceAccount) {
|
|
69
|
+
const args = ['auth', 'print-identity-token', `--audiences=${audience}`];
|
|
70
|
+
if (serviceAccount)
|
|
71
|
+
args.push(`--impersonate-service-account=${serviceAccount}`);
|
|
72
|
+
const { stdout } = await execFileAsync('gcloud', args, { timeout: 15_000, maxBuffer: 1024 * 1024 });
|
|
73
|
+
const token = stdout.trim();
|
|
74
|
+
if (!token)
|
|
75
|
+
throw new Error('gcloud returned an empty identity token');
|
|
76
|
+
return token;
|
|
77
|
+
}
|
|
78
|
+
function idTokenAudience(authorization) {
|
|
79
|
+
const token = authorization.replace(/^Bearer\s+/i, '');
|
|
80
|
+
const payload = token.split('.')[1];
|
|
81
|
+
if (!payload)
|
|
82
|
+
return null;
|
|
83
|
+
try {
|
|
84
|
+
const parsed = JSON.parse(Buffer.from(payload, 'base64url').toString('utf8'));
|
|
85
|
+
if (typeof parsed.aud === 'string')
|
|
86
|
+
return [parsed.aud];
|
|
87
|
+
return Array.isArray(parsed.aud)
|
|
88
|
+
? parsed.aud.filter((entry) => typeof entry === 'string')
|
|
89
|
+
: null;
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
return null;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
function exactAudienceAuthorization(authorization, audience) {
|
|
96
|
+
if (!authorization)
|
|
97
|
+
throw new Error('credential returned no authorization header');
|
|
98
|
+
const audiences = idTokenAudience(authorization);
|
|
99
|
+
if (!audiences?.includes(audience)) {
|
|
100
|
+
throw new Error(`credential minted an ID token for ${audiences?.join(', ') || 'an unreadable audience'}, not ${audience}`);
|
|
101
|
+
}
|
|
102
|
+
return authorization;
|
|
103
|
+
}
|
|
104
|
+
async function impersonatedIdentityToken(audience, serviceAccount) {
|
|
105
|
+
const sourceClient = await new GoogleAuth({
|
|
106
|
+
scopes: ['https://www.googleapis.com/auth/cloud-platform'],
|
|
107
|
+
}).getClient();
|
|
108
|
+
const impersonated = new Impersonated({
|
|
109
|
+
sourceClient,
|
|
110
|
+
targetPrincipal: serviceAccount,
|
|
111
|
+
targetScopes: ['https://www.googleapis.com/auth/cloud-platform'],
|
|
112
|
+
lifetime: 600,
|
|
113
|
+
});
|
|
114
|
+
const token = await impersonated.fetchIdToken(audience, { includeEmail: true });
|
|
115
|
+
return exactAudienceAuthorization(`Bearer ${token}`, audience);
|
|
116
|
+
}
|
|
117
|
+
export async function resolveOpsAuthHeader(env = process.env, requestedAudience, requestedServiceAccount) {
|
|
118
|
+
const audience = requestedAudience ?? env.DIAGNOSE_OIDC_AUDIENCE ?? 'newly-diagnose-ops';
|
|
119
|
+
const serviceAccount = env.DIAGNOSE_OPS_IMPERSONATE_SERVICE_ACCOUNT ?? requestedServiceAccount;
|
|
120
|
+
let adcError = 'unknown ADC error';
|
|
39
121
|
try {
|
|
40
|
-
const { GoogleAuth } = await import('google-auth-library');
|
|
41
122
|
const auth = new GoogleAuth();
|
|
42
123
|
const client = await auth.getIdTokenClient(audience);
|
|
43
|
-
|
|
44
|
-
if (authz) {
|
|
45
|
-
return authz;
|
|
46
|
-
}
|
|
124
|
+
return exactAudienceAuthorization(extractAuthorization(await client.getRequestHeaders()), audience);
|
|
47
125
|
}
|
|
48
|
-
catch {
|
|
126
|
+
catch (error) {
|
|
127
|
+
adcError = error instanceof Error ? error.message : String(error);
|
|
128
|
+
}
|
|
129
|
+
let impersonationError = 'not configured';
|
|
130
|
+
if (serviceAccount && env.DIAGNOSE_DISABLE_IMPERSONATION !== '1') {
|
|
131
|
+
try {
|
|
132
|
+
return await impersonatedIdentityToken(audience, serviceAccount);
|
|
133
|
+
}
|
|
134
|
+
catch (error) {
|
|
135
|
+
impersonationError = error instanceof Error ? error.message : String(error);
|
|
136
|
+
}
|
|
49
137
|
}
|
|
50
138
|
const token = env.DIAGNOSE_OPS_TOKEN;
|
|
51
139
|
if (token) {
|
|
52
140
|
return `Bearer ${token}`;
|
|
53
141
|
}
|
|
54
|
-
|
|
55
|
-
|
|
142
|
+
if (env.DIAGNOSE_DISABLE_GCLOUD !== '1') {
|
|
143
|
+
try {
|
|
144
|
+
return exactAudienceAuthorization(`Bearer ${await gcloudIdentityToken(audience, serviceAccount)}`, audience);
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
const gcloudError = error instanceof Error ? error.message : String(error);
|
|
148
|
+
throw new Error(`No usable ops credential for audience ${audience}.\n` +
|
|
149
|
+
`ADC failed: ${truncate(adcError, 500)}\n` +
|
|
150
|
+
`Service-account impersonation failed: ${truncate(impersonationError, 500)}\n` +
|
|
151
|
+
`gcloud failed: ${truncate(gcloudError, 500)}\n\n` +
|
|
152
|
+
'Repair access by adding the Google identity to control-plane `operators`, applying Terraform, ' +
|
|
153
|
+
'and running `gcloud auth application-default login`, then rerun. ' +
|
|
154
|
+
'For a headless agent, export DIAGNOSE_OPS_TOKEN with the read-only break-glass token.');
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
throw new Error(`No ops credential available for audience ${audience}. ` +
|
|
158
|
+
`ADC failed: ${truncate(adcError, 300)}. ` +
|
|
159
|
+
`Service-account impersonation failed: ${truncate(impersonationError, 300)}. ` +
|
|
160
|
+
'Run `gcloud auth application-default login` after the operator Terraform has been applied, or export DIAGNOSE_OPS_TOKEN.');
|
|
56
161
|
}
|
|
57
162
|
function truncate(value, max = 200) {
|
|
58
163
|
const trimmed = value.trim();
|
|
59
164
|
return trimmed.length > max ? `${trimmed.slice(0, max)}...` : trimmed;
|
|
60
165
|
}
|
|
166
|
+
function diagnosticHttpFailure(status, body, target) {
|
|
167
|
+
const detail = truncate(body, 500);
|
|
168
|
+
if (status === 403 && /<title>403 Forbidden|does not have permission/i.test(body)) {
|
|
169
|
+
return (`HTTP 403: Cloud Run denied invocation of ${target}. ` +
|
|
170
|
+
'The active Google identity needs roles/run.invoker through the control-plane operator roster.');
|
|
171
|
+
}
|
|
172
|
+
if (status === 403) {
|
|
173
|
+
return (`HTTP 403: ${target} rejected the operator identity. ` +
|
|
174
|
+
`Check DIAGNOSE_OPS_ALLOWED_EMAILS${detail ? ` (${detail})` : '.'}`);
|
|
175
|
+
}
|
|
176
|
+
if (status === 401) {
|
|
177
|
+
return `HTTP 401: ${target} rejected the ID-token audience or credential${detail ? ` (${detail})` : '.'}`;
|
|
178
|
+
}
|
|
179
|
+
if (status === 503) {
|
|
180
|
+
return (`HTTP 503: ${target} has no healthy serving revision. ` +
|
|
181
|
+
'Inspect the Cloud Run revision startup logs and deployed migration/config provenance.');
|
|
182
|
+
}
|
|
183
|
+
return `HTTP ${status}${detail ? `: ${detail}` : ''}`;
|
|
184
|
+
}
|
|
61
185
|
export async function fetchDiagnose(target, authHeader) {
|
|
62
186
|
try {
|
|
63
187
|
const res = await fetch(`${target.url}/diagnose`, { headers: { Authorization: authHeader } });
|
|
@@ -65,7 +189,7 @@ export async function fetchDiagnose(target, authHeader) {
|
|
|
65
189
|
const body = await res.text().catch(() => '');
|
|
66
190
|
return {
|
|
67
191
|
service: target.service,
|
|
68
|
-
error:
|
|
192
|
+
error: diagnosticHttpFailure(res.status, body, target.url),
|
|
69
193
|
};
|
|
70
194
|
}
|
|
71
195
|
return (await res.json());
|
|
@@ -100,14 +224,44 @@ export function rollupVerdict(services) {
|
|
|
100
224
|
}
|
|
101
225
|
export async function fanoutDiagnose(env) {
|
|
102
226
|
const targets = resolveTargets(env);
|
|
103
|
-
const
|
|
104
|
-
const results = await Promise.all(targets.map((target) => fetchDiagnose(target, authHeader)));
|
|
227
|
+
const results = await Promise.all(targets.map(async (target) => fetchDiagnose(target, await resolveOpsAuthHeader(process.env, target.url, target.serviceAccount))));
|
|
105
228
|
const services = {};
|
|
106
229
|
results.forEach((result, index) => {
|
|
107
230
|
services[targets[index].service] = result;
|
|
108
231
|
});
|
|
109
232
|
return { rollup: rollupVerdict(services), services };
|
|
110
233
|
}
|
|
234
|
+
export function projectIdFromInput(value) {
|
|
235
|
+
const trimmed = value.trim();
|
|
236
|
+
try {
|
|
237
|
+
const url = new URL(trimmed);
|
|
238
|
+
const match = /^\/project\/([^/?#]+)/.exec(url.pathname);
|
|
239
|
+
if (match?.[1])
|
|
240
|
+
return decodeURIComponent(match[1]);
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
}
|
|
244
|
+
if (/^[A-Za-z0-9_.:-]{1,160}$/.test(trimmed))
|
|
245
|
+
return trimmed;
|
|
246
|
+
throw new Error('--project must be a project id or a newly.app/project/<id> URL');
|
|
247
|
+
}
|
|
248
|
+
export async function fetchSessionDiagnosticReport(input) {
|
|
249
|
+
const params = new URLSearchParams();
|
|
250
|
+
if (input.session)
|
|
251
|
+
params.set('session', input.session);
|
|
252
|
+
if (input.project)
|
|
253
|
+
params.set('project', projectIdFromInput(input.project));
|
|
254
|
+
if (input.at)
|
|
255
|
+
params.set('at', input.at);
|
|
256
|
+
const res = await fetch(`${input.target.url}/diagnose/session?${params}`, {
|
|
257
|
+
headers: { Authorization: input.authHeader },
|
|
258
|
+
});
|
|
259
|
+
if (!res.ok) {
|
|
260
|
+
const body = await res.text().catch(() => '');
|
|
261
|
+
throw new Error(`Session diagnostics failed: ${diagnosticHttpFailure(res.status, body, input.target.url)}`);
|
|
262
|
+
}
|
|
263
|
+
return (await res.json());
|
|
264
|
+
}
|
|
111
265
|
export function renderRollup(result, session) {
|
|
112
266
|
const lines = [result.rollup];
|
|
113
267
|
if (session) {
|
|
@@ -135,21 +289,54 @@ export function renderRollup(result, session) {
|
|
|
135
289
|
export function registerDiagnose(program) {
|
|
136
290
|
program
|
|
137
291
|
.command('diagnose')
|
|
138
|
-
.description('
|
|
139
|
-
.option('--env <env>', 'dev or
|
|
140
|
-
.option('--session <id>', '
|
|
292
|
+
.description('Diagnose control-plane health or trace one cloud session end to end (dev/ops only)')
|
|
293
|
+
.option('--env <env>', 'dev, prod, or pr-<number>', 'dev')
|
|
294
|
+
.option('--session <id>', 'diagnose this exact session')
|
|
295
|
+
.option('--project <id-or-url>', 'diagnose the latest/nearest session for a project')
|
|
296
|
+
.option('--at <iso-time>', 'select the project session nearest this incident time')
|
|
297
|
+
.option('--out <dir>', 'write evidence, report, diagram, and agent repro/fix prompt')
|
|
141
298
|
.option('--json', 'emit the merged JSON document instead of the human rollup')
|
|
142
299
|
.action(async (opts) => {
|
|
143
300
|
try {
|
|
301
|
+
if (opts.session || opts.project) {
|
|
302
|
+
if (Boolean(opts.session) === Boolean(opts.project)) {
|
|
303
|
+
throw new Error('Provide exactly one of --session or --project');
|
|
304
|
+
}
|
|
305
|
+
if (opts.at && !opts.project) {
|
|
306
|
+
throw new Error('--at requires --project because an exact --session needs no time selection');
|
|
307
|
+
}
|
|
308
|
+
if (opts.at && Number.isNaN(Date.parse(opts.at))) {
|
|
309
|
+
throw new Error('--at must be an ISO timestamp');
|
|
310
|
+
}
|
|
311
|
+
const target = resolveSessionDiagnoseTarget(opts.env);
|
|
312
|
+
const authHeader = await resolveOpsAuthHeader(process.env, target.audience, target.serviceAccount);
|
|
313
|
+
const report = await fetchSessionDiagnosticReport({
|
|
314
|
+
target,
|
|
315
|
+
authHeader,
|
|
316
|
+
...(opts.session ? { session: opts.session } : {}),
|
|
317
|
+
...(opts.project ? { project: opts.project } : {}),
|
|
318
|
+
...(opts.at ? { at: opts.at } : {}),
|
|
319
|
+
});
|
|
320
|
+
if (opts.out) {
|
|
321
|
+
const output = await writeSessionDiagnosticBundle(report, opts.out);
|
|
322
|
+
if (!opts.json)
|
|
323
|
+
console.error(`Diagnostic evidence written to ${output}`);
|
|
324
|
+
}
|
|
325
|
+
console.log(opts.json ? JSON.stringify(report, null, 2) : renderSessionDiagnosticTerminal(report));
|
|
326
|
+
return;
|
|
327
|
+
}
|
|
328
|
+
if (opts.at || opts.out) {
|
|
329
|
+
throw new Error('--at and --out require --session or --project');
|
|
330
|
+
}
|
|
144
331
|
if (opts.env !== 'dev' && opts.env !== 'prod') {
|
|
145
|
-
throw new Error('--env
|
|
332
|
+
throw new Error('Global health diagnosis supports only --env dev or prod');
|
|
146
333
|
}
|
|
147
334
|
const result = await fanoutDiagnose(opts.env);
|
|
148
335
|
if (opts.json) {
|
|
149
|
-
console.log(JSON.stringify(
|
|
336
|
+
console.log(JSON.stringify(result, null, 2));
|
|
150
337
|
return;
|
|
151
338
|
}
|
|
152
|
-
console.log(renderRollup(result
|
|
339
|
+
console.log(renderRollup(result));
|
|
153
340
|
}
|
|
154
341
|
catch (e) {
|
|
155
342
|
console.error(e instanceof Error ? e.message : String(e));
|
|
@@ -0,0 +1,192 @@
|
|
|
1
|
+
import { mkdir, writeFile } from 'node:fs/promises';
|
|
2
|
+
import { resolve } from 'node:path';
|
|
3
|
+
const STATUS_TEXT = {
|
|
4
|
+
ok: 'OK',
|
|
5
|
+
failed: 'FAIL',
|
|
6
|
+
blocked: 'BLOCKED',
|
|
7
|
+
pending: 'WAIT',
|
|
8
|
+
not_requested: 'NOT SENT',
|
|
9
|
+
unknown: 'UNKNOWN',
|
|
10
|
+
};
|
|
11
|
+
const MERMAID_CLASS = {
|
|
12
|
+
ok: 'ok',
|
|
13
|
+
failed: 'failed',
|
|
14
|
+
blocked: 'failed',
|
|
15
|
+
pending: 'pending',
|
|
16
|
+
not_requested: 'unknown',
|
|
17
|
+
unknown: 'unknown',
|
|
18
|
+
};
|
|
19
|
+
function stage(report, id) {
|
|
20
|
+
return report.stages.find((candidate) => candidate.id === id);
|
|
21
|
+
}
|
|
22
|
+
function nodeLabel(report, id) {
|
|
23
|
+
const item = stage(report, id);
|
|
24
|
+
return `[${STATUS_TEXT[item.status]} ${item.label}]`;
|
|
25
|
+
}
|
|
26
|
+
export function renderAsciiDiagram(report) {
|
|
27
|
+
const main = ['session', 'venue', 'runner', 'command', 'metro', 'preview'];
|
|
28
|
+
const mainLine = main.map((id) => nodeLabel(report, id)).join(' -> ');
|
|
29
|
+
const runnerOffset = main.slice(0, 3).map((id) => nodeLabel(report, id)).join(' -> ').length;
|
|
30
|
+
const editor = nodeLabel(report, 'code_server');
|
|
31
|
+
return [mainLine, `${' '.repeat(Math.max(0, runnerOffset - 2))}\\-> ${editor}`].join('\n');
|
|
32
|
+
}
|
|
33
|
+
function mermaidEscape(value) {
|
|
34
|
+
return value
|
|
35
|
+
.replace(/&/g, '&')
|
|
36
|
+
.replace(/</g, '<')
|
|
37
|
+
.replace(/>/g, '>')
|
|
38
|
+
.replace(/\\/g, '\\\\')
|
|
39
|
+
.replace(/"/g, "'")
|
|
40
|
+
.replace(/\r?\n/g, ' ');
|
|
41
|
+
}
|
|
42
|
+
export function renderMermaidDiagram(report) {
|
|
43
|
+
const lines = [
|
|
44
|
+
'flowchart LR',
|
|
45
|
+
' session --> venue --> runner --> command --> metro --> preview',
|
|
46
|
+
' runner --> code_server',
|
|
47
|
+
];
|
|
48
|
+
for (const item of report.stages) {
|
|
49
|
+
lines.push(` ${item.id}["${mermaidEscape(`${STATUS_TEXT[item.status]}: ${item.label}<br/>${item.summary}`)}"]`, ` class ${item.id} ${MERMAID_CLASS[item.status]}`);
|
|
50
|
+
const source = report.sources.find((candidate) => item.sourceRefIds.includes(candidate.id));
|
|
51
|
+
if (source)
|
|
52
|
+
lines.push(` click ${item.id} "${source.url}" "Open ${mermaidEscape(source.symbol)}"`);
|
|
53
|
+
}
|
|
54
|
+
lines.push(' classDef ok fill:#d9f2e3,stroke:#237a45,color:#102a1c', ' classDef failed fill:#fde2e2,stroke:#b42318,color:#3b0a08,stroke-width:3px', ' classDef pending fill:#fff1cc,stroke:#a15c00,color:#3d2600', ' classDef unknown fill:#eceff3,stroke:#667085,color:#1d2939');
|
|
55
|
+
return lines.join('\n');
|
|
56
|
+
}
|
|
57
|
+
function markdownTable(rows) {
|
|
58
|
+
if (rows.length === 0)
|
|
59
|
+
return '';
|
|
60
|
+
const escaped = rows.map((row) => row.map((cell) => cell.replace(/\|/g, '\\|').replace(/\r?\n/g, ' ')));
|
|
61
|
+
const header = escaped[0];
|
|
62
|
+
return [
|
|
63
|
+
`| ${header.join(' | ')} |`,
|
|
64
|
+
`| ${header.map(() => '---').join(' | ')} |`,
|
|
65
|
+
...escaped.slice(1).map((row) => `| ${row.join(' | ')} |`),
|
|
66
|
+
].join('\n');
|
|
67
|
+
}
|
|
68
|
+
function compactData(data) {
|
|
69
|
+
const text = JSON.stringify(data);
|
|
70
|
+
return text.length > 240 ? `${text.slice(0, 237)}...` : text;
|
|
71
|
+
}
|
|
72
|
+
export function renderSessionDiagnosticMarkdown(report) {
|
|
73
|
+
const stageRows = report.stages.map((item) => {
|
|
74
|
+
const links = item.sourceRefIds
|
|
75
|
+
.map((id) => report.sources.find((source) => source.id === id))
|
|
76
|
+
.filter((source) => Boolean(source))
|
|
77
|
+
.map((source) => `[${source.symbol}](${source.url})`)
|
|
78
|
+
.join(', ');
|
|
79
|
+
return [item.label, STATUS_TEXT[item.status], item.summary, links || '-'];
|
|
80
|
+
});
|
|
81
|
+
const timelineRows = report.events.map((event) => [
|
|
82
|
+
event.ts,
|
|
83
|
+
event.severity.toUpperCase(),
|
|
84
|
+
event.kind,
|
|
85
|
+
event.correlationId ?? '-',
|
|
86
|
+
compactData(event.data),
|
|
87
|
+
]);
|
|
88
|
+
const sourceRows = report.sources.map((source) => [
|
|
89
|
+
source.label,
|
|
90
|
+
`[${source.path}:${source.line}](${source.url})`,
|
|
91
|
+
source.symbol,
|
|
92
|
+
]);
|
|
93
|
+
const runtime = report.session.runtime
|
|
94
|
+
? `\n\n\`\`\`json\n${JSON.stringify(report.session.runtime, null, 2)}\n\`\`\``
|
|
95
|
+
: '\n\nNo live runtime snapshot was captured.';
|
|
96
|
+
return `# Cloud Session Diagnosis
|
|
97
|
+
|
|
98
|
+
**Root cause:** \`${report.rootCause.code}\` at **${report.rootCause.stage ?? 'no single stage'}**
|
|
99
|
+
|
|
100
|
+
${report.rootCause.summary}
|
|
101
|
+
|
|
102
|
+
- Session: \`${report.session.id}\`
|
|
103
|
+
- Project: [\`${report.session.projectId}\`](${report.links.project})
|
|
104
|
+
- Environment: \`${report.env}\`
|
|
105
|
+
- Incident time: \`${report.query.at ?? 'latest session state'}\`
|
|
106
|
+
${report.query.evidenceUntil ? `- Historical evidence cutoff: \`${report.query.evidenceUntil}\`` : ''}
|
|
107
|
+
- Deployed revision: \`${report.deploy.revision ?? 'unknown'}\`
|
|
108
|
+
- Deployed commit: ${report.links.githubCommit ? `[\`${report.deploy.git_sha}\`](${report.links.githubCommit})` : 'unknown'}
|
|
109
|
+
- Venue: \`${report.session.venueRef ?? report.session.venue ?? 'not recorded'}\`
|
|
110
|
+
- [Correlated Cloud Run logs](${report.links.cloudLogs})
|
|
111
|
+
${report.links.venueLogsCommand ? `- Retained venue logs: \`${report.links.venueLogsCommand}\`` : ''}
|
|
112
|
+
|
|
113
|
+
## Failure Diagram
|
|
114
|
+
|
|
115
|
+
\`\`\`mermaid
|
|
116
|
+
${renderMermaidDiagram(report)}
|
|
117
|
+
\`\`\`
|
|
118
|
+
|
|
119
|
+
## Stage Diagnosis
|
|
120
|
+
|
|
121
|
+
${markdownTable([['Stage', 'State', 'Evidence-based conclusion', 'Implementation'], ...stageRows])}
|
|
122
|
+
|
|
123
|
+
## Runtime Snapshot${runtime}
|
|
124
|
+
|
|
125
|
+
## Diagnostic Timeline
|
|
126
|
+
|
|
127
|
+
${timelineRows.length > 0 ? markdownTable([['Time', 'Level', 'Event', 'Command', 'Data'], ...timelineRows]) : 'No diagnostic events were collected for this session.'}
|
|
128
|
+
|
|
129
|
+
## How The System Works
|
|
130
|
+
|
|
131
|
+
${markdownTable([['Component', 'Source lines', 'Symbol'], ...sourceRows])}
|
|
132
|
+
`;
|
|
133
|
+
}
|
|
134
|
+
export function renderSessionDiagnosticTerminal(report) {
|
|
135
|
+
const lines = [
|
|
136
|
+
`${report.rootCause.code}: ${report.rootCause.summary}`,
|
|
137
|
+
`session ${report.session.id} | project ${report.session.projectId} | env ${report.env}`,
|
|
138
|
+
`venue ${report.session.venueRef ?? report.session.venue ?? '(not recorded)'}`,
|
|
139
|
+
'',
|
|
140
|
+
renderAsciiDiagram(report),
|
|
141
|
+
'',
|
|
142
|
+
];
|
|
143
|
+
for (const item of report.stages) {
|
|
144
|
+
const source = report.sources.find((candidate) => item.sourceRefIds.includes(candidate.id));
|
|
145
|
+
lines.push(`${STATUS_TEXT[item.status].padEnd(8)} ${item.label}: ${item.summary}`);
|
|
146
|
+
if (source)
|
|
147
|
+
lines.push(` ${source.url}`);
|
|
148
|
+
}
|
|
149
|
+
lines.push('', `Cloud logs: ${report.links.cloudLogs}`);
|
|
150
|
+
if (report.links.venueLogsCommand) {
|
|
151
|
+
lines.push(`Venue logs: ${report.links.venueLogsCommand}`);
|
|
152
|
+
}
|
|
153
|
+
if (report.rootCause.code === 'evidence_missing') {
|
|
154
|
+
lines.push('', 'Evidence repair: verify the session_diagnostic_events migration, matching API/worker/launcher provenance, and the controlled PR preview smoke before diagnosing the product symptom.');
|
|
155
|
+
}
|
|
156
|
+
return lines.join('\n');
|
|
157
|
+
}
|
|
158
|
+
export function renderReproFixPrompt(report) {
|
|
159
|
+
const failedStage = report.rootCause.stage
|
|
160
|
+
? report.stages.find((candidate) => candidate.id === report.rootCause.stage)
|
|
161
|
+
: undefined;
|
|
162
|
+
const evidenceIds = failedStage?.evidenceEventIds.length
|
|
163
|
+
? failedStage.evidenceEventIds.map((id) => `#${id}`).join(', ')
|
|
164
|
+
: 'none attached to the primary stage';
|
|
165
|
+
return `# Reproduce And Fix This Cloud Session Failure
|
|
166
|
+
|
|
167
|
+
Investigate session \`${report.session.id}\` for project \`${report.session.projectId}\` in \`${report.env}\`.
|
|
168
|
+
The collected diagnosis is \`${report.rootCause.code}\`: ${report.rootCause.summary}
|
|
169
|
+
Primary-stage evidence event ids: ${evidenceIds}.
|
|
170
|
+
|
|
171
|
+
Use the adjacent \`bundle.json\` as evidence, not as a conclusion to repeat blindly. Follow the deployed source URLs in \`sources\`, inspect the correlated Cloud Run logs, and run \`${report.links.venueLogsCommand ?? 'the venue-specific retained log command from the report'}\` when applicable.
|
|
172
|
+
|
|
173
|
+
If the diagnosis is \`evidence_missing\`, repair evidence collection first: verify the \`session_diagnostic_events\` migration is applied, API/worker/launcher provenance matches, and the controlled PR preview smoke persists its full correlated timeline. Do not infer a product cause from absent telemetry.
|
|
174
|
+
|
|
175
|
+
1. Identify the first boundary whose recorded output does not match its required input.
|
|
176
|
+
2. Add or extend the smallest regression test that reproduces that exact failure. For cross-service delivery, use the normal PR environment smoke/e2e path instead of mocks alone.
|
|
177
|
+
3. Fix the production path durably in \`control-plane/\`, \`newly-desktop/\`, or \`newly-cli/\`. Do not add work to the frozen legacy Modal backend or Neon stack.
|
|
178
|
+
4. Prove the fix with the regression test, relevant unit/integration tests, and a deployed PR run that records the expected command/process timeline.
|
|
179
|
+
5. Re-run \`newly diagnose --env ${report.env} --session ${report.session.id}\` and explain any remaining \`FAIL\`, \`BLOCKED\`, or \`UNKNOWN\` stage. Do not call missing evidence healthy.
|
|
180
|
+
`;
|
|
181
|
+
}
|
|
182
|
+
export async function writeSessionDiagnosticBundle(report, outDir) {
|
|
183
|
+
const absolute = resolve(outDir);
|
|
184
|
+
await mkdir(absolute, { recursive: true });
|
|
185
|
+
await Promise.all([
|
|
186
|
+
writeFile(resolve(absolute, 'bundle.json'), `${JSON.stringify(report, null, 2)}\n`, 'utf8'),
|
|
187
|
+
writeFile(resolve(absolute, 'report.md'), `${renderSessionDiagnosticMarkdown(report)}\n`, 'utf8'),
|
|
188
|
+
writeFile(resolve(absolute, 'diagram.mmd'), `${renderMermaidDiagram(report)}\n`, 'utf8'),
|
|
189
|
+
writeFile(resolve(absolute, 'repro-fix-prompt.md'), renderReproFixPrompt(report), 'utf8'),
|
|
190
|
+
]);
|
|
191
|
+
return absolute;
|
|
192
|
+
}
|
package/dist/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const CLI_VERSION = '0.1.
|
|
1
|
+
export const CLI_VERSION = '0.1.103';
|