runcloud 0.2.166 → 0.2.167
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/run-cloud-fleet-observations.js +194 -0
- package/dist/commands/run-cloud-fleet-survey.js +252 -0
- package/dist/commands/run-cloud-fleet.js +24 -140
- package/dist/commands/run-cloud-machine-diagnosis.js +136 -0
- package/dist/version.js +1 -1
- package/package.json +1 -1
- package/runcloud-0.2.167.tgz +0 -0
- package/runcloud-0.2.166.tgz +0 -0
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
import { jenkinsObservedState, machineDiagnosis } from './run-cloud-machine-diagnosis.js';
|
|
2
|
+
export const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
3
|
+
export const rows = (value) => Array.isArray(value) ? value.map(row) : [];
|
|
4
|
+
export const str = (value) => typeof value === 'string' && value.length > 0 ? value : null;
|
|
5
|
+
export const num = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
|
|
6
|
+
export const stamp = (value) => str(value) && Number.isFinite(Date.parse(String(value))) ? new Date(String(value)).toISOString() : null;
|
|
7
|
+
export const time = (value) => stamp(value) ? Date.parse(String(value)) : 0;
|
|
8
|
+
export const validFleetId = (value) => typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,255}$/.test(value);
|
|
9
|
+
export function safeFleetText(value) {
|
|
10
|
+
const text = str(value);
|
|
11
|
+
return text ? text.slice(0, 8000)
|
|
12
|
+
.replace(/\bBearer\s+[^\s"'<>]+/gi, 'Bearer [redacted]')
|
|
13
|
+
.replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
|
|
14
|
+
.replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
|
|
15
|
+
.replace(/\b([a-zA-Z0-9_]*(?:token|password|secret|api[_-]?key)|authorization)\s*[=:]\s*[^\s,;]+/gi, '$1=[redacted]')
|
|
16
|
+
.replace(/(?:https?|wss?):\/\/[^\s<>"']+/gi, '[url redacted]') : null;
|
|
17
|
+
}
|
|
18
|
+
export function observation(value, now, ttl, error) {
|
|
19
|
+
const observedAt = stamp(value);
|
|
20
|
+
const ageSeconds = observedAt ? (now - Date.parse(observedAt)) / 1000 : null;
|
|
21
|
+
if (error)
|
|
22
|
+
return { observedAt, ageSeconds, state: 'unavailable', reason: 'collection_failed; retained evidence only' };
|
|
23
|
+
if (!observedAt)
|
|
24
|
+
return { observedAt, ageSeconds, state: 'missing', reason: 'observation_timestamp_missing' };
|
|
25
|
+
if (ageSeconds < -5)
|
|
26
|
+
return { observedAt, ageSeconds, state: 'unavailable', reason: 'observation_timestamp_in_future' };
|
|
27
|
+
return { observedAt, ageSeconds, state: ageSeconds > ttl ? 'stale' : 'fresh', reason: ageSeconds > ttl ? 'observation_expired' : null };
|
|
28
|
+
}
|
|
29
|
+
export function hostedObservation(host, now, ttl) {
|
|
30
|
+
const heartbeat = observation(host.observedAt, now, ttl);
|
|
31
|
+
const freshness = row(host.freshness);
|
|
32
|
+
if (!Object.keys(freshness).length)
|
|
33
|
+
return { ...heartbeat, ingestion: null, ingestionHeartbeat: null, reportedFreshness: null };
|
|
34
|
+
const ingestion = observation(freshness.lastSuccessfulUpdateAt, now, ttl);
|
|
35
|
+
const ingestionHeartbeat = observation(freshness.lastHeartbeatAt, now, ttl);
|
|
36
|
+
const reportedFreshness = str(freshness.status);
|
|
37
|
+
if (heartbeat.state === 'fresh' && (reportedFreshness !== 'fresh' || ingestion.state !== 'fresh' || ingestionHeartbeat.state !== 'fresh')) {
|
|
38
|
+
const state = [ingestion, ingestionHeartbeat].some(sample => sample.state === 'unavailable') ? 'unavailable' : reportedFreshness === 'never' ? 'missing' : 'stale';
|
|
39
|
+
const reason = ingestion.state !== 'fresh' ? `hosted_ingestion_${ingestion.state}` : ingestionHeartbeat.state !== 'fresh' ? `hosted_heartbeat_${ingestionHeartbeat.state}` : `hosted_ingestion_${reportedFreshness ?? 'unknown'}`;
|
|
40
|
+
return { ...heartbeat, state: state, reason, ingestion, ingestionHeartbeat, reportedFreshness };
|
|
41
|
+
}
|
|
42
|
+
return { ...heartbeat, ingestion, ingestionHeartbeat, reportedFreshness };
|
|
43
|
+
}
|
|
44
|
+
export function componentHosts(shared) {
|
|
45
|
+
const result = new Map();
|
|
46
|
+
const snapshots = rows(shared.snapshots).map(entry => row(entry.snapshot));
|
|
47
|
+
for (const snapshot of snapshots)
|
|
48
|
+
for (const raw of rows(snapshot.hosts)) {
|
|
49
|
+
const host = { ...raw, inventorySource: 'shared', collectorError: snapshot.collectorError ?? null };
|
|
50
|
+
const id = str(host.id);
|
|
51
|
+
if (!id)
|
|
52
|
+
continue;
|
|
53
|
+
const before = result.get(id) ?? {};
|
|
54
|
+
const newest = !result.has(id) || time(host.observedAt) >= time(before.observedAt) ? host : before;
|
|
55
|
+
const merged = { ...before, ...newest };
|
|
56
|
+
for (const key of ['hostTelemetry', 'hostDiagnostics', 'processScan']) {
|
|
57
|
+
const candidate = row(host[key]), previous = row(before[key]);
|
|
58
|
+
const observationKey = key === 'processScan' ? 'attemptedAt' : 'observedAt';
|
|
59
|
+
merged[key] = time(candidate[observationKey] ?? candidate.observedAt) >= time(previous[observationKey] ?? previous.observedAt) && Object.keys(candidate).length ? candidate : previous;
|
|
60
|
+
}
|
|
61
|
+
result.set(id, merged);
|
|
62
|
+
}
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
export function cpuReading(host, extra, now, fleetTtl) {
|
|
66
|
+
const telemetry = row(extra.hostTelemetry), cpu = row(telemetry.cpu);
|
|
67
|
+
const cpuValue = (value) => num(value) !== null && Number(value) <= 100 ? Number(value) : null;
|
|
68
|
+
const candidates = [
|
|
69
|
+
{ percent: cpuValue(cpu.busyPercent), source: 'operator_resource_probe', ...observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError) },
|
|
70
|
+
{ percent: cpuValue(host.cpuUtilizationPercent), source: 'host_heartbeat', ...hostedObservation(host, now, fleetTtl) },
|
|
71
|
+
].map(entry => entry.percent === null && entry.state === 'fresh' ? { ...entry, state: 'missing', reason: 'cpu_value_missing' } : entry);
|
|
72
|
+
candidates.sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
|
|
73
|
+
return { ...candidates[0], candidates, meaning: 'host busy percent, 0..100; fresh samples only in fleet mean' };
|
|
74
|
+
}
|
|
75
|
+
export const pick = (source, keys) => Object.fromEntries(keys.filter(key => source[key] !== undefined).map(key => [key, source[key]]));
|
|
76
|
+
function fence(value) {
|
|
77
|
+
const source = row(value);
|
|
78
|
+
return Object.keys(source).length ? { ...pick(source, ['id', 'kind', 'state', 'createdAt', 'expiresAt', 'component', 'owner', 'operationId']), reason: safeFleetText(source.reason) } : null;
|
|
79
|
+
}
|
|
80
|
+
export function hostReport(host, canonical, extra, diagnoses, now, fleetTtl, deep = {}, context = {}) {
|
|
81
|
+
const id = String(host.id ?? canonical.id ?? extra.id);
|
|
82
|
+
const machineType = str(canonical.machineType ?? host.machineType) ?? (extra.role === 'jenkins' ? 'build' : str(extra.role)) ?? 'simulator';
|
|
83
|
+
const role = machineType === 'build' ? 'jenkins' : machineType;
|
|
84
|
+
const sharedHealth = observation(extra.observedAt, now, context.operatorTtl ?? 120, extra.collectorError);
|
|
85
|
+
const useShared = sharedHealth.state === 'fresh' && time(extra.observedAt) > time(host.observedAt);
|
|
86
|
+
const primary = useShared ? extra : host;
|
|
87
|
+
const capacitySource = primary.capacity != null ? primary : useShared ? host : extra;
|
|
88
|
+
const capacity = row(capacitySource.capacity);
|
|
89
|
+
let health = useShared ? sharedHealth : hostedObservation(host, now, fleetTtl);
|
|
90
|
+
let state = health.state === 'fresh' ? str(primary.state) ?? 'unknown' : 'unknown';
|
|
91
|
+
let reportedState = str(primary.state);
|
|
92
|
+
let diagnosisTtl = useShared ? context.operatorTtl ?? 120 : fleetTtl;
|
|
93
|
+
let diagnosisSource = useShared ? 'shared' : 'hosted';
|
|
94
|
+
if (role === 'jenkins') {
|
|
95
|
+
const jenkins = context.jenkins ?? {};
|
|
96
|
+
const node = rows(jenkins.nodes).find(node => node.id === id || node.id === `${id}-jenkins`);
|
|
97
|
+
reportedState = node?.offline === true ? 'offline' : node?.offline === false ? 'ready' : null;
|
|
98
|
+
diagnosisTtl = num(jenkins.staleAfterSeconds) ?? 30;
|
|
99
|
+
health = observation(jenkins.observedAt, now, diagnosisTtl, jenkins.error || jenkins.state === 'unavailable');
|
|
100
|
+
state = jenkinsObservedState(node ? { offline: typeof node.offline === 'boolean' ? node.offline : undefined } : undefined, health.observedAt, now, diagnosisTtl, health.state === 'unavailable');
|
|
101
|
+
if (!node && health.state === 'fresh')
|
|
102
|
+
health = { ...health, state: 'missing', reason: 'jenkins_node_missing' };
|
|
103
|
+
diagnosisSource = 'jenkins';
|
|
104
|
+
}
|
|
105
|
+
const diagnostic = row(extra.hostDiagnostics), process = row(extra.processScan);
|
|
106
|
+
const supplementalChecks = observation(diagnostic.observedAt, now, num(diagnostic.staleAfterSeconds) ?? 900, diagnostic.collectionError);
|
|
107
|
+
const deepChecks = observation(row(deep.security).checkedAt, now, fleetTtl);
|
|
108
|
+
const useDeep = deepChecks.state === 'fresh' && Array.isArray(deep.services) && deep.services.length > 0 && time(deepChecks.observedAt) > time(supplementalChecks.observedAt);
|
|
109
|
+
const checks = useDeep ? deepChecks : supplementalChecks;
|
|
110
|
+
const processCheck = observation(process.observedAt, now, num(process.staleAfterSeconds) ?? 900, process.error);
|
|
111
|
+
const release = row(host.release);
|
|
112
|
+
const services = rows(useDeep ? deep.services : diagnostic.services).map(service => ({ name: safeFleetText(service.name), state: safeFleetText(service.state), lastExitCode: service.lastExitCode ?? null,
|
|
113
|
+
error: safeFleetText(service.lastError ?? service.error) }));
|
|
114
|
+
const findings = rows(process.findings).map(finding => ({ ...pick(finding, ['fingerprint', 'reason', 'confidence', 'firstObservedAt', 'lastObservedAt', 'ownerUid', 'ageSeconds']), message: safeFleetText(finding.message ?? finding.summary) }));
|
|
115
|
+
const diagnosesForHost = diagnoses.filter(diagnosis => diagnosis.hostId === id).map(diagnosis => ({ ...pick(diagnosis, ['code', 'severity', 'deviceId']),
|
|
116
|
+
title: safeFleetText(diagnosis.title), message: safeFleetText(diagnosis.message), reportedAt: stamp(diagnosis.observedAt) }));
|
|
117
|
+
const admissionObservation = observation(canonical.admissionObservedAt, now, context.inventoryTtl ?? 30);
|
|
118
|
+
const acceptsLeases = canonical.enabled === false || machineType !== 'simulator' || canonical.admissionReason === 'routing-disabled' ? false :
|
|
119
|
+
admissionObservation.state === 'fresh' && typeof canonical.acceptsLeases === 'boolean' ? canonical.acceptsLeases : null;
|
|
120
|
+
const rawRequestedFence = Object.hasOwn(canonical, 'requestedFence') ? canonical.requestedFence : host.requestedFence ?? extra.requestedFence;
|
|
121
|
+
const rawFence = Object.hasOwn(canonical, 'fence') ? canonical.fence : host.fence ?? extra.fence;
|
|
122
|
+
const requestedFence = fence(rawRequestedFence), appliedFence = fence(rawFence);
|
|
123
|
+
if (appliedFence && admissionObservation.state !== 'fresh')
|
|
124
|
+
appliedFence.state = 'unknown';
|
|
125
|
+
const operation = context.operations?.filter(operation => operation.id === id).sort((a, b) => time(b.updatedAt) - time(a.updatedAt))[0];
|
|
126
|
+
const operationFence = operation && acceptsLeases !== true ? fence(operation.fence) : null;
|
|
127
|
+
if (operationFence && (operation.stale || (!['complete', 'reserved', 'cancelled', 'rolled-back'].includes(String(operation.status)) && now - time(operation.updatedAt) > 60000)))
|
|
128
|
+
operationFence.state = 'unknown';
|
|
129
|
+
const displayFence = appliedFence ?? requestedFence ?? operationFence;
|
|
130
|
+
const sample = useDeep ? { observedAt: deepChecks.observedAt, staleAfterSeconds: fleetTtl, services: deep.services } : diagnostic;
|
|
131
|
+
const diagnosisHost = {
|
|
132
|
+
id, role, observedAt: health.observedAt, acceptsLeases,
|
|
133
|
+
capacity: role === 'jenkins' ? null : { dirty: num(capacity.dirty) },
|
|
134
|
+
fence: displayFence ? { kind: str(displayFence.kind) ?? 'unknown', state: str(displayFence.state), reason: str(row(rawFence ?? rawRequestedFence ?? operation?.fence).reason) ?? 'reason missing' } : null,
|
|
135
|
+
requestedFence: requestedFence ? { kind: str(requestedFence.kind) ?? 'unknown' } : null,
|
|
136
|
+
hostDiagnostics: Object.keys(sample).length ? {
|
|
137
|
+
observedAt: stamp(sample.observedAt), staleAfterSeconds: num(sample.staleAfterSeconds) ?? 900,
|
|
138
|
+
collectionError: str(sample.collectionError) ?? (!Array.isArray(sample.services) ? 'service list missing' : null),
|
|
139
|
+
services: rows(sample.services).map(service => ({ name: str(service.name) ?? '', state: str(service.state) })),
|
|
140
|
+
packageMatchesService: typeof sample.packageMatchesService === 'boolean' ? sample.packageMatchesService : null,
|
|
141
|
+
power: { thermalWarning: row(sample.power).thermalWarning === true, performanceWarning: row(sample.power).performanceWarning === true },
|
|
142
|
+
coreSimulator: { orphaned: num(row(sample.coreSimulator).orphaned), externalBooted: num(row(sample.coreSimulator).externalBooted) },
|
|
143
|
+
recording: { failed: num(row(sample.recording).failed) },
|
|
144
|
+
} : null,
|
|
145
|
+
processScan: Object.keys(process).length ? {
|
|
146
|
+
observedAt: stamp(process.observedAt), staleAfterSeconds: num(process.staleAfterSeconds) ?? 900,
|
|
147
|
+
sourceUnavailable: process.sourceUnavailable === true || Boolean(extra.collectorError), status: str(process.status) ?? 'unknown', error: str(process.error), root: process.root === true,
|
|
148
|
+
iocs: rows(process.iocs).map(ioc => ({ status: str(ioc.status) ?? 'unknown', contained: ioc.contained === true })),
|
|
149
|
+
findings: rows(process.findings).map(finding => ({ reason: str(finding.reason) ?? 'unknown' })),
|
|
150
|
+
coverage: { iocs: str(row(process.coverage).iocs) ?? 'partial' }, baseline: { state: str(row(process.baseline).state) ?? 'missing' },
|
|
151
|
+
} : null,
|
|
152
|
+
};
|
|
153
|
+
const observedState = role === 'jenkins' ? state : health.state === 'fresh' ? state : health.state === 'missing' ? 'unknown' : 'stale';
|
|
154
|
+
const releaseFacts = {
|
|
155
|
+
matches: typeof release.matches === 'boolean' ? release.matches : null,
|
|
156
|
+
expectedPackage: str(release.expectedPackage), actualPackage: str(release.actualPackage),
|
|
157
|
+
expectedVersion: str(release.expectedVersion), actualVersion: str(release.actualVersion),
|
|
158
|
+
expectedBuild: str(release.expectedBuild), actualBuild: str(release.actualBuild),
|
|
159
|
+
};
|
|
160
|
+
const diagnosisDetails = machineDiagnosis(diagnosisHost, { release: releaseFacts }, observedState, now, diagnosisTtl);
|
|
161
|
+
const cpu = cpuReading(host, extra, now, fleetTtl);
|
|
162
|
+
return { id, name: safeFleetText(canonical.name ?? host.name ?? extra.name), platform: canonical.platform ?? host.platform ?? extra.platform ?? null,
|
|
163
|
+
role, machineType, inventoryPresent: Object.keys(canonical).length > 0, livePresent: Object.keys(host).length > 0,
|
|
164
|
+
health: { reportedState, effectiveState: state, source: diagnosisSource, ...health,
|
|
165
|
+
...(role === 'jenkins' ? { hostedEvidence: { reportedState: str(host.state), ...hostedObservation(host, now, fleetTtl) } } : {}) }, diagnosis: diagnosisDetails.summary, diagnosisDetails, cpu,
|
|
166
|
+
diagnosisObservation: { observedAt: health.observedAt, observedState, staleAfterSeconds: diagnosisTtl, source: diagnosisSource,
|
|
167
|
+
hostedObservedAt: stamp(host.observedAt), diagnosticsObservedAt: stamp(sample.observedAt), processObservedAt: stamp(process.observedAt) },
|
|
168
|
+
release: { ...releaseFacts, version: str(host.version), build: str(host.build), observedAt: stamp(host.observedAt) },
|
|
169
|
+
admission: { enabled: canonical.enabled ?? null, acceptsLeases, observation: admissionObservation, observedAt: stamp(canonical.admissionObservedAt), reason: safeFleetText(canonical.admissionReason), requestedFence, fence: appliedFence, operationFence,
|
|
170
|
+
operation: operation ? { operationId: safeFleetText(operation.operationId), kind: safeFleetText(operation.kind), component: safeFleetText(operation.component),
|
|
171
|
+
phase: safeFleetText(operation.phase), status: safeFleetText(operation.status), progress: num(operation.progress), updatedAt: stamp(operation.updatedAt), owner: safeFleetText(operation.owner),
|
|
172
|
+
...observation(operation.updatedAt, now, 60, operation.stale), error: safeFleetText(operation.error) } : null },
|
|
173
|
+
capacity: role === 'jenkins' ? null : pick(capacity, ['clean', 'leased', 'installing', 'dirty', 'preparing', 'recycling', 'total']),
|
|
174
|
+
capacityObservation: capacitySource === host ? { source: 'hosted', ...hostedObservation(host, now, fleetTtl) } : {
|
|
175
|
+
source: 'shared', ...observation(extra.observedAt, now, context.operatorTtl ?? 120, extra.collectorError)
|
|
176
|
+
},
|
|
177
|
+
serviceChecks: { ...checks, source: useDeep ? 'hosted_deep_evidence' : 'operator_diagnostics', services,
|
|
178
|
+
software: { ...pick(row(diagnostic.software), ['serveSimPackage', 'serveSimVersion', 'serveSimRevision', 'xcodeVersion']), observation: supplementalChecks, packageMatchesService: diagnostic.packageMatchesService ?? null } },
|
|
179
|
+
processChecks: { ...processCheck, ...pick(process, ['attemptedAt', 'inventoryObservedAt', 'maxVariableAgeSeconds', 'root', 'mode', 'coverage', 'counts']),
|
|
180
|
+
status: process.status ?? null, baseline: pick(row(process.baseline), ['state', 'profileIds', 'methods']),
|
|
181
|
+
transportTrust: pick(row(process.transportTrust), ['state', 'verifiedAt', 'pinsSha256']), findings,
|
|
182
|
+
iocs: rows(process.iocs).map(ioc => ({ ...pick(ioc, ['family', 'status', 'contained', 'lastObservedAt']), signals: Array.isArray(ioc.signals) ? ioc.signals.map(safeFleetText) : [] })) },
|
|
183
|
+
resources: { ...observation(row(extra.hostTelemetry).observedAt, now, 45, row(extra.hostTelemetry).collectionError),
|
|
184
|
+
memory: pick(row(row(extra.hostTelemetry).memory), ['totalBytes', 'usedBytes', 'availableBytes', 'swapUsedBytes']),
|
|
185
|
+
perDeviceCpu: rows(row(row(extra.hostTelemetry).cpu).perDevice).map(device => pick(device, ['id', 'cpuPercent', 'rssBytes', 'processes'])),
|
|
186
|
+
processCpuMeaning: 'OS process percent may exceed 100 on multicore hosts; independent from host busy percent' },
|
|
187
|
+
devices: rows(host.devices).map(device => ({ id: str(device.id), ...pick(device, ['state', 'sessionId', 'leaseId', 'updatedAt']), helper: {
|
|
188
|
+
...pick(row(device.helper), ['state', 'pid', 'lastSeenAt']), observation: observation(row(device.helper).lastSeenAt, now, fleetTtl),
|
|
189
|
+
observationMeaning: 'last helper evidence; freshness uses fleet heartbeat threshold', error: safeFleetText(row(device.helper).error)
|
|
190
|
+
}, lastError: safeFleetText(device.lastError) })),
|
|
191
|
+
deviceObservation: { source: 'hosted', ...hostedObservation(host, now, fleetTtl), meaning: 'provider device records only; shared discovery does not establish current leases' },
|
|
192
|
+
diagnoses: diagnosesForHost,
|
|
193
|
+
};
|
|
194
|
+
}
|
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
import { ApiError } from '../api.js';
|
|
2
|
+
import { componentHosts, hostReport, hostedObservation, num, observation, row, rows, safeFleetText, stamp, str, time, validFleetId } from './run-cloud-fleet-observations.js';
|
|
3
|
+
export const SURVEY_REQUEST_BUDGET_MS = 8000;
|
|
4
|
+
const ROOT = '/run-cloud/admin/simulators';
|
|
5
|
+
const RANGES = ['1h', '6h', '24h', '7d'];
|
|
6
|
+
const DEVICE_LIMIT = 512, POINT_LIMIT = 800, TYPE_LIMIT = 100;
|
|
7
|
+
function count(value) { const number = num(value); return number !== null && Number.isSafeInteger(number) ? number : null; }
|
|
8
|
+
function counts(value) {
|
|
9
|
+
const data = row(value), errors = count(data.errors), warnings = count(data.warnings), sessions = count(data.sessions);
|
|
10
|
+
return errors === null || warnings === null || sessions === null ? null : { errors, warnings, sessions };
|
|
11
|
+
}
|
|
12
|
+
const unavailable = (reason) => ({ state: 'unavailable', data: {}, reason });
|
|
13
|
+
function resourceReading(host, extra, kind, now, fleetTtl) {
|
|
14
|
+
const telemetry = row(extra.hostTelemetry), volumes = rows(row(telemetry.storage).volumes);
|
|
15
|
+
const probe = kind === 'memory' ? row(telemetry.memory) : volumes.find(volume => volume.label === 'data') ?? volumes[0] ?? {};
|
|
16
|
+
const candidate = (values, source, observed) => {
|
|
17
|
+
const usedBytes = num(values.usedBytes), totalBytes = num(values.totalBytes), availableBytes = num(values.availableBytes);
|
|
18
|
+
const missing = usedBytes === null || totalBytes === null;
|
|
19
|
+
return { source, ...observed, usedBytes, totalBytes, availableBytes,
|
|
20
|
+
percent: usedBytes !== null && totalBytes !== null && totalBytes > 0 ? usedBytes / totalBytes * 100 : null,
|
|
21
|
+
...(missing && observed.state === 'fresh' ? { state: 'missing', reason: `${kind}_value_missing` } : {}) };
|
|
22
|
+
};
|
|
23
|
+
const candidates = [
|
|
24
|
+
candidate(probe, 'operator_resource_probe', observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError)),
|
|
25
|
+
candidate({ usedBytes: host[`${kind}UsedBytes`], totalBytes: host[`${kind}TotalBytes`] }, 'host_heartbeat', hostedObservation(host, now, fleetTtl)),
|
|
26
|
+
].sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
|
|
27
|
+
return { ...candidates[0], candidates, ...(kind === 'storage' ? { volumes: volumes.map(volume => ({ label: safeFleetText(volume.label),
|
|
28
|
+
usedBytes: num(volume.usedBytes), totalBytes: num(volume.totalBytes), availableBytes: num(volume.availableBytes) })),
|
|
29
|
+
volumeObservation: observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError),
|
|
30
|
+
meaning: 'one reported volume or heartbeat capacity; volumes may share backing storage and are never summed' } : {}) };
|
|
31
|
+
}
|
|
32
|
+
function alertReport(source, now, requestedRange, hostId) {
|
|
33
|
+
const data = source.data;
|
|
34
|
+
const raw = rows(data.points);
|
|
35
|
+
const valid = source.state === 'available' && data.range === requestedRange && stamp(data.generatedAt) && time(data.generatedAt) <= now + 5000 && num(data.bucketSeconds) &&
|
|
36
|
+
raw.every(point => stamp(point.timestamp) && time(point.timestamp) <= time(data.generatedAt) && counts(point) && Array.isArray(point.errorTypes) && rows(point.errorTypes).every(type => str(type.type) && count(type.errors) !== null && count(type.warnings) !== null));
|
|
37
|
+
if (!valid)
|
|
38
|
+
return { state: 'unavailable', reason: source.reason ?? 'invalid_alert_aggregate', byType: null, totals: null, trend: null,
|
|
39
|
+
coverage: { ingestionCompleteness: 'unknown; aggregate API does not certify event ingestion', hostScope: hostId ? [hostId] : 'all hosts' } };
|
|
40
|
+
const points = raw.slice(-POINT_LIMIT).sort((a, b) => time(a.timestamp) - time(b.timestamp));
|
|
41
|
+
const generatedAt = stamp(data.generatedAt), bucketSeconds = Number(data.bucketSeconds);
|
|
42
|
+
const completeBuckets = points.filter(point => time(point.timestamp) + bucketSeconds * 1000 <= time(generatedAt));
|
|
43
|
+
const recent = completeBuckets.at(-1), previous = completeBuckets.at(-2);
|
|
44
|
+
const comparable = recent && previous && time(recent.timestamp) - time(previous.timestamp) === bucketSeconds * 1000;
|
|
45
|
+
const types = new Map();
|
|
46
|
+
let omittedTypeRows = 0;
|
|
47
|
+
for (const point of points) {
|
|
48
|
+
const entries = rows(point.errorTypes);
|
|
49
|
+
omittedTypeRows += Math.max(0, entries.length - TYPE_LIMIT);
|
|
50
|
+
for (const entry of entries.slice(0, TYPE_LIMIT)) {
|
|
51
|
+
const type = String(entry.type), before = types.get(type) ?? { type, errors: 0, warnings: 0, recent: null, previous: null, delta: null };
|
|
52
|
+
before.errors += Number(entry.errors);
|
|
53
|
+
before.warnings += Number(entry.warnings);
|
|
54
|
+
types.set(type, before);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
for (const entry of types.values())
|
|
58
|
+
if (comparable && omittedTypeRows === 0) {
|
|
59
|
+
const inBucket = (point) => rows(point.errorTypes).filter(type => type.type === entry.type).reduce((sum, type) => sum + Number(type.errors) + Number(type.warnings), 0);
|
|
60
|
+
entry.recent = inBucket(recent);
|
|
61
|
+
entry.previous = inBucket(previous);
|
|
62
|
+
entry.delta = entry.recent - entry.previous;
|
|
63
|
+
}
|
|
64
|
+
const window = (point) => ({ since: stamp(point.timestamp), until: new Date(time(point.timestamp) + bucketSeconds * 1000).toISOString(), ...counts(point) });
|
|
65
|
+
return { state: 'available', reason: null, generatedAt, responseAgeSeconds: (now - time(generatedAt)) / 1000,
|
|
66
|
+
byType: [...types.values()].sort((a, b) => b.errors + b.warnings - a.errors - a.warnings || a.type.localeCompare(b.type)),
|
|
67
|
+
observedBucketCounts: points.reduce((total, point) => ({ errors: total.errors + Number(point.errors), warnings: total.warnings + Number(point.warnings) }), { errors: 0, warnings: 0 }),
|
|
68
|
+
totals: { hour: counts(row(data.totals).hour), today: counts(row(data.totals).today), yesterday: counts(row(data.totals).yesterday) },
|
|
69
|
+
windows: { hour: stamp(row(data.windows).hour), today: stamp(row(data.windows).today), yesterday: stamp(row(data.windows).yesterday), yesterdayEnd: stamp(row(data.windows).yesterdayEnd),
|
|
70
|
+
timeZone: str(data.timeZone), until: generatedAt },
|
|
71
|
+
trend: comparable ? { recent: window(recent), previous: window(previous), delta: Number(recent.errors) + Number(recent.warnings) - Number(previous.errors) - Number(previous.warnings),
|
|
72
|
+
meaning: 'last two complete equal buckets; current partial bucket excluded' } : null,
|
|
73
|
+
coverage: { requestedRange, hostScope: hostId ? [hostId] : 'all hosts', bucketSeconds, firstBucketAt: stamp(points[0]?.timestamp), lastBucketAt: stamp(points.at(-1)?.timestamp),
|
|
74
|
+
buckets: points.length, omittedBuckets: Math.max(0, raw.length - POINT_LIMIT), omittedTypeRows, unattributedAlertRows: count(data.unattributedAlerts),
|
|
75
|
+
unattributedMeaning: 'missing customer attribution over the API query window, which may extend before the plotted range',
|
|
76
|
+
meaning: 'observed warning/critical event rows after server bookkeeping exclusions; retries can repeat rows; not unique incidents or a failure rate',
|
|
77
|
+
ingestionCompleteness: 'unknown; aggregate API does not certify event ingestion', windowAlignment: 'API bucket boundaries can extend before the requested rolling window' } };
|
|
78
|
+
}
|
|
79
|
+
function lagReport(source, selection, eligibleDevices, now, range) {
|
|
80
|
+
const data = source.data, selected = new Set(selection.map(([host, device]) => `${host} ${device}`));
|
|
81
|
+
const validSample = (sample) => count(sample.samples) !== null && (sample.samples === 0 ? sample.medianMs === null : num(sample.medianMs) !== null);
|
|
82
|
+
const valid = source.state === 'available' && data.metric === 'input_lag_visual_v1' && data.range === range && stamp(data.generatedAt) && time(data.generatedAt) <= now + 5000 &&
|
|
83
|
+
num(data.bucketSeconds) && count(data.selectedDevices) === selection.length && Array.isArray(data.points) &&
|
|
84
|
+
rows(data.points).every(point => stamp(point.timestamp) && time(point.timestamp) <= time(data.generatedAt) && validSample(point) && count(point.missing) !== null) &&
|
|
85
|
+
data.devices && typeof data.devices === 'object' && !Array.isArray(data.devices) && Object.values(row(data.devices)).every(sample => validSample(row(sample)));
|
|
86
|
+
const points = valid ? rows(data.points).slice(-POINT_LIMIT).sort((a, b) => time(a.timestamp) - time(b.timestamp)) : [];
|
|
87
|
+
const observed = points.filter(point => stamp(point.timestamp) && count(point.samples) !== null && Number(point.samples) > 0 && num(point.medianMs) !== null);
|
|
88
|
+
const latest = observed.at(-1);
|
|
89
|
+
const devices = valid && data.devices && typeof data.devices === 'object' && !Array.isArray(data.devices) ? Object.entries(row(data.devices))
|
|
90
|
+
.filter(([key]) => selected.has(key)).map(([key, value]) => {
|
|
91
|
+
const sample = row(value), samples = count(sample.samples), medianMs = samples !== null && samples > 0 ? num(sample.medianMs) : null;
|
|
92
|
+
const [hostId, deviceId] = key.split(' ');
|
|
93
|
+
return { hostId, deviceId, medianMs, samples, outcomes: sample.outcomes ? outcomeCounts(sample.outcomes) : null,
|
|
94
|
+
observationAgeSeconds: null, meaning: 'median across valid changed samples in query window; API provides no per-device last sample time' };
|
|
95
|
+
}) : null;
|
|
96
|
+
return { state: valid ? 'available' : source.state === 'not_requested' ? 'not_requested' : 'unavailable',
|
|
97
|
+
reason: valid ? null : source.reason ?? 'invalid_input_lag_aggregate', metric: 'input_lag_visual_v1', generatedAt: stamp(data.generatedAt),
|
|
98
|
+
responseAgeSeconds: stamp(data.generatedAt) ? (now - time(data.generatedAt)) / 1000 : null,
|
|
99
|
+
latestSampledBucket: latest ? { timestamp: stamp(latest.timestamp), bucketSeconds: Number(data.bucketSeconds), bucketAgeSeconds: (now - time(latest.timestamp)) / 1000,
|
|
100
|
+
medianMs: Number(latest.medianMs), samples: count(latest.samples), missing: count(latest.missing),
|
|
101
|
+
meaning: 'bucket aggregate for selected devices; bucket age is not the age of its last individual sample' } : null,
|
|
102
|
+
outcomes: valid && data.totals ? outcomeCounts(data.totals) : null, devices,
|
|
103
|
+
coverage: { requestedRange: range, eligibleDevices, requestedDevices: selection.length, apiSelectedDevices: valid ? count(data.selectedDevices) : null,
|
|
104
|
+
omittedDevices: Math.max(0, eligibleDevices - selection.length), deviceLimit: DEVICE_LIMIT,
|
|
105
|
+
unrequestedDeviceRecords: valid ? Object.keys(row(data.devices)).filter(key => !selected.has(key)).length : null,
|
|
106
|
+
returnedDevices: devices?.length ?? null, sampledDevices: devices?.filter(device => device.medianMs !== null).length ?? null,
|
|
107
|
+
firstBucketAt: stamp(points[0]?.timestamp), lastBucketAt: stamp(points.at(-1)?.timestamp), bucketSeconds: valid ? num(data.bucketSeconds) : null,
|
|
108
|
+
omittedBuckets: valid ? Math.max(0, rows(data.points).length - POINT_LIMIT) : null,
|
|
109
|
+
ingestionCompleteness: 'unknown; missing, no-change, superseded, hidden and sample-error outcomes are not successful latency samples' } };
|
|
110
|
+
}
|
|
111
|
+
function outcomeCounts(value) {
|
|
112
|
+
const data = row(value);
|
|
113
|
+
return Object.fromEntries(['changed', 'no_change', 'superseded', 'hidden', 'sample_error'].map(key => [key, count(data[key])]));
|
|
114
|
+
}
|
|
115
|
+
export async function surveyFleet(client, options = {}, now, requestBudgetMs = SURVEY_REQUEST_BUDGET_MS) {
|
|
116
|
+
const range = options.range ?? '24h';
|
|
117
|
+
if (!RANGES.includes(range))
|
|
118
|
+
throw new Error(`survey range must be ${RANGES.join(', ')}`);
|
|
119
|
+
if (options.hostId && !validFleetId(options.hostId))
|
|
120
|
+
throw new Error('Invalid host ID');
|
|
121
|
+
if (!Number.isSafeInteger(requestBudgetMs) || requestBudgetMs < 1 || requestBudgetMs > SURVEY_REQUEST_BUDGET_MS)
|
|
122
|
+
throw new Error('Invalid survey request budget');
|
|
123
|
+
const started = performance.now(), signal = AbortSignal.timeout(requestBudgetMs);
|
|
124
|
+
const read = async (path, key, body) => {
|
|
125
|
+
try {
|
|
126
|
+
const value = body ? await client.post(path, body, { signal }) : await client.get(path, { signal });
|
|
127
|
+
const data = row(value);
|
|
128
|
+
if (value instanceof Response || !Array.isArray(data[key]))
|
|
129
|
+
return unavailable('invalid_response_shape');
|
|
130
|
+
if (path === '/run-cloud/admin/jenkins' && data.state === 'unavailable')
|
|
131
|
+
return { state: 'unavailable', data, reason: 'jenkins_collection_unavailable' };
|
|
132
|
+
if (path === '/run-cloud/admin/fleet-operations' && (data.unavailable === true || data.configured === false))
|
|
133
|
+
return { state: 'unavailable', data, reason: 'operation_storage_unavailable_or_unconfigured' };
|
|
134
|
+
return { state: 'available', data, reason: null };
|
|
135
|
+
}
|
|
136
|
+
catch (error) {
|
|
137
|
+
return unavailable(error instanceof ApiError ? `HTTP ${error.status}` : signal.aborted ? 'request_budget_exhausted' : 'read_failed');
|
|
138
|
+
}
|
|
139
|
+
};
|
|
140
|
+
const fleetPromise = read(ROOT, 'hosts');
|
|
141
|
+
const inventoryPromise = read('/run-cloud/admin/fleet-inventory', 'hosts');
|
|
142
|
+
const sharedPromise = read(`${ROOT}/operator-inventory`, 'snapshots');
|
|
143
|
+
const jenkinsPromise = read('/run-cloud/admin/jenkins', 'nodes');
|
|
144
|
+
const operationsPromise = read('/run-cloud/admin/fleet-operations', 'hosts');
|
|
145
|
+
const analyticsPromise = read(`${ROOT}/analytics`, 'points', { range, ...(options.hostId ? { hostIds: [options.hostId] } : {}) });
|
|
146
|
+
let selection = [], eligibleDevices = 0;
|
|
147
|
+
const lagPromise = fleetPromise.then(async (fleet) => {
|
|
148
|
+
if (fleet.state !== 'available')
|
|
149
|
+
return unavailable('fleet_device_selection_unavailable');
|
|
150
|
+
const unique = new Map();
|
|
151
|
+
for (const host of rows(fleet.data.hosts))
|
|
152
|
+
if (!host.monitoringOnly && (!options.hostId || host.id === options.hostId)) {
|
|
153
|
+
for (const device of rows(host.devices))
|
|
154
|
+
if (validFleetId(host.id) && validFleetId(device.id))
|
|
155
|
+
unique.set(`${host.id} ${device.id}`, [String(host.id), String(device.id)]);
|
|
156
|
+
}
|
|
157
|
+
const devices = [...unique.values()].sort((a, b) => a.join(' ').localeCompare(b.join(' ')));
|
|
158
|
+
eligibleDevices = devices.length;
|
|
159
|
+
selection = devices.slice(0, DEVICE_LIMIT);
|
|
160
|
+
return selection.length ? read(`${ROOT}/input-lag/history`, 'points', { range, devices: selection }) : { state: 'not_requested', data: {}, reason: 'no_queryable_devices' };
|
|
161
|
+
});
|
|
162
|
+
const [fleet, inventory, shared, jenkins, operations, analytics, lag] = await Promise.all([
|
|
163
|
+
fleetPromise, inventoryPromise, sharedPromise, jenkinsPromise, operationsPromise, analyticsPromise, lagPromise,
|
|
164
|
+
]);
|
|
165
|
+
if (fleet.state !== 'available')
|
|
166
|
+
throw new Error(`Fleet read unavailable (${fleet.reason}); use an admin identity with read permission`);
|
|
167
|
+
now ??= Date.now();
|
|
168
|
+
const live = new Map(rows(fleet.data.hosts).filter(host => validFleetId(host.id)).map(host => [String(host.id), host]));
|
|
169
|
+
const canonical = new Map(rows(inventory.data.hosts).filter(host => validFleetId(host.id)).map(host => [String(host.id), host]));
|
|
170
|
+
const components = componentHosts(shared.data), supplemental = new Map([...components].filter(([id]) => validFleetId(id))), ids = new Set([...canonical.keys(), ...live.keys(), ...supplemental.keys()]);
|
|
171
|
+
if (options.hostId && !ids.has(options.hostId))
|
|
172
|
+
throw new Error(`Host ${options.hostId} is absent from returned inventories`);
|
|
173
|
+
const inputLag = lagReport(lag, selection, eligibleDevices, now, range), fleetTtl = num(fleet.data.staleAfterSeconds) ?? 45;
|
|
174
|
+
const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => {
|
|
175
|
+
const host = live.get(id) ?? {}, extra = supplemental.get(id) ?? {};
|
|
176
|
+
const report = hostReport(host, canonical.get(id) ?? {}, extra, [], now, fleetTtl, {}, {
|
|
177
|
+
inventoryTtl: num(inventory.data.staleAfterSeconds) ?? 30, jenkins: jenkins.data,
|
|
178
|
+
operations: rows(operations.data.hosts).map(operation => ({ ...operation, stale: operation.stale || operations.state !== 'available' })),
|
|
179
|
+
});
|
|
180
|
+
const devices = inputLag.devices?.filter(device => device.hostId === id) ?? null;
|
|
181
|
+
const sampled = devices?.filter(device => device.medianMs !== null) ?? [];
|
|
182
|
+
return { ...report, memory: resourceReading(host, extra, 'memory', now, fleetTtl), storage: resourceReading(host, extra, 'storage', now, fleetTtl),
|
|
183
|
+
inputLag: { devices, selectedDevices: selection.filter(([hostId]) => hostId === id).length, sampledDevices: devices ? sampled.length : null,
|
|
184
|
+
highestDeviceMedianMs: sampled.length ? Math.max(...sampled.map(device => device.medianMs)) : null,
|
|
185
|
+
meaning: 'per-device window medians; highest is a device ranking, not a host or fleet percentile; no per-device observation time' } };
|
|
186
|
+
});
|
|
187
|
+
const alerts = alertReport(analytics, now, range, options.hostId);
|
|
188
|
+
const sources = Object.fromEntries(Object.entries({ fleet, inventory, shared, jenkins, operations,
|
|
189
|
+
analytics: { ...analytics, state: alerts.state, reason: alerts.reason }, inputLag: { ...lag, state: inputLag.state, reason: inputLag.reason } }).map(([key, source]) => [key, {
|
|
190
|
+
state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt ?? source.data.observedAt),
|
|
191
|
+
responseAgeSeconds: stamp(source.data.generatedAt ?? source.data.observedAt) ? (now - time(source.data.generatedAt ?? source.data.observedAt)) / 1000 : null,
|
|
192
|
+
meaning: 'response/collector generation time; does not renew component observations',
|
|
193
|
+
}]));
|
|
194
|
+
const cpu = hosts.filter(host => host.cpu.state === 'fresh' && host.cpu.percent !== null);
|
|
195
|
+
return { schemaVersion: 1, generatedAt: new Date(now).toISOString(), readOnly: true, scope: options.hostId ?? 'whole fleet', range,
|
|
196
|
+
durationSeconds: (performance.now() - started) / 1000, requestBudgetSeconds: requestBudgetMs / 1000, sources,
|
|
197
|
+
summary: { hosts: hosts.length, canonicalHosts: inventory.state === 'available' ? canonical.size : null, hostedHosts: live.size,
|
|
198
|
+
diagnosisStates: hosts.reduce((counts, host) => { counts[host.diagnosisDetails.status] = (counts[host.diagnosisDetails.status] ?? 0) + 1; return counts; }, {}),
|
|
199
|
+
cpu: { meanBusyPercent: cpu.length ? cpu.reduce((sum, host) => sum + host.cpu.percent, 0) / cpu.length : null, freshHosts: cpu.length, totalHosts: hosts.length,
|
|
200
|
+
meaning: 'arithmetic mean across fresh host busy percentages; not core weighted; inventory aliases can share physical machines' } },
|
|
201
|
+
coverage: { sourcesAvailable: Object.values(sources).filter(source => source.state === 'available').length, sourcesTotal: 7,
|
|
202
|
+
invalidHostedRecords: rows(fleet.data.hosts).length - live.size, invalidCanonicalRecords: rows(inventory.data.hosts).length - canonical.size,
|
|
203
|
+
invalidSupplementalRecords: components.size - supplemental.size,
|
|
204
|
+
sharedSnapshots: rows(shared.data.snapshots).map(entry => {
|
|
205
|
+
const snapshot = row(entry.snapshot);
|
|
206
|
+
return { publisherId: safeFleetText(entry.publisherId), hosts: rows(snapshot.hosts).length,
|
|
207
|
+
...observation(snapshot.observedAt, now, num(snapshot.staleAfterSeconds) ?? 120, snapshot.collectorError) };
|
|
208
|
+
}),
|
|
209
|
+
localCollector: 'not queried; shared published samples only',
|
|
210
|
+
limits: 'one read per source; at most 512 devices, 800 buckets and 100 alert types per bucket; no event pagination, SSH, scans or new latency probes',
|
|
211
|
+
missingSources: Object.entries(sources).filter(([, source]) => source.state !== 'available').map(([name, source]) => ({ name, reason: source.reason })),
|
|
212
|
+
consistency: 'independent source reads, not a transaction; component observations and ages retained' }, alerts, inputLag, hosts };
|
|
213
|
+
}
|
|
214
|
+
function age(value) { return `${value.state}, age ${value.ageSeconds === null ? 'unknown' : `${Math.round(value.ageSeconds)}s`}${value.reason ? `, ${value.reason}` : ''}`; }
|
|
215
|
+
function bytes(value) { return value === null ? 'unknown' : `${(value / 1024 ** 3).toFixed(1)} GiB`; }
|
|
216
|
+
export function surveyText(report) {
|
|
217
|
+
const lines = [`Fleet survey · ${report.scope} · ${report.generatedAt} · ${report.durationSeconds.toFixed(2)}s (request budget ${report.requestBudgetSeconds}s)`,
|
|
218
|
+
`${report.summary.hosts} machine records (${report.summary.canonicalHosts ?? 'unknown'} canonical); ${report.coverage.sourcesAvailable}/7 sources available.`,
|
|
219
|
+
`Diagnosis states: ${Object.entries(report.summary.diagnosisStates).map(([state, count]) => `${state}=${count}`).join(', ')}.`,
|
|
220
|
+
`CPU mean: ${report.summary.cpu.meanBusyPercent?.toFixed(1) ?? 'unknown'}% from ${report.summary.cpu.freshHosts}/${report.summary.cpu.totalHosts} fresh host samples. ${report.summary.cpu.meaning}.`];
|
|
221
|
+
if (report.alerts.state === 'available') {
|
|
222
|
+
lines.push(`Alerts · ${report.range} aligned buckets · ${report.alerts.observedBucketCounts.errors} critical / ${report.alerts.observedBucketCounts.warnings} warning rows; retries can repeat rows. Aggregate ends ${report.alerts.generatedAt} (response age ${Math.round(report.alerts.responseAgeSeconds)}s).`);
|
|
223
|
+
for (const type of report.alerts.byType)
|
|
224
|
+
lines.push(` ${type.type}: ${type.errors} critical, ${type.warnings} warning; complete-bucket trend ${type.previous ?? 'unknown'} → ${type.recent ?? 'unknown'}${type.delta === null ? '' : ` (${type.delta >= 0 ? '+' : ''}${type.delta})`}`);
|
|
225
|
+
if (!report.alerts.byType.length)
|
|
226
|
+
lines.push(' No observed alert rows in returned buckets; ingestion completeness is unknown.');
|
|
227
|
+
if (report.alerts.trend)
|
|
228
|
+
lines.push(` Trend windows: ${report.alerts.trend.previous.since}..${report.alerts.trend.previous.until} → ${report.alerts.trend.recent.since}..${report.alerts.trend.recent.until}; current partial bucket excluded.`);
|
|
229
|
+
for (const period of ['hour', 'today', 'yesterday']) {
|
|
230
|
+
const totals = report.alerts.totals[period], windows = report.alerts.windows;
|
|
231
|
+
lines.push(` ${period}: ${totals ? `${totals.errors} critical, ${totals.warnings} warning, ${totals.sessions} sessions` : 'unknown'}; ${windows[period] ?? 'unknown'}..${period === 'yesterday' ? windows.yesterdayEnd ?? 'unknown' : windows.until} (${windows.timeZone ?? 'timezone unknown'}).`);
|
|
232
|
+
}
|
|
233
|
+
lines.push(` Customer-unattributed alert rows: ${report.alerts.coverage.unattributedAlertRows ?? 'unknown'} (API query window may be wider). Ingestion completeness unknown; counts are not incident counts or failure rates.`);
|
|
234
|
+
if (report.alerts.coverage.omittedBuckets || report.alerts.coverage.omittedTypeRows)
|
|
235
|
+
lines.push(` Capped coverage: ${report.alerts.coverage.omittedBuckets} buckets / ${report.alerts.coverage.omittedTypeRows} type rows omitted.`);
|
|
236
|
+
}
|
|
237
|
+
else
|
|
238
|
+
lines.push(`Alerts: unavailable (${report.alerts.reason}); counts and trends unknown.`);
|
|
239
|
+
const lag = report.inputLag.latestSampledBucket, coverage = report.inputLag.coverage;
|
|
240
|
+
lines.push(`Input lag: ${lag ? `${lag.medianMs.toFixed(1)}ms median, ${lag.samples} samples in bucket ${lag.timestamp} (bucket age ${Math.round(lag.bucketAgeSeconds)}s; individual sample age unknown)` : `no sampled bucket (${report.inputLag.reason ?? 'no changed samples'})`}.`, ` Device coverage ${coverage.sampledDevices ?? 'unknown'} sampled / ${coverage.requestedDevices} requested / ${coverage.eligibleDevices} eligible; ${coverage.omittedDevices} omitted (limit ${coverage.deviceLimit}).`, ` Outcomes: ${report.inputLag.outcomes ? Object.entries(report.inputLag.outcomes).map(([key, count]) => `${key}=${count ?? 'unknown'}`).join(', ') : 'unknown'}. Missing outcomes are not successful latency samples.`);
|
|
241
|
+
for (const host of report.hosts) {
|
|
242
|
+
lines.push('', `${host.id} · ${host.role} · CPU ${host.cpu.percent?.toFixed(1) ?? 'unknown'}% [${age(host.cpu)}] · RAM ${bytes(host.memory.usedBytes)}/${bytes(host.memory.totalBytes)} [${age(host.memory)}] · Storage ${bytes(host.storage.usedBytes)}/${bytes(host.storage.totalBytes)} [${age(host.storage)}]`, ` ${host.diagnosis}`, ` Service ${age(host.health)} at ${host.health.observedAt ?? 'missing'}; details ${age(host.serviceChecks)}; process ${age(host.processChecks)}; admission ${age(host.admission.observation)}.`, ` Release ${host.release.actualVersion ?? host.release.version ?? 'unknown'} / expected ${host.release.expectedVersion ?? 'unknown'} at ${host.release.observedAt ?? 'missing'}; input lag highest device median ${host.inputLag.highestDeviceMedianMs?.toFixed(1) ?? 'unknown'}ms (${host.inputLag.sampledDevices ?? 'unknown'}/${host.inputLag.selectedDevices} selected devices sampled; window median, individual age unknown).`);
|
|
243
|
+
if (host.admission.operation)
|
|
244
|
+
lines.push(` Operation ${host.admission.operation.operationId ?? 'unknown'} · owner ${host.admission.operation.owner ?? 'unknown'} · ${host.admission.operation.phase ?? 'unknown'} / ${host.admission.operation.status ?? 'unknown'} [${age(host.admission.operation)}].`);
|
|
245
|
+
}
|
|
246
|
+
for (const source of report.coverage.missingSources)
|
|
247
|
+
lines.push(`Missing source: ${source.name} · ${source.reason}.`);
|
|
248
|
+
for (const [name, source] of Object.entries(report.sources))
|
|
249
|
+
lines.push(`Source ${name}: ${source.state}, generated ${source.generatedAt ?? 'unknown'}, response age ${source.responseAgeSeconds === null ? 'unknown' : `${Math.round(source.responseAgeSeconds)}s`}${source.reason ? `, ${source.reason}` : ''}.`);
|
|
250
|
+
lines.push(`Coverage: ${report.coverage.localCollector}. ${report.coverage.consistency}. ${report.coverage.limits}.`);
|
|
251
|
+
return lines.join('\n');
|
|
252
|
+
}
|
|
@@ -1,48 +1,12 @@
|
|
|
1
|
+
import { row, rows, str, num, stamp, time, pick, observation, componentHosts, hostReport, safeFleetText, validFleetId } from './run-cloud-fleet-observations.js';
|
|
2
|
+
export { safeFleetText } from './run-cloud-fleet-observations.js';
|
|
1
3
|
import { readFileSync, statSync } from 'node:fs';
|
|
2
4
|
import { ApiClient, ApiError } from '../api.js';
|
|
3
5
|
import { requireCredentials } from '../config.js';
|
|
6
|
+
import { surveyFleet, surveyText } from './run-cloud-fleet-survey.js';
|
|
4
7
|
const ROOT = '/run-cloud/admin/simulators';
|
|
5
8
|
const RANGES = ['1h', '6h', '24h', '7d', '30d', '90d', '1y', '2y'];
|
|
6
9
|
export const ACCESSIBILITY_CODES = ['accessibility_device_unavailable', 'accessibility_capture_failed'];
|
|
7
|
-
const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
|
|
8
|
-
const rows = (value) => Array.isArray(value) ? value.map(row) : [];
|
|
9
|
-
const str = (value) => typeof value === 'string' && value.length > 0 ? value : null;
|
|
10
|
-
const num = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
|
|
11
|
-
const stamp = (value) => str(value) && Number.isFinite(Date.parse(String(value))) ? new Date(String(value)).toISOString() : null;
|
|
12
|
-
const time = (value) => stamp(value) ? Date.parse(String(value)) : 0;
|
|
13
|
-
export function safeFleetText(value) {
|
|
14
|
-
const text = str(value);
|
|
15
|
-
return text ? text.slice(0, 8000)
|
|
16
|
-
.replace(/\bBearer\s+[^\s"'<>]+/gi, 'Bearer [redacted]')
|
|
17
|
-
.replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
|
|
18
|
-
.replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
|
|
19
|
-
.replace(/\b([a-zA-Z0-9_]*(?:token|password|secret|api[_-]?key)|authorization)\s*[=:]\s*[^\s,;]+/gi, '$1=[redacted]')
|
|
20
|
-
.replace(/(?:https?|wss?):\/\/[^\s<>"']+/gi, '[url redacted]') : null;
|
|
21
|
-
}
|
|
22
|
-
function observation(value, now, ttl, error) {
|
|
23
|
-
const observedAt = stamp(value);
|
|
24
|
-
const ageSeconds = observedAt ? Math.round((now - Date.parse(observedAt)) / 1000) : null;
|
|
25
|
-
if (error)
|
|
26
|
-
return { observedAt, ageSeconds, state: 'unavailable', reason: 'collection_failed; retained evidence only' };
|
|
27
|
-
if (!observedAt)
|
|
28
|
-
return { observedAt, ageSeconds, state: 'missing', reason: 'observation_timestamp_missing' };
|
|
29
|
-
if (ageSeconds < -5)
|
|
30
|
-
return { observedAt, ageSeconds, state: 'unavailable', reason: 'observation_timestamp_in_future' };
|
|
31
|
-
return { observedAt, ageSeconds, state: ageSeconds > ttl ? 'stale' : 'fresh', reason: ageSeconds > ttl ? 'observation_expired' : null };
|
|
32
|
-
}
|
|
33
|
-
function hostedObservation(host, now, ttl) {
|
|
34
|
-
const heartbeat = observation(host.observedAt, now, ttl);
|
|
35
|
-
const freshness = row(host.freshness);
|
|
36
|
-
if (!Object.keys(freshness).length)
|
|
37
|
-
return { ...heartbeat, ingestion: null, reportedFreshness: null };
|
|
38
|
-
const ingestion = observation(freshness.lastSuccessfulUpdateAt, now, ttl);
|
|
39
|
-
const reportedFreshness = str(freshness.status);
|
|
40
|
-
if (heartbeat.state === 'fresh' && (reportedFreshness !== 'fresh' || ingestion.state !== 'fresh')) {
|
|
41
|
-
const state = ingestion.state === 'unavailable' ? 'unavailable' : reportedFreshness === 'never' ? 'missing' : 'stale';
|
|
42
|
-
return { ...heartbeat, state: state, reason: `hosted_ingestion_${ingestion.state === 'fresh' ? reportedFreshness ?? 'unknown' : ingestion.state}`, ingestion, reportedFreshness };
|
|
43
|
-
}
|
|
44
|
-
return { ...heartbeat, ingestion, reportedFreshness };
|
|
45
|
-
}
|
|
46
10
|
function bound(value, fallback, max, name) {
|
|
47
11
|
const result = value ?? fallback;
|
|
48
12
|
if (!Number.isSafeInteger(result) || result < 1 || result > max)
|
|
@@ -52,7 +16,7 @@ function bound(value, fallback, max, name) {
|
|
|
52
16
|
function validate(options) {
|
|
53
17
|
if (!RANGES.includes(options.range ?? '24h'))
|
|
54
18
|
throw new Error(`range must be ${RANGES.join(', ')}`);
|
|
55
|
-
if (options.hostId &&
|
|
19
|
+
if (options.hostId && !validFleetId(options.hostId))
|
|
56
20
|
throw new Error('Invalid host ID');
|
|
57
21
|
if (options.deep && !options.hostId)
|
|
58
22
|
throw new Error('--deep requires a host ID');
|
|
@@ -107,101 +71,6 @@ export async function readFleetEvents(client, options = {}) {
|
|
|
107
71
|
reason: reason ?? (cursor ? 'page_budget_reached; matching failures may be outside returned pages' : null), generatedAt,
|
|
108
72
|
oldestReturnedAt: stamp(items.at(-1)?.timestamp), newestReturnedAt: stamp(items[0]?.timestamp) } };
|
|
109
73
|
}
|
|
110
|
-
function componentHosts(shared) {
|
|
111
|
-
const result = new Map();
|
|
112
|
-
const snapshots = rows(shared.snapshots).map(entry => row(entry.snapshot));
|
|
113
|
-
for (const snapshot of snapshots)
|
|
114
|
-
for (const host of rows(snapshot.hosts)) {
|
|
115
|
-
const id = str(host.id);
|
|
116
|
-
if (!id)
|
|
117
|
-
continue;
|
|
118
|
-
const before = result.get(id) ?? {};
|
|
119
|
-
const newest = !result.has(id) || time(host.observedAt) > time(before.observedAt) ? host : before;
|
|
120
|
-
const merged = { ...before, ...newest };
|
|
121
|
-
for (const key of ['hostTelemetry', 'hostDiagnostics', 'processScan']) {
|
|
122
|
-
const candidate = row(host[key]), previous = row(before[key]);
|
|
123
|
-
const observationKey = key === 'processScan' ? 'attemptedAt' : 'observedAt';
|
|
124
|
-
merged[key] = time(candidate[observationKey] ?? candidate.observedAt) > time(previous[observationKey] ?? previous.observedAt) ? candidate : previous;
|
|
125
|
-
}
|
|
126
|
-
result.set(id, merged);
|
|
127
|
-
}
|
|
128
|
-
return result;
|
|
129
|
-
}
|
|
130
|
-
function cpuReading(host, extra, now, fleetTtl) {
|
|
131
|
-
const telemetry = row(extra.hostTelemetry), cpu = row(telemetry.cpu);
|
|
132
|
-
const cpuValue = (value) => num(value) !== null && Number(value) <= 100 ? Number(value) : null;
|
|
133
|
-
const candidates = [
|
|
134
|
-
{ percent: cpuValue(cpu.busyPercent), source: 'operator_resource_probe', ...observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError) },
|
|
135
|
-
{ percent: cpuValue(host.cpuUtilizationPercent), source: 'host_heartbeat', ...hostedObservation(host, now, fleetTtl) },
|
|
136
|
-
].map(entry => entry.percent === null && entry.state === 'fresh' ? { ...entry, state: 'missing', reason: 'cpu_value_missing' } : entry);
|
|
137
|
-
candidates.sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
|
|
138
|
-
return { ...candidates[0], candidates, meaning: 'host busy percent, 0..100; fresh samples only in fleet mean' };
|
|
139
|
-
}
|
|
140
|
-
const pick = (source, keys) => Object.fromEntries(keys.filter(key => source[key] !== undefined).map(key => [key, source[key]]));
|
|
141
|
-
function fence(value) {
|
|
142
|
-
const source = row(value);
|
|
143
|
-
return Object.keys(source).length ? { ...pick(source, ['id', 'kind', 'state', 'createdAt', 'expiresAt', 'component']), reason: safeFleetText(source.reason) } : null;
|
|
144
|
-
}
|
|
145
|
-
function hostReport(host, canonical, extra, diagnoses, now, fleetTtl, deep = {}) {
|
|
146
|
-
const id = String(host.id ?? canonical.id ?? extra.id);
|
|
147
|
-
const health = hostedObservation(host, now, fleetTtl);
|
|
148
|
-
const diagnostic = row(extra.hostDiagnostics), process = row(extra.processScan);
|
|
149
|
-
const supplementalChecks = observation(diagnostic.observedAt, now, num(diagnostic.staleAfterSeconds) ?? 900, diagnostic.collectionError);
|
|
150
|
-
const deepChecks = observation(row(deep.security).checkedAt, now, fleetTtl);
|
|
151
|
-
const useDeep = deepChecks.state === 'fresh' && Array.isArray(deep.services) && deep.services.length > 0 && time(deepChecks.observedAt) > time(supplementalChecks.observedAt);
|
|
152
|
-
const checks = useDeep ? deepChecks : supplementalChecks;
|
|
153
|
-
const processCheck = observation(process.observedAt, now, num(process.staleAfterSeconds) ?? 900, process.error);
|
|
154
|
-
const release = row(host.release);
|
|
155
|
-
const services = rows(useDeep ? deep.services : diagnostic.services).map(service => ({ name: safeFleetText(service.name), state: safeFleetText(service.state), lastExitCode: service.lastExitCode ?? null,
|
|
156
|
-
error: safeFleetText(service.lastError ?? service.error) }));
|
|
157
|
-
const faults = checks.state === 'fresh' ? services.filter(service => ['stopped', 'failed', 'crashed', 'inactive', 'missing', 'exited', 'disabled'].includes(service.state ?? '')) : [];
|
|
158
|
-
const findings = rows(process.findings).map(finding => ({ ...pick(finding, ['fingerprint', 'reason', 'confidence', 'firstObservedAt', 'lastObservedAt', 'ownerUid', 'ageSeconds']), message: safeFleetText(finding.message ?? finding.summary) }));
|
|
159
|
-
const diagnosesForHost = diagnoses.filter(diagnosis => diagnosis.hostId === id).map(diagnosis => ({ ...pick(diagnosis, ['code', 'severity', 'deviceId']),
|
|
160
|
-
title: safeFleetText(diagnosis.title), message: safeFleetText(diagnosis.message), reportedAt: stamp(diagnosis.observedAt) }));
|
|
161
|
-
const requestedFence = fence(canonical.requestedFence ?? host.requestedFence), appliedFence = fence(canonical.fence ?? host.fence);
|
|
162
|
-
const state = health.state === 'fresh' ? str(host.state) ?? 'unknown' : 'unknown';
|
|
163
|
-
const sentences = [`${id}: ${state}${health.state === 'fresh' ? '' : ` (${health.state} heartbeat)`}.`];
|
|
164
|
-
if (appliedFence || requestedFence)
|
|
165
|
-
sentences.push(`Hold ${appliedFence ? 'reported applied' : 'requested'}; admission evidence is separate.`);
|
|
166
|
-
if (health.state === 'fresh' && release.matches === false)
|
|
167
|
-
sentences.push('Release drift reported.');
|
|
168
|
-
if (faults.length)
|
|
169
|
-
sentences.push(`Service faults: ${faults.map(service => `${service.name} ${service.state}`).join(', ')}.`);
|
|
170
|
-
if (checks.state !== 'fresh')
|
|
171
|
-
sentences.push(`Service checks ${checks.state}; current service health unproved.`);
|
|
172
|
-
if (processCheck.state !== 'fresh')
|
|
173
|
-
sentences.push(`Process checks ${processCheck.state}.`);
|
|
174
|
-
if (process.status === 'critical' || rows(process.iocs).some(ioc => ioc.status === 'present') || rows(process.findings).some(finding => finding.reason === 'known_ioc'))
|
|
175
|
-
sentences.push('Known malicious scanner evidence retained; inspect signals and scan age.');
|
|
176
|
-
else if (rows(process.findings).some(finding => ['rooted_workload', 'long_lived_workload', 'identity_changed', 'unreviewed_persistence'].includes(String(finding.reason))))
|
|
177
|
-
sentences.push('Process identities require review; inspect findings and scan age.');
|
|
178
|
-
if (rows(process.iocs).some(ioc => ioc.contained === true))
|
|
179
|
-
sentences.push('Contained IOC evidence retained.');
|
|
180
|
-
const cpu = cpuReading(host, extra, now, fleetTtl);
|
|
181
|
-
sentences.push(cpu.state === 'fresh' ? `CPU ${cpu.percent.toFixed(1)}%.` : `CPU ${cpu.state}.`);
|
|
182
|
-
return { id, name: safeFleetText(canonical.name ?? host.name ?? extra.name), platform: canonical.platform ?? host.platform ?? extra.platform ?? null,
|
|
183
|
-
role: canonical.machineType ?? extra.role ?? null, inventoryPresent: Object.keys(canonical).length > 0, livePresent: Object.keys(host).length > 0,
|
|
184
|
-
health: { reportedState: str(host.state), effectiveState: state, ...health }, diagnosis: sentences.join(' '), cpu,
|
|
185
|
-
release: { ...pick(release, ['expectedPackage', 'actualPackage', 'expectedVersion', 'actualVersion', 'expectedBuild', 'actualBuild', 'matches']), version: str(host.version), build: str(host.build), observedAt: stamp(host.observedAt) },
|
|
186
|
-
admission: { enabled: canonical.enabled ?? null, acceptsLeases: canonical.acceptsLeases ?? null, observedAt: stamp(canonical.admissionObservedAt), reason: safeFleetText(canonical.admissionReason), requestedFence, fence: appliedFence },
|
|
187
|
-
capacity: pick(row(host.capacity), ['clean', 'leased', 'installing', 'dirty', 'preparing', 'recycling', 'total']),
|
|
188
|
-
serviceChecks: { ...checks, source: useDeep ? 'hosted_deep_evidence' : 'operator_diagnostics', services,
|
|
189
|
-
software: { ...pick(row(diagnostic.software), ['serveSimPackage', 'serveSimVersion', 'serveSimRevision', 'xcodeVersion']), observation: supplementalChecks, packageMatchesService: diagnostic.packageMatchesService ?? null } },
|
|
190
|
-
processChecks: { ...processCheck, ...pick(process, ['attemptedAt', 'inventoryObservedAt', 'maxVariableAgeSeconds', 'root', 'mode', 'coverage', 'counts']),
|
|
191
|
-
status: process.status ?? null, baseline: pick(row(process.baseline), ['state', 'profileIds', 'methods']),
|
|
192
|
-
transportTrust: pick(row(process.transportTrust), ['state', 'verifiedAt', 'pinsSha256']), findings,
|
|
193
|
-
iocs: rows(process.iocs).map(ioc => ({ ...pick(ioc, ['family', 'status', 'contained', 'lastObservedAt']), signals: Array.isArray(ioc.signals) ? ioc.signals.map(safeFleetText) : [] })) },
|
|
194
|
-
resources: { ...observation(row(extra.hostTelemetry).observedAt, now, 45, row(extra.hostTelemetry).collectionError),
|
|
195
|
-
memory: pick(row(row(extra.hostTelemetry).memory), ['totalBytes', 'usedBytes', 'availableBytes', 'swapUsedBytes']),
|
|
196
|
-
perDeviceCpu: rows(row(row(extra.hostTelemetry).cpu).perDevice).map(device => pick(device, ['id', 'cpuPercent', 'rssBytes', 'processes'])),
|
|
197
|
-
processCpuMeaning: 'OS process percent may exceed 100 on multicore hosts; independent from host busy percent' },
|
|
198
|
-
devices: rows(host.devices).map(device => ({ id: str(device.id), ...pick(device, ['state', 'sessionId', 'leaseId', 'updatedAt']), helper: {
|
|
199
|
-
...pick(row(device.helper), ['state', 'pid', 'lastSeenAt']), observation: observation(row(device.helper).lastSeenAt, now, fleetTtl),
|
|
200
|
-
observationMeaning: 'last helper evidence; freshness uses fleet heartbeat threshold', error: safeFleetText(row(device.helper).error)
|
|
201
|
-
}, lastError: safeFleetText(device.lastError) })),
|
|
202
|
-
diagnoses: diagnosesForHost,
|
|
203
|
-
};
|
|
204
|
-
}
|
|
205
74
|
function originalLogsUrl(value) {
|
|
206
75
|
try {
|
|
207
76
|
const url = new URL(String(value));
|
|
@@ -300,20 +169,25 @@ export async function inspectFleet(client, options = {}, now) {
|
|
|
300
169
|
const fleet = await readSource(client, ROOT);
|
|
301
170
|
if (fleet.state === 'unavailable' || !Array.isArray(fleet.data.hosts))
|
|
302
171
|
throw new Error(`Fleet read unavailable (${fleet.reason ?? 'invalid_response'}); use an admin identity with read permission`);
|
|
303
|
-
const [inventory, shared, diagnostics, events, history] = await Promise.all([
|
|
172
|
+
const [inventory, shared, diagnostics, events, history, jenkins] = await Promise.all([
|
|
304
173
|
readSource(client, '/run-cloud/admin/fleet-inventory'),
|
|
305
174
|
readSource(client, `${ROOT}/operator-inventory`),
|
|
306
175
|
readSource(client, `${ROOT}/diagnostics?${query}`),
|
|
307
176
|
readFleetEvents(client, options),
|
|
308
177
|
options.hostId ? readSource(client, `${ROOT}/history?${new URLSearchParams({ range: options.range ?? '24h', hostId: options.hostId })}`) : Promise.resolve(null),
|
|
178
|
+
readSource(client, '/run-cloud/admin/jenkins'),
|
|
309
179
|
]);
|
|
310
|
-
for (const [source, key] of [[inventory, 'hosts'], [shared, 'snapshots'], [diagnostics, 'diagnoses']]) {
|
|
180
|
+
for (const [source, key] of [[inventory, 'hosts'], [shared, 'snapshots'], [diagnostics, 'diagnoses'], [jenkins, 'nodes']]) {
|
|
311
181
|
if (source.state === 'available' && !Array.isArray(source.data[key])) {
|
|
312
182
|
source.state = 'unavailable';
|
|
313
183
|
source.reason = 'invalid_response_shape';
|
|
314
184
|
source.data = {};
|
|
315
185
|
}
|
|
316
186
|
}
|
|
187
|
+
if (jenkins.data.state === 'unavailable') {
|
|
188
|
+
jenkins.state = 'unavailable';
|
|
189
|
+
jenkins.reason = 'jenkins_collection_unavailable';
|
|
190
|
+
}
|
|
317
191
|
if (history?.state === 'available' && !Array.isArray(history.data.points)) {
|
|
318
192
|
history.state = 'unavailable';
|
|
319
193
|
history.reason = 'invalid_response_shape';
|
|
@@ -326,7 +200,9 @@ export async function inspectFleet(client, options = {}, now) {
|
|
|
326
200
|
const ids = new Set([...expected.keys(), ...live.keys(), ...supplemental.keys()]);
|
|
327
201
|
if (options.hostId && !ids.has(options.hostId))
|
|
328
202
|
throw new Error(`Host ${options.hostId} is absent from returned inventories`);
|
|
329
|
-
const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => hostReport(live.get(id) ?? {}, expected.get(id) ?? {}, supplemental.get(id) ?? {}, rows(diagnostics.data.diagnoses), now, num(fleet.data.staleAfterSeconds) ?? 45, options.deep ? row(row(diagnostics.data.hostDetails)[id]) : {}
|
|
203
|
+
const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => hostReport(live.get(id) ?? {}, expected.get(id) ?? {}, supplemental.get(id) ?? {}, rows(diagnostics.data.diagnoses), now, num(fleet.data.staleAfterSeconds) ?? 45, options.deep ? row(row(diagnostics.data.hostDetails)[id]) : {}, {
|
|
204
|
+
inventoryTtl: num(inventory.data.staleAfterSeconds) ?? 30, jenkins: jenkins.data,
|
|
205
|
+
}));
|
|
330
206
|
const freshCpu = hosts.filter(host => host.cpu.state === 'fresh' && host.cpu.percent !== null);
|
|
331
207
|
const codes = options.codes ?? [];
|
|
332
208
|
const failureInputs = [...events.items.map(eventEvidence), ...(options.failures ?? []).map(failureEvidence)];
|
|
@@ -334,9 +210,9 @@ export async function inspectFleet(client, options = {}, now) {
|
|
|
334
210
|
const failures = failureInputs.filter(event => selectedFailure(event, codes) && (!event.timestamp || (time(event.timestamp) >= start && time(event.timestamp) <= now)))
|
|
335
211
|
.map(event => ({ ...correlate(event, rows(fleet.data.hosts), hosts, now), windowMembership: event.timestamp ? 'inside_requested_window' : 'unknown; original timestamp missing' }))
|
|
336
212
|
.filter(event => !options.hostId || event.hostId === options.hostId || event.hostId === null);
|
|
337
|
-
const sourceStatus = (source) => ({ state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt), meaning: 'response generation time; does not renew component observations' });
|
|
213
|
+
const sourceStatus = (source) => ({ state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt ?? source.data.observedAt), meaning: 'response generation time; does not renew component observations' });
|
|
338
214
|
return { schemaVersion: 1, generatedAt: new Date(now).toISOString(), readOnly: true,
|
|
339
|
-
sources: { fleet: sourceStatus(fleet), inventory: sourceStatus(inventory), shared: sourceStatus(shared), diagnostics: sourceStatus(diagnostics),
|
|
215
|
+
sources: { fleet: sourceStatus(fleet), inventory: sourceStatus(inventory), shared: sourceStatus(shared), diagnostics: sourceStatus(diagnostics), jenkins: sourceStatus(jenkins),
|
|
340
216
|
...(history ? { history: sourceStatus(history) } : {}) },
|
|
341
217
|
summary: { hosts: hosts.length, canonicalInventoryHosts: inventory.state === 'available' ? expected.size : null, liveHosts: live.size,
|
|
342
218
|
healthCounts: hosts.reduce((counts, host) => { const state = host.health.effectiveState; counts[state] = (counts[state] ?? 0) + 1; return counts; }, { ready: 0, degraded: 0, unreachable: 0, unknown: 0 }),
|
|
@@ -421,6 +297,14 @@ function printReport(report, json) {
|
|
|
421
297
|
}
|
|
422
298
|
export function registerRunCloudFleet(program) {
|
|
423
299
|
const fleet = program.command('fleet').description('Read-only admin fleet telemetry, diagnosis, events and original failure evidence');
|
|
300
|
+
fleet.command('survey [hostId]').description('Survey all fleet signals in seconds; authenticated admin reads, no local services')
|
|
301
|
+
.option('--json', 'Structured metrics, diagnosis, observations and coverage')
|
|
302
|
+
.option('--range <range>', 'Alert/input-lag window: 1h, 6h, 24h, 7d', '24h')
|
|
303
|
+
.action((hostId, opts) => action(async () => {
|
|
304
|
+
const credentials = requireCredentials(), apiOrigin = new URL(credentials.apiUrl).origin;
|
|
305
|
+
const report = await surveyFleet(new ApiClient(credentials.apiUrl, credentials.token), { hostId, range: opts.range });
|
|
306
|
+
console.log(opts.json ? JSON.stringify({ ...report, apiOrigin }, null, 2) : `${apiOrigin}\n${surveyText(report)}`);
|
|
307
|
+
}));
|
|
424
308
|
fleet.command('inspect [hostId]').description('Inspect whole fleet or one machine; existing admin read authorization required')
|
|
425
309
|
.option('--json', 'Structured evidence with source observations and missing data')
|
|
426
310
|
.option('--range <range>', 'Event/history window', '24h').option('--limit <rows>', 'Events per page (1..500)', '200')
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
export function processFreshness(scan, now) {
|
|
2
|
+
if (!scan?.observedAt)
|
|
3
|
+
return 'never observed';
|
|
4
|
+
if (scan.sourceUnavailable)
|
|
5
|
+
return 'source unavailable';
|
|
6
|
+
if (scan.error)
|
|
7
|
+
return 'collection failed';
|
|
8
|
+
const timestamp = Date.parse(scan.observedAt);
|
|
9
|
+
if (!Number.isFinite(timestamp) || timestamp > now + 5000 || now - timestamp > scan.staleAfterSeconds * 1000)
|
|
10
|
+
return 'stale';
|
|
11
|
+
return 'fresh';
|
|
12
|
+
}
|
|
13
|
+
export function leaseStateLabel(state) {
|
|
14
|
+
return { accepting: 'Lease gate open', rejecting: 'Lease gate closed', unknown: 'Lease gate unknown', held: 'Operator hold' }[state] ?? state;
|
|
15
|
+
}
|
|
16
|
+
export function operatorHeld(host) {
|
|
17
|
+
return [host.fence, host.requestedFence].some(fence => fence && ['experiment', 'maintenance'].includes(fence.kind));
|
|
18
|
+
}
|
|
19
|
+
export function hostLeaseState(host) {
|
|
20
|
+
if (operatorHeld(host))
|
|
21
|
+
return 'held';
|
|
22
|
+
return host.acceptsLeases === true ? 'accepting' : host.acceptsLeases === false ? 'rejecting' : 'unknown';
|
|
23
|
+
}
|
|
24
|
+
export function hostLeaseStatus(host) {
|
|
25
|
+
const gate = leaseStateLabel(host.acceptsLeases === true ? 'accepting' : host.acceptsLeases === false ? 'rejecting' : 'unknown');
|
|
26
|
+
return operatorHeld(host) ? `Operator hold · ${gate.toLowerCase()}` : gate;
|
|
27
|
+
}
|
|
28
|
+
export function redactSensitiveText(value) {
|
|
29
|
+
if (!value)
|
|
30
|
+
return value ?? '';
|
|
31
|
+
return value
|
|
32
|
+
.replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
|
|
33
|
+
.replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
|
|
34
|
+
.replace(/(\bBearer\s+)[A-Za-z0-9._~+/=-]+/gi, '$1[redacted]')
|
|
35
|
+
.replace(/([?&](?:token|authorization|secret|credential|signature|api[_-]?key|x-amz-(?:credential|signature)|x-goog-(?:credential|signature))=)[^\s&#]+/gi, '$1[redacted]')
|
|
36
|
+
.replace(/((?:token|authorization|secret|credential|signature|api[_-]?key)\s*[:=]\s*)[^\s,;}]+/gi, '$1[redacted]');
|
|
37
|
+
}
|
|
38
|
+
const ready = (state) => ['clean', 'ready', 'healthy', 'serving', 'connected'].includes(state);
|
|
39
|
+
const responding = (state) => ready(state) || state === 'busy';
|
|
40
|
+
export const observationCurrent = (at, now, seconds) => {
|
|
41
|
+
const time = at ? Date.parse(at) : NaN;
|
|
42
|
+
return Number.isFinite(time) && time <= now + 5000 && now - time <= seconds * 1000;
|
|
43
|
+
};
|
|
44
|
+
export function jenkinsObservedState(node, at, now, seconds = 30, unavailable = false) {
|
|
45
|
+
if (unavailable || !node || !at || !Number.isFinite(Date.parse(at)) || Date.parse(at) > now + 5000)
|
|
46
|
+
return 'unknown';
|
|
47
|
+
if (!observationCurrent(at, now, seconds))
|
|
48
|
+
return 'stale';
|
|
49
|
+
return node.offline === true ? 'offline' : node.offline === false ? 'ready' : 'unknown';
|
|
50
|
+
}
|
|
51
|
+
export function machineDiagnosis(host, hosted, observedState, now, staleAfterSeconds = 120) {
|
|
52
|
+
const faults = [], findings = [], checks = [];
|
|
53
|
+
let status = ready(observedState) ? 'ready' : observedState;
|
|
54
|
+
const observed = observationCurrent(host.observedAt, now, staleAfterSeconds);
|
|
55
|
+
const serviceCurrent = !['stale', 'unknown', 'inspection-unavailable'].includes(observedState) && observed;
|
|
56
|
+
if (host.observedAt && !observed && status !== 'unknown')
|
|
57
|
+
status = 'stale';
|
|
58
|
+
if (!host.observedAt) {
|
|
59
|
+
status = 'unknown';
|
|
60
|
+
checks.push('service never observed');
|
|
61
|
+
}
|
|
62
|
+
else if (!serviceCurrent)
|
|
63
|
+
checks.push('service checks stale or unavailable; last state retained');
|
|
64
|
+
else if (observedState === 'unreachable')
|
|
65
|
+
checks.push('service unreachable from collector; cause unconfirmed');
|
|
66
|
+
else if (['down', 'runtime-unavailable', 'agent-running-no-health', 'offline'].includes(observedState))
|
|
67
|
+
faults.push(`service reports ${observedState}`);
|
|
68
|
+
const release = hosted?.release;
|
|
69
|
+
const drift = release?.matches === false && [
|
|
70
|
+
[release.expectedPackage, release.actualPackage], [release.expectedVersion, release.actualVersion], [release.expectedBuild, release.actualBuild],
|
|
71
|
+
].some(([expected, actual]) => expected && actual && expected !== actual);
|
|
72
|
+
if (drift)
|
|
73
|
+
findings.push(`release drift: ${release.actualVersion ?? 'reported artifact'}; expected ${release.expectedVersion ?? 'configured artifact'}`);
|
|
74
|
+
else if (release?.matches !== true && host.role !== 'jenkins')
|
|
75
|
+
checks.push('release match not verified');
|
|
76
|
+
if (observedState === 'degraded' && !drift)
|
|
77
|
+
checks.push('reported degradation has no confirmed cause');
|
|
78
|
+
const sample = host.hostDiagnostics;
|
|
79
|
+
if (!sample)
|
|
80
|
+
checks.push('detailed service checks missing');
|
|
81
|
+
else if (sample.collectionError || !observationCurrent(sample.observedAt, now, sample.staleAfterSeconds))
|
|
82
|
+
checks.push('detailed service checks stale or unavailable');
|
|
83
|
+
else {
|
|
84
|
+
const service = sample.services.find(row => row.name === 'serve-sim');
|
|
85
|
+
if (host.role === 'simulator' && service && ['failed', 'stopped', 'exited', 'disabled'].includes(service.state ?? ''))
|
|
86
|
+
faults.push(`serve-sim is ${service.state}`);
|
|
87
|
+
if (host.role === 'simulator' && sample.packageMatchesService === false)
|
|
88
|
+
faults.push('service executable differs from installed package');
|
|
89
|
+
if (sample.power.thermalWarning || sample.power.performanceWarning)
|
|
90
|
+
findings.push('host reports thermal or performance warnings');
|
|
91
|
+
if ((sample.coreSimulator.orphaned ?? 0) > 0)
|
|
92
|
+
findings.push(`${sample.coreSimulator.orphaned} orphaned CoreSimulator bootstrap(s)`);
|
|
93
|
+
if ((sample.coreSimulator.externalBooted ?? 0) > 0)
|
|
94
|
+
findings.push(`${sample.coreSimulator.externalBooted} external booted simulator(s) consume memory`);
|
|
95
|
+
if ((sample.recording.failed ?? 0) > 0)
|
|
96
|
+
checks.push(`${sample.recording.failed} retained recording failure(s); current service scored separately`);
|
|
97
|
+
}
|
|
98
|
+
if (serviceCurrent && (host.capacity?.dirty ?? 0) > 0)
|
|
99
|
+
faults.push(`${host.capacity.dirty} dirty pool record(s)`);
|
|
100
|
+
const scan = host.processScan;
|
|
101
|
+
if (!scan)
|
|
102
|
+
checks.push('process checks missing');
|
|
103
|
+
else {
|
|
104
|
+
const freshness = processFreshness(scan, now);
|
|
105
|
+
const malicious = scan.iocs.some(ioc => ['present', 'active'].includes(ioc.status)) || scan.findings.some(finding => finding.reason === 'known_ioc');
|
|
106
|
+
if (malicious)
|
|
107
|
+
findings.unshift(`known malicious daemon detected${freshness === 'fresh' ? '' : ' in retained evidence; recheck required'}`);
|
|
108
|
+
else if (scan.status === 'critical')
|
|
109
|
+
findings.unshift('critical process finding requires review');
|
|
110
|
+
const review = scan.findings.filter(finding => ['rooted_workload', 'long_lived_workload', 'identity_changed', 'unreviewed_persistence'].includes(finding.reason));
|
|
111
|
+
if (review.length)
|
|
112
|
+
findings.push(`${review.length} process or persistence finding(s) need review`);
|
|
113
|
+
if (scan.iocs.some(ioc => ioc.contained))
|
|
114
|
+
findings.push('quarantined daemon evidence is contained');
|
|
115
|
+
if (freshness !== 'fresh' || scan.error)
|
|
116
|
+
checks.push(`process checks ${scan.error ?? freshness}`);
|
|
117
|
+
else if (!scan.root || scan.coverage.iocs === 'partial' || scan.iocs.some(ioc => ioc.status === 'unknown'))
|
|
118
|
+
checks.push('process checks incomplete; privileged visibility not established');
|
|
119
|
+
if (scan.baseline.state !== 'reviewed')
|
|
120
|
+
checks.push('process baseline not enrolled');
|
|
121
|
+
if (scan.status === 'warning' && !review.length && !malicious)
|
|
122
|
+
checks.push('process checks warning; see scanner details');
|
|
123
|
+
}
|
|
124
|
+
const actionable = findings.filter(finding => !finding.startsWith('quarantined'));
|
|
125
|
+
if (faults.length || actionable.length)
|
|
126
|
+
status = 'degraded';
|
|
127
|
+
const parts = [`${host.id}: ${status}.`,
|
|
128
|
+
faults.length ? `Service faults: ${faults.join('; ')}.` : serviceCurrent && (responding(observedState) || (observedState === 'degraded' && drift)) ? `Service responding${observedState === 'busy' ? '; capacity busy' : ''}.` : 'Current service health unverified.',
|
|
129
|
+
findings.length ? `Findings: ${findings.join('; ')}.` : '',
|
|
130
|
+
`Admission: ${hostLeaseStatus(host)}${host.fence ? `; ${host.fence.state ?? 'unconfirmed'} ${host.fence.kind} fence — ${host.fence.reason}` : ''}.`,
|
|
131
|
+
checks.length ? `Checks: ${checks.join('; ')}.` : 'Checks current.',
|
|
132
|
+
faults.length ? 'Next: inspect retained service diagnostics.' : actionable.length ? 'Next: review the release or scanner evidence.' : checks.length ? 'Next: confirm missing checks with read-only collection.' : '',
|
|
133
|
+
];
|
|
134
|
+
return { status, summary: redactSensitiveText(parts.filter(Boolean).join(' ')),
|
|
135
|
+
faults: faults.map(redactSensitiveText), findings: findings.map(redactSensitiveText), checks: checks.map(redactSensitiveText) };
|
|
136
|
+
}
|
package/dist/version.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export const CLI_VERSION = '0.2.
|
|
1
|
+
export const CLI_VERSION = '0.2.167';
|
package/package.json
CHANGED
|
Binary file
|
package/runcloud-0.2.166.tgz
DELETED
|
Binary file
|