runcloud 0.2.166 → 0.2.168

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -30,6 +30,7 @@ const JUDGMENT_IDS = new Set([
30
30
  'background_mode_justification',
31
31
  'push_notification_setup',
32
32
  'deep_link_config',
33
+ 'rating_prompt',
33
34
  ]);
34
35
  const LIVE_STORE_IDS = new Set([
35
36
  'apple_privacy_policy',
@@ -210,6 +211,8 @@ function evaluateLocalItem(root, appJson, item) {
210
211
  return evaluatePermissionDescriptions(root, appJson, item);
211
212
  case 'privacy_manifest':
212
213
  return evaluatePrivacyManifest(root, appJson, item);
214
+ case 'rating_prompt':
215
+ return evaluateRatingPrompt(root, item);
213
216
  default:
214
217
  if (STORE_REMINDER_IDS.has(item.id)) {
215
218
  return makeItem(item, 'info', 'Complete this declaration in Google Play Console.', 'Play Console questionnaire');
@@ -259,6 +262,12 @@ function evaluateRestorePurchases(root, item) {
259
262
  return makeItem(item, 'pass', 'Restore purchases flow appears in source.', evidence(restore));
260
263
  return makeItem(item, 'fail', 'IAP APIs found but no restore purchases flow was found.', evidence(iap));
261
264
  }
265
+ function evaluateRatingPrompt(root, item) {
266
+ const asks = scan(root, /expo-store-review|react-native-in-app-review|requestReview|SKStoreReviewController/);
267
+ if (asks.length)
268
+ return null;
269
+ return makeItem(item, 'warning', 'The app never asks people for a rating.', 'scan: no store rating prompt found');
270
+ }
262
271
  const TRACKING_SDK = /["'](?:@react-native-firebase\/analytics|react-native-fbsdk(?:-next)?|expo-facebook|react-native-appsflyer|@amplitude\/[\w.-]+|@segment\/[\w.-]+)(?:\/[^"']*)?["']/;
263
272
  function evaluateTrackingWithoutAtt(root, item) {
264
273
  const tracking = scan(root, TRACKING_SDK);
@@ -0,0 +1,194 @@
1
+ import { jenkinsObservedState, machineDiagnosis } from './run-cloud-machine-diagnosis.js';
2
+ export const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
3
+ export const rows = (value) => Array.isArray(value) ? value.map(row) : [];
4
+ export const str = (value) => typeof value === 'string' && value.length > 0 ? value : null;
5
+ export const num = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
6
+ export const stamp = (value) => str(value) && Number.isFinite(Date.parse(String(value))) ? new Date(String(value)).toISOString() : null;
7
+ export const time = (value) => stamp(value) ? Date.parse(String(value)) : 0;
8
+ export const validFleetId = (value) => typeof value === 'string' && /^[a-zA-Z0-9][a-zA-Z0-9._:-]{0,255}$/.test(value);
9
+ export function safeFleetText(value) {
10
+ const text = str(value);
11
+ return text ? text.slice(0, 8000)
12
+ .replace(/\bBearer\s+[^\s"'<>]+/gi, 'Bearer [redacted]')
13
+ .replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
14
+ .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
15
+ .replace(/\b([a-zA-Z0-9_]*(?:token|password|secret|api[_-]?key)|authorization)\s*[=:]\s*[^\s,;]+/gi, '$1=[redacted]')
16
+ .replace(/(?:https?|wss?):\/\/[^\s<>"']+/gi, '[url redacted]') : null;
17
+ }
18
+ export function observation(value, now, ttl, error) {
19
+ const observedAt = stamp(value);
20
+ const ageSeconds = observedAt ? (now - Date.parse(observedAt)) / 1000 : null;
21
+ if (error)
22
+ return { observedAt, ageSeconds, state: 'unavailable', reason: 'collection_failed; retained evidence only' };
23
+ if (!observedAt)
24
+ return { observedAt, ageSeconds, state: 'missing', reason: 'observation_timestamp_missing' };
25
+ if (ageSeconds < -5)
26
+ return { observedAt, ageSeconds, state: 'unavailable', reason: 'observation_timestamp_in_future' };
27
+ return { observedAt, ageSeconds, state: ageSeconds > ttl ? 'stale' : 'fresh', reason: ageSeconds > ttl ? 'observation_expired' : null };
28
+ }
29
+ export function hostedObservation(host, now, ttl) {
30
+ const heartbeat = observation(host.observedAt, now, ttl);
31
+ const freshness = row(host.freshness);
32
+ if (!Object.keys(freshness).length)
33
+ return { ...heartbeat, ingestion: null, ingestionHeartbeat: null, reportedFreshness: null };
34
+ const ingestion = observation(freshness.lastSuccessfulUpdateAt, now, ttl);
35
+ const ingestionHeartbeat = observation(freshness.lastHeartbeatAt, now, ttl);
36
+ const reportedFreshness = str(freshness.status);
37
+ if (heartbeat.state === 'fresh' && (reportedFreshness !== 'fresh' || ingestion.state !== 'fresh' || ingestionHeartbeat.state !== 'fresh')) {
38
+ const state = [ingestion, ingestionHeartbeat].some(sample => sample.state === 'unavailable') ? 'unavailable' : reportedFreshness === 'never' ? 'missing' : 'stale';
39
+ const reason = ingestion.state !== 'fresh' ? `hosted_ingestion_${ingestion.state}` : ingestionHeartbeat.state !== 'fresh' ? `hosted_heartbeat_${ingestionHeartbeat.state}` : `hosted_ingestion_${reportedFreshness ?? 'unknown'}`;
40
+ return { ...heartbeat, state: state, reason, ingestion, ingestionHeartbeat, reportedFreshness };
41
+ }
42
+ return { ...heartbeat, ingestion, ingestionHeartbeat, reportedFreshness };
43
+ }
44
+ export function componentHosts(shared) {
45
+ const result = new Map();
46
+ const snapshots = rows(shared.snapshots).map(entry => row(entry.snapshot));
47
+ for (const snapshot of snapshots)
48
+ for (const raw of rows(snapshot.hosts)) {
49
+ const host = { ...raw, inventorySource: 'shared', collectorError: snapshot.collectorError ?? null };
50
+ const id = str(host.id);
51
+ if (!id)
52
+ continue;
53
+ const before = result.get(id) ?? {};
54
+ const newest = !result.has(id) || time(host.observedAt) >= time(before.observedAt) ? host : before;
55
+ const merged = { ...before, ...newest };
56
+ for (const key of ['hostTelemetry', 'hostDiagnostics', 'processScan']) {
57
+ const candidate = row(host[key]), previous = row(before[key]);
58
+ const observationKey = key === 'processScan' ? 'attemptedAt' : 'observedAt';
59
+ merged[key] = time(candidate[observationKey] ?? candidate.observedAt) >= time(previous[observationKey] ?? previous.observedAt) && Object.keys(candidate).length ? candidate : previous;
60
+ }
61
+ result.set(id, merged);
62
+ }
63
+ return result;
64
+ }
65
+ export function cpuReading(host, extra, now, fleetTtl) {
66
+ const telemetry = row(extra.hostTelemetry), cpu = row(telemetry.cpu);
67
+ const cpuValue = (value) => num(value) !== null && Number(value) <= 100 ? Number(value) : null;
68
+ const candidates = [
69
+ { percent: cpuValue(cpu.busyPercent), source: 'operator_resource_probe', ...observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError) },
70
+ { percent: cpuValue(host.cpuUtilizationPercent), source: 'host_heartbeat', ...hostedObservation(host, now, fleetTtl) },
71
+ ].map(entry => entry.percent === null && entry.state === 'fresh' ? { ...entry, state: 'missing', reason: 'cpu_value_missing' } : entry);
72
+ candidates.sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
73
+ return { ...candidates[0], candidates, meaning: 'host busy percent, 0..100; fresh samples only in fleet mean' };
74
+ }
75
+ export const pick = (source, keys) => Object.fromEntries(keys.filter(key => source[key] !== undefined).map(key => [key, source[key]]));
76
+ function fence(value) {
77
+ const source = row(value);
78
+ return Object.keys(source).length ? { ...pick(source, ['id', 'kind', 'state', 'createdAt', 'expiresAt', 'component', 'owner', 'operationId']), reason: safeFleetText(source.reason) } : null;
79
+ }
80
+ export function hostReport(host, canonical, extra, diagnoses, now, fleetTtl, deep = {}, context = {}) {
81
+ const id = String(host.id ?? canonical.id ?? extra.id);
82
+ const machineType = str(canonical.machineType ?? host.machineType) ?? (extra.role === 'jenkins' ? 'build' : str(extra.role)) ?? 'simulator';
83
+ const role = machineType === 'build' ? 'jenkins' : machineType;
84
+ const sharedHealth = observation(extra.observedAt, now, context.operatorTtl ?? 120, extra.collectorError);
85
+ const useShared = sharedHealth.state === 'fresh' && time(extra.observedAt) > time(host.observedAt);
86
+ const primary = useShared ? extra : host;
87
+ const capacitySource = primary.capacity != null ? primary : useShared ? host : extra;
88
+ const capacity = row(capacitySource.capacity);
89
+ let health = useShared ? sharedHealth : hostedObservation(host, now, fleetTtl);
90
+ let state = health.state === 'fresh' ? str(primary.state) ?? 'unknown' : 'unknown';
91
+ let reportedState = str(primary.state);
92
+ let diagnosisTtl = useShared ? context.operatorTtl ?? 120 : fleetTtl;
93
+ let diagnosisSource = useShared ? 'shared' : 'hosted';
94
+ if (role === 'jenkins') {
95
+ const jenkins = context.jenkins ?? {};
96
+ const node = rows(jenkins.nodes).find(node => node.id === id || node.id === `${id}-jenkins`);
97
+ reportedState = node?.offline === true ? 'offline' : node?.offline === false ? 'ready' : null;
98
+ diagnosisTtl = num(jenkins.staleAfterSeconds) ?? 30;
99
+ health = observation(jenkins.observedAt, now, diagnosisTtl, jenkins.error || jenkins.state === 'unavailable');
100
+ state = jenkinsObservedState(node ? { offline: typeof node.offline === 'boolean' ? node.offline : undefined } : undefined, health.observedAt, now, diagnosisTtl, health.state === 'unavailable');
101
+ if (!node && health.state === 'fresh')
102
+ health = { ...health, state: 'missing', reason: 'jenkins_node_missing' };
103
+ diagnosisSource = 'jenkins';
104
+ }
105
+ const diagnostic = row(extra.hostDiagnostics), process = row(extra.processScan);
106
+ const supplementalChecks = observation(diagnostic.observedAt, now, num(diagnostic.staleAfterSeconds) ?? 900, diagnostic.collectionError);
107
+ const deepChecks = observation(row(deep.security).checkedAt, now, fleetTtl);
108
+ const useDeep = deepChecks.state === 'fresh' && Array.isArray(deep.services) && deep.services.length > 0 && time(deepChecks.observedAt) > time(supplementalChecks.observedAt);
109
+ const checks = useDeep ? deepChecks : supplementalChecks;
110
+ const processCheck = observation(process.observedAt, now, num(process.staleAfterSeconds) ?? 900, process.error);
111
+ const release = row(host.release);
112
+ const services = rows(useDeep ? deep.services : diagnostic.services).map(service => ({ name: safeFleetText(service.name), state: safeFleetText(service.state), lastExitCode: service.lastExitCode ?? null,
113
+ error: safeFleetText(service.lastError ?? service.error) }));
114
+ const findings = rows(process.findings).map(finding => ({ ...pick(finding, ['fingerprint', 'reason', 'confidence', 'firstObservedAt', 'lastObservedAt', 'ownerUid', 'ageSeconds']), message: safeFleetText(finding.message ?? finding.summary) }));
115
+ const diagnosesForHost = diagnoses.filter(diagnosis => diagnosis.hostId === id).map(diagnosis => ({ ...pick(diagnosis, ['code', 'severity', 'deviceId']),
116
+ title: safeFleetText(diagnosis.title), message: safeFleetText(diagnosis.message), reportedAt: stamp(diagnosis.observedAt) }));
117
+ const admissionObservation = observation(canonical.admissionObservedAt, now, context.inventoryTtl ?? 30);
118
+ const acceptsLeases = canonical.enabled === false || machineType !== 'simulator' || canonical.admissionReason === 'routing-disabled' ? false :
119
+ admissionObservation.state === 'fresh' && typeof canonical.acceptsLeases === 'boolean' ? canonical.acceptsLeases : null;
120
+ const rawRequestedFence = Object.hasOwn(canonical, 'requestedFence') ? canonical.requestedFence : host.requestedFence ?? extra.requestedFence;
121
+ const rawFence = Object.hasOwn(canonical, 'fence') ? canonical.fence : host.fence ?? extra.fence;
122
+ const requestedFence = fence(rawRequestedFence), appliedFence = fence(rawFence);
123
+ if (appliedFence && admissionObservation.state !== 'fresh')
124
+ appliedFence.state = 'unknown';
125
+ const operation = context.operations?.filter(operation => operation.id === id).sort((a, b) => time(b.updatedAt) - time(a.updatedAt))[0];
126
+ const operationFence = operation && acceptsLeases !== true ? fence(operation.fence) : null;
127
+ if (operationFence && (operation.stale || (!['complete', 'reserved', 'cancelled', 'rolled-back'].includes(String(operation.status)) && now - time(operation.updatedAt) > 60000)))
128
+ operationFence.state = 'unknown';
129
+ const displayFence = appliedFence ?? requestedFence ?? operationFence;
130
+ const sample = useDeep ? { observedAt: deepChecks.observedAt, staleAfterSeconds: fleetTtl, services: deep.services } : diagnostic;
131
+ const diagnosisHost = {
132
+ id, role, observedAt: health.observedAt, acceptsLeases,
133
+ capacity: role === 'jenkins' ? null : { dirty: num(capacity.dirty) },
134
+ fence: displayFence ? { kind: str(displayFence.kind) ?? 'unknown', state: str(displayFence.state), reason: str(row(rawFence ?? rawRequestedFence ?? operation?.fence).reason) ?? 'reason missing' } : null,
135
+ requestedFence: requestedFence ? { kind: str(requestedFence.kind) ?? 'unknown' } : null,
136
+ hostDiagnostics: Object.keys(sample).length ? {
137
+ observedAt: stamp(sample.observedAt), staleAfterSeconds: num(sample.staleAfterSeconds) ?? 900,
138
+ collectionError: str(sample.collectionError) ?? (!Array.isArray(sample.services) ? 'service list missing' : null),
139
+ services: rows(sample.services).map(service => ({ name: str(service.name) ?? '', state: str(service.state) })),
140
+ packageMatchesService: typeof sample.packageMatchesService === 'boolean' ? sample.packageMatchesService : null,
141
+ power: { thermalWarning: row(sample.power).thermalWarning === true, performanceWarning: row(sample.power).performanceWarning === true },
142
+ coreSimulator: { orphaned: num(row(sample.coreSimulator).orphaned), externalBooted: num(row(sample.coreSimulator).externalBooted) },
143
+ recording: { failed: num(row(sample.recording).failed) },
144
+ } : null,
145
+ processScan: Object.keys(process).length ? {
146
+ observedAt: stamp(process.observedAt), staleAfterSeconds: num(process.staleAfterSeconds) ?? 900,
147
+ sourceUnavailable: process.sourceUnavailable === true || Boolean(extra.collectorError), status: str(process.status) ?? 'unknown', error: str(process.error), root: process.root === true,
148
+ iocs: rows(process.iocs).map(ioc => ({ status: str(ioc.status) ?? 'unknown', contained: ioc.contained === true })),
149
+ findings: rows(process.findings).map(finding => ({ reason: str(finding.reason) ?? 'unknown' })),
150
+ coverage: { iocs: str(row(process.coverage).iocs) ?? 'partial' }, baseline: { state: str(row(process.baseline).state) ?? 'missing' },
151
+ } : null,
152
+ };
153
+ const observedState = role === 'jenkins' ? state : health.state === 'fresh' ? state : health.state === 'missing' ? 'unknown' : 'stale';
154
+ const releaseFacts = {
155
+ matches: typeof release.matches === 'boolean' ? release.matches : null,
156
+ expectedPackage: str(release.expectedPackage), actualPackage: str(release.actualPackage),
157
+ expectedVersion: str(release.expectedVersion), actualVersion: str(release.actualVersion),
158
+ expectedBuild: str(release.expectedBuild), actualBuild: str(release.actualBuild),
159
+ };
160
+ const diagnosisDetails = machineDiagnosis(diagnosisHost, { release: releaseFacts }, observedState, now, diagnosisTtl);
161
+ const cpu = cpuReading(host, extra, now, fleetTtl);
162
+ return { id, name: safeFleetText(canonical.name ?? host.name ?? extra.name), platform: canonical.platform ?? host.platform ?? extra.platform ?? null,
163
+ role, machineType, inventoryPresent: Object.keys(canonical).length > 0, livePresent: Object.keys(host).length > 0,
164
+ health: { reportedState, effectiveState: state, source: diagnosisSource, ...health,
165
+ ...(role === 'jenkins' ? { hostedEvidence: { reportedState: str(host.state), ...hostedObservation(host, now, fleetTtl) } } : {}) }, diagnosis: diagnosisDetails.summary, diagnosisDetails, cpu,
166
+ diagnosisObservation: { observedAt: health.observedAt, observedState, staleAfterSeconds: diagnosisTtl, source: diagnosisSource,
167
+ hostedObservedAt: stamp(host.observedAt), diagnosticsObservedAt: stamp(sample.observedAt), processObservedAt: stamp(process.observedAt) },
168
+ release: { ...releaseFacts, version: str(host.version), build: str(host.build), observedAt: stamp(host.observedAt) },
169
+ admission: { enabled: canonical.enabled ?? null, acceptsLeases, observation: admissionObservation, observedAt: stamp(canonical.admissionObservedAt), reason: safeFleetText(canonical.admissionReason), requestedFence, fence: appliedFence, operationFence,
170
+ operation: operation ? { operationId: safeFleetText(operation.operationId), kind: safeFleetText(operation.kind), component: safeFleetText(operation.component),
171
+ phase: safeFleetText(operation.phase), status: safeFleetText(operation.status), progress: num(operation.progress), updatedAt: stamp(operation.updatedAt), owner: safeFleetText(operation.owner),
172
+ ...observation(operation.updatedAt, now, 60, operation.stale), error: safeFleetText(operation.error) } : null },
173
+ capacity: role === 'jenkins' ? null : pick(capacity, ['clean', 'leased', 'installing', 'dirty', 'preparing', 'recycling', 'total']),
174
+ capacityObservation: capacitySource === host ? { source: 'hosted', ...hostedObservation(host, now, fleetTtl) } : {
175
+ source: 'shared', ...observation(extra.observedAt, now, context.operatorTtl ?? 120, extra.collectorError)
176
+ },
177
+ serviceChecks: { ...checks, source: useDeep ? 'hosted_deep_evidence' : 'operator_diagnostics', services,
178
+ software: { ...pick(row(diagnostic.software), ['serveSimPackage', 'serveSimVersion', 'serveSimRevision', 'xcodeVersion']), observation: supplementalChecks, packageMatchesService: diagnostic.packageMatchesService ?? null } },
179
+ processChecks: { ...processCheck, ...pick(process, ['attemptedAt', 'inventoryObservedAt', 'maxVariableAgeSeconds', 'root', 'mode', 'coverage', 'counts']),
180
+ status: process.status ?? null, baseline: pick(row(process.baseline), ['state', 'profileIds', 'methods']),
181
+ transportTrust: pick(row(process.transportTrust), ['state', 'verifiedAt', 'pinsSha256']), findings,
182
+ iocs: rows(process.iocs).map(ioc => ({ ...pick(ioc, ['family', 'status', 'contained', 'lastObservedAt']), signals: Array.isArray(ioc.signals) ? ioc.signals.map(safeFleetText) : [] })) },
183
+ resources: { ...observation(row(extra.hostTelemetry).observedAt, now, 45, row(extra.hostTelemetry).collectionError),
184
+ memory: pick(row(row(extra.hostTelemetry).memory), ['totalBytes', 'usedBytes', 'availableBytes', 'swapUsedBytes']),
185
+ perDeviceCpu: rows(row(row(extra.hostTelemetry).cpu).perDevice).map(device => pick(device, ['id', 'cpuPercent', 'rssBytes', 'processes'])),
186
+ processCpuMeaning: 'OS process percent may exceed 100 on multicore hosts; independent from host busy percent' },
187
+ devices: rows(host.devices).map(device => ({ id: str(device.id), ...pick(device, ['state', 'sessionId', 'leaseId', 'updatedAt']), helper: {
188
+ ...pick(row(device.helper), ['state', 'pid', 'lastSeenAt']), observation: observation(row(device.helper).lastSeenAt, now, fleetTtl),
189
+ observationMeaning: 'last helper evidence; freshness uses fleet heartbeat threshold', error: safeFleetText(row(device.helper).error)
190
+ }, lastError: safeFleetText(device.lastError) })),
191
+ deviceObservation: { source: 'hosted', ...hostedObservation(host, now, fleetTtl), meaning: 'provider device records only; shared discovery does not establish current leases' },
192
+ diagnoses: diagnosesForHost,
193
+ };
194
+ }
@@ -0,0 +1,252 @@
1
+ import { ApiError } from '../api.js';
2
+ import { componentHosts, hostReport, hostedObservation, num, observation, row, rows, safeFleetText, stamp, str, time, validFleetId } from './run-cloud-fleet-observations.js';
3
+ export const SURVEY_REQUEST_BUDGET_MS = 8000;
4
+ const ROOT = '/run-cloud/admin/simulators';
5
+ const RANGES = ['1h', '6h', '24h', '7d'];
6
+ const DEVICE_LIMIT = 512, POINT_LIMIT = 800, TYPE_LIMIT = 100;
7
+ function count(value) { const number = num(value); return number !== null && Number.isSafeInteger(number) ? number : null; }
8
+ function counts(value) {
9
+ const data = row(value), errors = count(data.errors), warnings = count(data.warnings), sessions = count(data.sessions);
10
+ return errors === null || warnings === null || sessions === null ? null : { errors, warnings, sessions };
11
+ }
12
+ const unavailable = (reason) => ({ state: 'unavailable', data: {}, reason });
13
+ function resourceReading(host, extra, kind, now, fleetTtl) {
14
+ const telemetry = row(extra.hostTelemetry), volumes = rows(row(telemetry.storage).volumes);
15
+ const probe = kind === 'memory' ? row(telemetry.memory) : volumes.find(volume => volume.label === 'data') ?? volumes[0] ?? {};
16
+ const candidate = (values, source, observed) => {
17
+ const usedBytes = num(values.usedBytes), totalBytes = num(values.totalBytes), availableBytes = num(values.availableBytes);
18
+ const missing = usedBytes === null || totalBytes === null;
19
+ return { source, ...observed, usedBytes, totalBytes, availableBytes,
20
+ percent: usedBytes !== null && totalBytes !== null && totalBytes > 0 ? usedBytes / totalBytes * 100 : null,
21
+ ...(missing && observed.state === 'fresh' ? { state: 'missing', reason: `${kind}_value_missing` } : {}) };
22
+ };
23
+ const candidates = [
24
+ candidate(probe, 'operator_resource_probe', observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError)),
25
+ candidate({ usedBytes: host[`${kind}UsedBytes`], totalBytes: host[`${kind}TotalBytes`] }, 'host_heartbeat', hostedObservation(host, now, fleetTtl)),
26
+ ].sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
27
+ return { ...candidates[0], candidates, ...(kind === 'storage' ? { volumes: volumes.map(volume => ({ label: safeFleetText(volume.label),
28
+ usedBytes: num(volume.usedBytes), totalBytes: num(volume.totalBytes), availableBytes: num(volume.availableBytes) })),
29
+ volumeObservation: observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError),
30
+ meaning: 'one reported volume or heartbeat capacity; volumes may share backing storage and are never summed' } : {}) };
31
+ }
32
+ function alertReport(source, now, requestedRange, hostId) {
33
+ const data = source.data;
34
+ const raw = rows(data.points);
35
+ const valid = source.state === 'available' && data.range === requestedRange && stamp(data.generatedAt) && time(data.generatedAt) <= now + 5000 && num(data.bucketSeconds) &&
36
+ raw.every(point => stamp(point.timestamp) && time(point.timestamp) <= time(data.generatedAt) && counts(point) && Array.isArray(point.errorTypes) && rows(point.errorTypes).every(type => str(type.type) && count(type.errors) !== null && count(type.warnings) !== null));
37
+ if (!valid)
38
+ return { state: 'unavailable', reason: source.reason ?? 'invalid_alert_aggregate', byType: null, totals: null, trend: null,
39
+ coverage: { ingestionCompleteness: 'unknown; aggregate API does not certify event ingestion', hostScope: hostId ? [hostId] : 'all hosts' } };
40
+ const points = raw.slice(-POINT_LIMIT).sort((a, b) => time(a.timestamp) - time(b.timestamp));
41
+ const generatedAt = stamp(data.generatedAt), bucketSeconds = Number(data.bucketSeconds);
42
+ const completeBuckets = points.filter(point => time(point.timestamp) + bucketSeconds * 1000 <= time(generatedAt));
43
+ const recent = completeBuckets.at(-1), previous = completeBuckets.at(-2);
44
+ const comparable = recent && previous && time(recent.timestamp) - time(previous.timestamp) === bucketSeconds * 1000;
45
+ const types = new Map();
46
+ let omittedTypeRows = 0;
47
+ for (const point of points) {
48
+ const entries = rows(point.errorTypes);
49
+ omittedTypeRows += Math.max(0, entries.length - TYPE_LIMIT);
50
+ for (const entry of entries.slice(0, TYPE_LIMIT)) {
51
+ const type = String(entry.type), before = types.get(type) ?? { type, errors: 0, warnings: 0, recent: null, previous: null, delta: null };
52
+ before.errors += Number(entry.errors);
53
+ before.warnings += Number(entry.warnings);
54
+ types.set(type, before);
55
+ }
56
+ }
57
+ for (const entry of types.values())
58
+ if (comparable && omittedTypeRows === 0) {
59
+ const inBucket = (point) => rows(point.errorTypes).filter(type => type.type === entry.type).reduce((sum, type) => sum + Number(type.errors) + Number(type.warnings), 0);
60
+ entry.recent = inBucket(recent);
61
+ entry.previous = inBucket(previous);
62
+ entry.delta = entry.recent - entry.previous;
63
+ }
64
+ const window = (point) => ({ since: stamp(point.timestamp), until: new Date(time(point.timestamp) + bucketSeconds * 1000).toISOString(), ...counts(point) });
65
+ return { state: 'available', reason: null, generatedAt, responseAgeSeconds: (now - time(generatedAt)) / 1000,
66
+ byType: [...types.values()].sort((a, b) => b.errors + b.warnings - a.errors - a.warnings || a.type.localeCompare(b.type)),
67
+ observedBucketCounts: points.reduce((total, point) => ({ errors: total.errors + Number(point.errors), warnings: total.warnings + Number(point.warnings) }), { errors: 0, warnings: 0 }),
68
+ totals: { hour: counts(row(data.totals).hour), today: counts(row(data.totals).today), yesterday: counts(row(data.totals).yesterday) },
69
+ windows: { hour: stamp(row(data.windows).hour), today: stamp(row(data.windows).today), yesterday: stamp(row(data.windows).yesterday), yesterdayEnd: stamp(row(data.windows).yesterdayEnd),
70
+ timeZone: str(data.timeZone), until: generatedAt },
71
+ trend: comparable ? { recent: window(recent), previous: window(previous), delta: Number(recent.errors) + Number(recent.warnings) - Number(previous.errors) - Number(previous.warnings),
72
+ meaning: 'last two complete equal buckets; current partial bucket excluded' } : null,
73
+ coverage: { requestedRange, hostScope: hostId ? [hostId] : 'all hosts', bucketSeconds, firstBucketAt: stamp(points[0]?.timestamp), lastBucketAt: stamp(points.at(-1)?.timestamp),
74
+ buckets: points.length, omittedBuckets: Math.max(0, raw.length - POINT_LIMIT), omittedTypeRows, unattributedAlertRows: count(data.unattributedAlerts),
75
+ unattributedMeaning: 'missing customer attribution over the API query window, which may extend before the plotted range',
76
+ meaning: 'observed warning/critical event rows after server bookkeeping exclusions; retries can repeat rows; not unique incidents or a failure rate',
77
+ ingestionCompleteness: 'unknown; aggregate API does not certify event ingestion', windowAlignment: 'API bucket boundaries can extend before the requested rolling window' } };
78
+ }
79
+ function lagReport(source, selection, eligibleDevices, now, range) {
80
+ const data = source.data, selected = new Set(selection.map(([host, device]) => `${host} ${device}`));
81
+ const validSample = (sample) => count(sample.samples) !== null && (sample.samples === 0 ? sample.medianMs === null : num(sample.medianMs) !== null);
82
+ const valid = source.state === 'available' && data.metric === 'input_lag_visual_v1' && data.range === range && stamp(data.generatedAt) && time(data.generatedAt) <= now + 5000 &&
83
+ num(data.bucketSeconds) && count(data.selectedDevices) === selection.length && Array.isArray(data.points) &&
84
+ rows(data.points).every(point => stamp(point.timestamp) && time(point.timestamp) <= time(data.generatedAt) && validSample(point) && count(point.missing) !== null) &&
85
+ data.devices && typeof data.devices === 'object' && !Array.isArray(data.devices) && Object.values(row(data.devices)).every(sample => validSample(row(sample)));
86
+ const points = valid ? rows(data.points).slice(-POINT_LIMIT).sort((a, b) => time(a.timestamp) - time(b.timestamp)) : [];
87
+ const observed = points.filter(point => stamp(point.timestamp) && count(point.samples) !== null && Number(point.samples) > 0 && num(point.medianMs) !== null);
88
+ const latest = observed.at(-1);
89
+ const devices = valid && data.devices && typeof data.devices === 'object' && !Array.isArray(data.devices) ? Object.entries(row(data.devices))
90
+ .filter(([key]) => selected.has(key)).map(([key, value]) => {
91
+ const sample = row(value), samples = count(sample.samples), medianMs = samples !== null && samples > 0 ? num(sample.medianMs) : null;
92
+ const [hostId, deviceId] = key.split(' ');
93
+ return { hostId, deviceId, medianMs, samples, outcomes: sample.outcomes ? outcomeCounts(sample.outcomes) : null,
94
+ observationAgeSeconds: null, meaning: 'median across valid changed samples in query window; API provides no per-device last sample time' };
95
+ }) : null;
96
+ return { state: valid ? 'available' : source.state === 'not_requested' ? 'not_requested' : 'unavailable',
97
+ reason: valid ? null : source.reason ?? 'invalid_input_lag_aggregate', metric: 'input_lag_visual_v1', generatedAt: stamp(data.generatedAt),
98
+ responseAgeSeconds: stamp(data.generatedAt) ? (now - time(data.generatedAt)) / 1000 : null,
99
+ latestSampledBucket: latest ? { timestamp: stamp(latest.timestamp), bucketSeconds: Number(data.bucketSeconds), bucketAgeSeconds: (now - time(latest.timestamp)) / 1000,
100
+ medianMs: Number(latest.medianMs), samples: count(latest.samples), missing: count(latest.missing),
101
+ meaning: 'bucket aggregate for selected devices; bucket age is not the age of its last individual sample' } : null,
102
+ outcomes: valid && data.totals ? outcomeCounts(data.totals) : null, devices,
103
+ coverage: { requestedRange: range, eligibleDevices, requestedDevices: selection.length, apiSelectedDevices: valid ? count(data.selectedDevices) : null,
104
+ omittedDevices: Math.max(0, eligibleDevices - selection.length), deviceLimit: DEVICE_LIMIT,
105
+ unrequestedDeviceRecords: valid ? Object.keys(row(data.devices)).filter(key => !selected.has(key)).length : null,
106
+ returnedDevices: devices?.length ?? null, sampledDevices: devices?.filter(device => device.medianMs !== null).length ?? null,
107
+ firstBucketAt: stamp(points[0]?.timestamp), lastBucketAt: stamp(points.at(-1)?.timestamp), bucketSeconds: valid ? num(data.bucketSeconds) : null,
108
+ omittedBuckets: valid ? Math.max(0, rows(data.points).length - POINT_LIMIT) : null,
109
+ ingestionCompleteness: 'unknown; missing, no-change, superseded, hidden and sample-error outcomes are not successful latency samples' } };
110
+ }
111
+ function outcomeCounts(value) {
112
+ const data = row(value);
113
+ return Object.fromEntries(['changed', 'no_change', 'superseded', 'hidden', 'sample_error'].map(key => [key, count(data[key])]));
114
+ }
115
+ export async function surveyFleet(client, options = {}, now, requestBudgetMs = SURVEY_REQUEST_BUDGET_MS) {
116
+ const range = options.range ?? '24h';
117
+ if (!RANGES.includes(range))
118
+ throw new Error(`survey range must be ${RANGES.join(', ')}`);
119
+ if (options.hostId && !validFleetId(options.hostId))
120
+ throw new Error('Invalid host ID');
121
+ if (!Number.isSafeInteger(requestBudgetMs) || requestBudgetMs < 1 || requestBudgetMs > SURVEY_REQUEST_BUDGET_MS)
122
+ throw new Error('Invalid survey request budget');
123
+ const started = performance.now(), signal = AbortSignal.timeout(requestBudgetMs);
124
+ const read = async (path, key, body) => {
125
+ try {
126
+ const value = body ? await client.post(path, body, { signal }) : await client.get(path, { signal });
127
+ const data = row(value);
128
+ if (value instanceof Response || !Array.isArray(data[key]))
129
+ return unavailable('invalid_response_shape');
130
+ if (path === '/run-cloud/admin/jenkins' && data.state === 'unavailable')
131
+ return { state: 'unavailable', data, reason: 'jenkins_collection_unavailable' };
132
+ if (path === '/run-cloud/admin/fleet-operations' && (data.unavailable === true || data.configured === false))
133
+ return { state: 'unavailable', data, reason: 'operation_storage_unavailable_or_unconfigured' };
134
+ return { state: 'available', data, reason: null };
135
+ }
136
+ catch (error) {
137
+ return unavailable(error instanceof ApiError ? `HTTP ${error.status}` : signal.aborted ? 'request_budget_exhausted' : 'read_failed');
138
+ }
139
+ };
140
+ const fleetPromise = read(ROOT, 'hosts');
141
+ const inventoryPromise = read('/run-cloud/admin/fleet-inventory', 'hosts');
142
+ const sharedPromise = read(`${ROOT}/operator-inventory`, 'snapshots');
143
+ const jenkinsPromise = read('/run-cloud/admin/jenkins', 'nodes');
144
+ const operationsPromise = read('/run-cloud/admin/fleet-operations', 'hosts');
145
+ const analyticsPromise = read(`${ROOT}/analytics`, 'points', { range, ...(options.hostId ? { hostIds: [options.hostId] } : {}) });
146
+ let selection = [], eligibleDevices = 0;
147
+ const lagPromise = fleetPromise.then(async (fleet) => {
148
+ if (fleet.state !== 'available')
149
+ return unavailable('fleet_device_selection_unavailable');
150
+ const unique = new Map();
151
+ for (const host of rows(fleet.data.hosts))
152
+ if (!host.monitoringOnly && (!options.hostId || host.id === options.hostId)) {
153
+ for (const device of rows(host.devices))
154
+ if (validFleetId(host.id) && validFleetId(device.id))
155
+ unique.set(`${host.id} ${device.id}`, [String(host.id), String(device.id)]);
156
+ }
157
+ const devices = [...unique.values()].sort((a, b) => a.join(' ').localeCompare(b.join(' ')));
158
+ eligibleDevices = devices.length;
159
+ selection = devices.slice(0, DEVICE_LIMIT);
160
+ return selection.length ? read(`${ROOT}/input-lag/history`, 'points', { range, devices: selection }) : { state: 'not_requested', data: {}, reason: 'no_queryable_devices' };
161
+ });
162
+ const [fleet, inventory, shared, jenkins, operations, analytics, lag] = await Promise.all([
163
+ fleetPromise, inventoryPromise, sharedPromise, jenkinsPromise, operationsPromise, analyticsPromise, lagPromise,
164
+ ]);
165
+ if (fleet.state !== 'available')
166
+ throw new Error(`Fleet read unavailable (${fleet.reason}); use an admin identity with read permission`);
167
+ now ??= Date.now();
168
+ const live = new Map(rows(fleet.data.hosts).filter(host => validFleetId(host.id)).map(host => [String(host.id), host]));
169
+ const canonical = new Map(rows(inventory.data.hosts).filter(host => validFleetId(host.id)).map(host => [String(host.id), host]));
170
+ const components = componentHosts(shared.data), supplemental = new Map([...components].filter(([id]) => validFleetId(id))), ids = new Set([...canonical.keys(), ...live.keys(), ...supplemental.keys()]);
171
+ if (options.hostId && !ids.has(options.hostId))
172
+ throw new Error(`Host ${options.hostId} is absent from returned inventories`);
173
+ const inputLag = lagReport(lag, selection, eligibleDevices, now, range), fleetTtl = num(fleet.data.staleAfterSeconds) ?? 45;
174
+ const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => {
175
+ const host = live.get(id) ?? {}, extra = supplemental.get(id) ?? {};
176
+ const report = hostReport(host, canonical.get(id) ?? {}, extra, [], now, fleetTtl, {}, {
177
+ inventoryTtl: num(inventory.data.staleAfterSeconds) ?? 30, jenkins: jenkins.data,
178
+ operations: rows(operations.data.hosts).map(operation => ({ ...operation, stale: operation.stale || operations.state !== 'available' })),
179
+ });
180
+ const devices = inputLag.devices?.filter(device => device.hostId === id) ?? null;
181
+ const sampled = devices?.filter(device => device.medianMs !== null) ?? [];
182
+ return { ...report, memory: resourceReading(host, extra, 'memory', now, fleetTtl), storage: resourceReading(host, extra, 'storage', now, fleetTtl),
183
+ inputLag: { devices, selectedDevices: selection.filter(([hostId]) => hostId === id).length, sampledDevices: devices ? sampled.length : null,
184
+ highestDeviceMedianMs: sampled.length ? Math.max(...sampled.map(device => device.medianMs)) : null,
185
+ meaning: 'per-device window medians; highest is a device ranking, not a host or fleet percentile; no per-device observation time' } };
186
+ });
187
+ const alerts = alertReport(analytics, now, range, options.hostId);
188
+ const sources = Object.fromEntries(Object.entries({ fleet, inventory, shared, jenkins, operations,
189
+ analytics: { ...analytics, state: alerts.state, reason: alerts.reason }, inputLag: { ...lag, state: inputLag.state, reason: inputLag.reason } }).map(([key, source]) => [key, {
190
+ state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt ?? source.data.observedAt),
191
+ responseAgeSeconds: stamp(source.data.generatedAt ?? source.data.observedAt) ? (now - time(source.data.generatedAt ?? source.data.observedAt)) / 1000 : null,
192
+ meaning: 'response/collector generation time; does not renew component observations',
193
+ }]));
194
+ const cpu = hosts.filter(host => host.cpu.state === 'fresh' && host.cpu.percent !== null);
195
+ return { schemaVersion: 1, generatedAt: new Date(now).toISOString(), readOnly: true, scope: options.hostId ?? 'whole fleet', range,
196
+ durationSeconds: (performance.now() - started) / 1000, requestBudgetSeconds: requestBudgetMs / 1000, sources,
197
+ summary: { hosts: hosts.length, canonicalHosts: inventory.state === 'available' ? canonical.size : null, hostedHosts: live.size,
198
+ diagnosisStates: hosts.reduce((counts, host) => { counts[host.diagnosisDetails.status] = (counts[host.diagnosisDetails.status] ?? 0) + 1; return counts; }, {}),
199
+ cpu: { meanBusyPercent: cpu.length ? cpu.reduce((sum, host) => sum + host.cpu.percent, 0) / cpu.length : null, freshHosts: cpu.length, totalHosts: hosts.length,
200
+ meaning: 'arithmetic mean across fresh host busy percentages; not core weighted; inventory aliases can share physical machines' } },
201
+ coverage: { sourcesAvailable: Object.values(sources).filter(source => source.state === 'available').length, sourcesTotal: 7,
202
+ invalidHostedRecords: rows(fleet.data.hosts).length - live.size, invalidCanonicalRecords: rows(inventory.data.hosts).length - canonical.size,
203
+ invalidSupplementalRecords: components.size - supplemental.size,
204
+ sharedSnapshots: rows(shared.data.snapshots).map(entry => {
205
+ const snapshot = row(entry.snapshot);
206
+ return { publisherId: safeFleetText(entry.publisherId), hosts: rows(snapshot.hosts).length,
207
+ ...observation(snapshot.observedAt, now, num(snapshot.staleAfterSeconds) ?? 120, snapshot.collectorError) };
208
+ }),
209
+ localCollector: 'not queried; shared published samples only',
210
+ limits: 'one read per source; at most 512 devices, 800 buckets and 100 alert types per bucket; no event pagination, SSH, scans or new latency probes',
211
+ missingSources: Object.entries(sources).filter(([, source]) => source.state !== 'available').map(([name, source]) => ({ name, reason: source.reason })),
212
+ consistency: 'independent source reads, not a transaction; component observations and ages retained' }, alerts, inputLag, hosts };
213
+ }
214
+ function age(value) { return `${value.state}, age ${value.ageSeconds === null ? 'unknown' : `${Math.round(value.ageSeconds)}s`}${value.reason ? `, ${value.reason}` : ''}`; }
215
+ function bytes(value) { return value === null ? 'unknown' : `${(value / 1024 ** 3).toFixed(1)} GiB`; }
216
+ export function surveyText(report) {
217
+ const lines = [`Fleet survey · ${report.scope} · ${report.generatedAt} · ${report.durationSeconds.toFixed(2)}s (request budget ${report.requestBudgetSeconds}s)`,
218
+ `${report.summary.hosts} machine records (${report.summary.canonicalHosts ?? 'unknown'} canonical); ${report.coverage.sourcesAvailable}/7 sources available.`,
219
+ `Diagnosis states: ${Object.entries(report.summary.diagnosisStates).map(([state, count]) => `${state}=${count}`).join(', ')}.`,
220
+ `CPU mean: ${report.summary.cpu.meanBusyPercent?.toFixed(1) ?? 'unknown'}% from ${report.summary.cpu.freshHosts}/${report.summary.cpu.totalHosts} fresh host samples. ${report.summary.cpu.meaning}.`];
221
+ if (report.alerts.state === 'available') {
222
+ lines.push(`Alerts · ${report.range} aligned buckets · ${report.alerts.observedBucketCounts.errors} critical / ${report.alerts.observedBucketCounts.warnings} warning rows; retries can repeat rows. Aggregate ends ${report.alerts.generatedAt} (response age ${Math.round(report.alerts.responseAgeSeconds)}s).`);
223
+ for (const type of report.alerts.byType)
224
+ lines.push(` ${type.type}: ${type.errors} critical, ${type.warnings} warning; complete-bucket trend ${type.previous ?? 'unknown'} → ${type.recent ?? 'unknown'}${type.delta === null ? '' : ` (${type.delta >= 0 ? '+' : ''}${type.delta})`}`);
225
+ if (!report.alerts.byType.length)
226
+ lines.push(' No observed alert rows in returned buckets; ingestion completeness is unknown.');
227
+ if (report.alerts.trend)
228
+ lines.push(` Trend windows: ${report.alerts.trend.previous.since}..${report.alerts.trend.previous.until} → ${report.alerts.trend.recent.since}..${report.alerts.trend.recent.until}; current partial bucket excluded.`);
229
+ for (const period of ['hour', 'today', 'yesterday']) {
230
+ const totals = report.alerts.totals[period], windows = report.alerts.windows;
231
+ lines.push(` ${period}: ${totals ? `${totals.errors} critical, ${totals.warnings} warning, ${totals.sessions} sessions` : 'unknown'}; ${windows[period] ?? 'unknown'}..${period === 'yesterday' ? windows.yesterdayEnd ?? 'unknown' : windows.until} (${windows.timeZone ?? 'timezone unknown'}).`);
232
+ }
233
+ lines.push(` Customer-unattributed alert rows: ${report.alerts.coverage.unattributedAlertRows ?? 'unknown'} (API query window may be wider). Ingestion completeness unknown; counts are not incident counts or failure rates.`);
234
+ if (report.alerts.coverage.omittedBuckets || report.alerts.coverage.omittedTypeRows)
235
+ lines.push(` Capped coverage: ${report.alerts.coverage.omittedBuckets} buckets / ${report.alerts.coverage.omittedTypeRows} type rows omitted.`);
236
+ }
237
+ else
238
+ lines.push(`Alerts: unavailable (${report.alerts.reason}); counts and trends unknown.`);
239
+ const lag = report.inputLag.latestSampledBucket, coverage = report.inputLag.coverage;
240
+ lines.push(`Input lag: ${lag ? `${lag.medianMs.toFixed(1)}ms median, ${lag.samples} samples in bucket ${lag.timestamp} (bucket age ${Math.round(lag.bucketAgeSeconds)}s; individual sample age unknown)` : `no sampled bucket (${report.inputLag.reason ?? 'no changed samples'})`}.`, ` Device coverage ${coverage.sampledDevices ?? 'unknown'} sampled / ${coverage.requestedDevices} requested / ${coverage.eligibleDevices} eligible; ${coverage.omittedDevices} omitted (limit ${coverage.deviceLimit}).`, ` Outcomes: ${report.inputLag.outcomes ? Object.entries(report.inputLag.outcomes).map(([key, count]) => `${key}=${count ?? 'unknown'}`).join(', ') : 'unknown'}. Missing outcomes are not successful latency samples.`);
241
+ for (const host of report.hosts) {
242
+ lines.push('', `${host.id} · ${host.role} · CPU ${host.cpu.percent?.toFixed(1) ?? 'unknown'}% [${age(host.cpu)}] · RAM ${bytes(host.memory.usedBytes)}/${bytes(host.memory.totalBytes)} [${age(host.memory)}] · Storage ${bytes(host.storage.usedBytes)}/${bytes(host.storage.totalBytes)} [${age(host.storage)}]`, ` ${host.diagnosis}`, ` Service ${age(host.health)} at ${host.health.observedAt ?? 'missing'}; details ${age(host.serviceChecks)}; process ${age(host.processChecks)}; admission ${age(host.admission.observation)}.`, ` Release ${host.release.actualVersion ?? host.release.version ?? 'unknown'} / expected ${host.release.expectedVersion ?? 'unknown'} at ${host.release.observedAt ?? 'missing'}; input lag highest device median ${host.inputLag.highestDeviceMedianMs?.toFixed(1) ?? 'unknown'}ms (${host.inputLag.sampledDevices ?? 'unknown'}/${host.inputLag.selectedDevices} selected devices sampled; window median, individual age unknown).`);
243
+ if (host.admission.operation)
244
+ lines.push(` Operation ${host.admission.operation.operationId ?? 'unknown'} · owner ${host.admission.operation.owner ?? 'unknown'} · ${host.admission.operation.phase ?? 'unknown'} / ${host.admission.operation.status ?? 'unknown'} [${age(host.admission.operation)}].`);
245
+ }
246
+ for (const source of report.coverage.missingSources)
247
+ lines.push(`Missing source: ${source.name} · ${source.reason}.`);
248
+ for (const [name, source] of Object.entries(report.sources))
249
+ lines.push(`Source ${name}: ${source.state}, generated ${source.generatedAt ?? 'unknown'}, response age ${source.responseAgeSeconds === null ? 'unknown' : `${Math.round(source.responseAgeSeconds)}s`}${source.reason ? `, ${source.reason}` : ''}.`);
250
+ lines.push(`Coverage: ${report.coverage.localCollector}. ${report.coverage.consistency}. ${report.coverage.limits}.`);
251
+ return lines.join('\n');
252
+ }
@@ -1,48 +1,12 @@
1
+ import { row, rows, str, num, stamp, time, pick, observation, componentHosts, hostReport, safeFleetText, validFleetId } from './run-cloud-fleet-observations.js';
2
+ export { safeFleetText } from './run-cloud-fleet-observations.js';
1
3
  import { readFileSync, statSync } from 'node:fs';
2
4
  import { ApiClient, ApiError } from '../api.js';
3
5
  import { requireCredentials } from '../config.js';
6
+ import { surveyFleet, surveyText } from './run-cloud-fleet-survey.js';
4
7
  const ROOT = '/run-cloud/admin/simulators';
5
8
  const RANGES = ['1h', '6h', '24h', '7d', '30d', '90d', '1y', '2y'];
6
9
  export const ACCESSIBILITY_CODES = ['accessibility_device_unavailable', 'accessibility_capture_failed'];
7
- const row = (value) => value !== null && typeof value === 'object' && !Array.isArray(value) ? value : {};
8
- const rows = (value) => Array.isArray(value) ? value.map(row) : [];
9
- const str = (value) => typeof value === 'string' && value.length > 0 ? value : null;
10
- const num = (value) => typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : null;
11
- const stamp = (value) => str(value) && Number.isFinite(Date.parse(String(value))) ? new Date(String(value)).toISOString() : null;
12
- const time = (value) => stamp(value) ? Date.parse(String(value)) : 0;
13
- export function safeFleetText(value) {
14
- const text = str(value);
15
- return text ? text.slice(0, 8000)
16
- .replace(/\bBearer\s+[^\s"'<>]+/gi, 'Bearer [redacted]')
17
- .replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
18
- .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
19
- .replace(/\b([a-zA-Z0-9_]*(?:token|password|secret|api[_-]?key)|authorization)\s*[=:]\s*[^\s,;]+/gi, '$1=[redacted]')
20
- .replace(/(?:https?|wss?):\/\/[^\s<>"']+/gi, '[url redacted]') : null;
21
- }
22
- function observation(value, now, ttl, error) {
23
- const observedAt = stamp(value);
24
- const ageSeconds = observedAt ? Math.round((now - Date.parse(observedAt)) / 1000) : null;
25
- if (error)
26
- return { observedAt, ageSeconds, state: 'unavailable', reason: 'collection_failed; retained evidence only' };
27
- if (!observedAt)
28
- return { observedAt, ageSeconds, state: 'missing', reason: 'observation_timestamp_missing' };
29
- if (ageSeconds < -5)
30
- return { observedAt, ageSeconds, state: 'unavailable', reason: 'observation_timestamp_in_future' };
31
- return { observedAt, ageSeconds, state: ageSeconds > ttl ? 'stale' : 'fresh', reason: ageSeconds > ttl ? 'observation_expired' : null };
32
- }
33
- function hostedObservation(host, now, ttl) {
34
- const heartbeat = observation(host.observedAt, now, ttl);
35
- const freshness = row(host.freshness);
36
- if (!Object.keys(freshness).length)
37
- return { ...heartbeat, ingestion: null, reportedFreshness: null };
38
- const ingestion = observation(freshness.lastSuccessfulUpdateAt, now, ttl);
39
- const reportedFreshness = str(freshness.status);
40
- if (heartbeat.state === 'fresh' && (reportedFreshness !== 'fresh' || ingestion.state !== 'fresh')) {
41
- const state = ingestion.state === 'unavailable' ? 'unavailable' : reportedFreshness === 'never' ? 'missing' : 'stale';
42
- return { ...heartbeat, state: state, reason: `hosted_ingestion_${ingestion.state === 'fresh' ? reportedFreshness ?? 'unknown' : ingestion.state}`, ingestion, reportedFreshness };
43
- }
44
- return { ...heartbeat, ingestion, reportedFreshness };
45
- }
46
10
  function bound(value, fallback, max, name) {
47
11
  const result = value ?? fallback;
48
12
  if (!Number.isSafeInteger(result) || result < 1 || result > max)
@@ -52,7 +16,7 @@ function bound(value, fallback, max, name) {
52
16
  function validate(options) {
53
17
  if (!RANGES.includes(options.range ?? '24h'))
54
18
  throw new Error(`range must be ${RANGES.join(', ')}`);
55
- if (options.hostId && !/^[a-zA-Z0-9][a-zA-Z0-9._-]{0,127}$/.test(options.hostId))
19
+ if (options.hostId && !validFleetId(options.hostId))
56
20
  throw new Error('Invalid host ID');
57
21
  if (options.deep && !options.hostId)
58
22
  throw new Error('--deep requires a host ID');
@@ -107,101 +71,6 @@ export async function readFleetEvents(client, options = {}) {
107
71
  reason: reason ?? (cursor ? 'page_budget_reached; matching failures may be outside returned pages' : null), generatedAt,
108
72
  oldestReturnedAt: stamp(items.at(-1)?.timestamp), newestReturnedAt: stamp(items[0]?.timestamp) } };
109
73
  }
110
- function componentHosts(shared) {
111
- const result = new Map();
112
- const snapshots = rows(shared.snapshots).map(entry => row(entry.snapshot));
113
- for (const snapshot of snapshots)
114
- for (const host of rows(snapshot.hosts)) {
115
- const id = str(host.id);
116
- if (!id)
117
- continue;
118
- const before = result.get(id) ?? {};
119
- const newest = !result.has(id) || time(host.observedAt) > time(before.observedAt) ? host : before;
120
- const merged = { ...before, ...newest };
121
- for (const key of ['hostTelemetry', 'hostDiagnostics', 'processScan']) {
122
- const candidate = row(host[key]), previous = row(before[key]);
123
- const observationKey = key === 'processScan' ? 'attemptedAt' : 'observedAt';
124
- merged[key] = time(candidate[observationKey] ?? candidate.observedAt) > time(previous[observationKey] ?? previous.observedAt) ? candidate : previous;
125
- }
126
- result.set(id, merged);
127
- }
128
- return result;
129
- }
130
- function cpuReading(host, extra, now, fleetTtl) {
131
- const telemetry = row(extra.hostTelemetry), cpu = row(telemetry.cpu);
132
- const cpuValue = (value) => num(value) !== null && Number(value) <= 100 ? Number(value) : null;
133
- const candidates = [
134
- { percent: cpuValue(cpu.busyPercent), source: 'operator_resource_probe', ...observation(telemetry.observedAt, now, num(telemetry.staleAfterSeconds) ?? 45, telemetry.collectionError) },
135
- { percent: cpuValue(host.cpuUtilizationPercent), source: 'host_heartbeat', ...hostedObservation(host, now, fleetTtl) },
136
- ].map(entry => entry.percent === null && entry.state === 'fresh' ? { ...entry, state: 'missing', reason: 'cpu_value_missing' } : entry);
137
- candidates.sort((a, b) => Number(b.state === 'fresh') - Number(a.state === 'fresh') || time(b.observedAt) - time(a.observedAt));
138
- return { ...candidates[0], candidates, meaning: 'host busy percent, 0..100; fresh samples only in fleet mean' };
139
- }
140
- const pick = (source, keys) => Object.fromEntries(keys.filter(key => source[key] !== undefined).map(key => [key, source[key]]));
141
- function fence(value) {
142
- const source = row(value);
143
- return Object.keys(source).length ? { ...pick(source, ['id', 'kind', 'state', 'createdAt', 'expiresAt', 'component']), reason: safeFleetText(source.reason) } : null;
144
- }
145
- function hostReport(host, canonical, extra, diagnoses, now, fleetTtl, deep = {}) {
146
- const id = String(host.id ?? canonical.id ?? extra.id);
147
- const health = hostedObservation(host, now, fleetTtl);
148
- const diagnostic = row(extra.hostDiagnostics), process = row(extra.processScan);
149
- const supplementalChecks = observation(diagnostic.observedAt, now, num(diagnostic.staleAfterSeconds) ?? 900, diagnostic.collectionError);
150
- const deepChecks = observation(row(deep.security).checkedAt, now, fleetTtl);
151
- const useDeep = deepChecks.state === 'fresh' && Array.isArray(deep.services) && deep.services.length > 0 && time(deepChecks.observedAt) > time(supplementalChecks.observedAt);
152
- const checks = useDeep ? deepChecks : supplementalChecks;
153
- const processCheck = observation(process.observedAt, now, num(process.staleAfterSeconds) ?? 900, process.error);
154
- const release = row(host.release);
155
- const services = rows(useDeep ? deep.services : diagnostic.services).map(service => ({ name: safeFleetText(service.name), state: safeFleetText(service.state), lastExitCode: service.lastExitCode ?? null,
156
- error: safeFleetText(service.lastError ?? service.error) }));
157
- const faults = checks.state === 'fresh' ? services.filter(service => ['stopped', 'failed', 'crashed', 'inactive', 'missing', 'exited', 'disabled'].includes(service.state ?? '')) : [];
158
- const findings = rows(process.findings).map(finding => ({ ...pick(finding, ['fingerprint', 'reason', 'confidence', 'firstObservedAt', 'lastObservedAt', 'ownerUid', 'ageSeconds']), message: safeFleetText(finding.message ?? finding.summary) }));
159
- const diagnosesForHost = diagnoses.filter(diagnosis => diagnosis.hostId === id).map(diagnosis => ({ ...pick(diagnosis, ['code', 'severity', 'deviceId']),
160
- title: safeFleetText(diagnosis.title), message: safeFleetText(diagnosis.message), reportedAt: stamp(diagnosis.observedAt) }));
161
- const requestedFence = fence(canonical.requestedFence ?? host.requestedFence), appliedFence = fence(canonical.fence ?? host.fence);
162
- const state = health.state === 'fresh' ? str(host.state) ?? 'unknown' : 'unknown';
163
- const sentences = [`${id}: ${state}${health.state === 'fresh' ? '' : ` (${health.state} heartbeat)`}.`];
164
- if (appliedFence || requestedFence)
165
- sentences.push(`Hold ${appliedFence ? 'reported applied' : 'requested'}; admission evidence is separate.`);
166
- if (health.state === 'fresh' && release.matches === false)
167
- sentences.push('Release drift reported.');
168
- if (faults.length)
169
- sentences.push(`Service faults: ${faults.map(service => `${service.name} ${service.state}`).join(', ')}.`);
170
- if (checks.state !== 'fresh')
171
- sentences.push(`Service checks ${checks.state}; current service health unproved.`);
172
- if (processCheck.state !== 'fresh')
173
- sentences.push(`Process checks ${processCheck.state}.`);
174
- if (process.status === 'critical' || rows(process.iocs).some(ioc => ioc.status === 'present') || rows(process.findings).some(finding => finding.reason === 'known_ioc'))
175
- sentences.push('Known malicious scanner evidence retained; inspect signals and scan age.');
176
- else if (rows(process.findings).some(finding => ['rooted_workload', 'long_lived_workload', 'identity_changed', 'unreviewed_persistence'].includes(String(finding.reason))))
177
- sentences.push('Process identities require review; inspect findings and scan age.');
178
- if (rows(process.iocs).some(ioc => ioc.contained === true))
179
- sentences.push('Contained IOC evidence retained.');
180
- const cpu = cpuReading(host, extra, now, fleetTtl);
181
- sentences.push(cpu.state === 'fresh' ? `CPU ${cpu.percent.toFixed(1)}%.` : `CPU ${cpu.state}.`);
182
- return { id, name: safeFleetText(canonical.name ?? host.name ?? extra.name), platform: canonical.platform ?? host.platform ?? extra.platform ?? null,
183
- role: canonical.machineType ?? extra.role ?? null, inventoryPresent: Object.keys(canonical).length > 0, livePresent: Object.keys(host).length > 0,
184
- health: { reportedState: str(host.state), effectiveState: state, ...health }, diagnosis: sentences.join(' '), cpu,
185
- release: { ...pick(release, ['expectedPackage', 'actualPackage', 'expectedVersion', 'actualVersion', 'expectedBuild', 'actualBuild', 'matches']), version: str(host.version), build: str(host.build), observedAt: stamp(host.observedAt) },
186
- admission: { enabled: canonical.enabled ?? null, acceptsLeases: canonical.acceptsLeases ?? null, observedAt: stamp(canonical.admissionObservedAt), reason: safeFleetText(canonical.admissionReason), requestedFence, fence: appliedFence },
187
- capacity: pick(row(host.capacity), ['clean', 'leased', 'installing', 'dirty', 'preparing', 'recycling', 'total']),
188
- serviceChecks: { ...checks, source: useDeep ? 'hosted_deep_evidence' : 'operator_diagnostics', services,
189
- software: { ...pick(row(diagnostic.software), ['serveSimPackage', 'serveSimVersion', 'serveSimRevision', 'xcodeVersion']), observation: supplementalChecks, packageMatchesService: diagnostic.packageMatchesService ?? null } },
190
- processChecks: { ...processCheck, ...pick(process, ['attemptedAt', 'inventoryObservedAt', 'maxVariableAgeSeconds', 'root', 'mode', 'coverage', 'counts']),
191
- status: process.status ?? null, baseline: pick(row(process.baseline), ['state', 'profileIds', 'methods']),
192
- transportTrust: pick(row(process.transportTrust), ['state', 'verifiedAt', 'pinsSha256']), findings,
193
- iocs: rows(process.iocs).map(ioc => ({ ...pick(ioc, ['family', 'status', 'contained', 'lastObservedAt']), signals: Array.isArray(ioc.signals) ? ioc.signals.map(safeFleetText) : [] })) },
194
- resources: { ...observation(row(extra.hostTelemetry).observedAt, now, 45, row(extra.hostTelemetry).collectionError),
195
- memory: pick(row(row(extra.hostTelemetry).memory), ['totalBytes', 'usedBytes', 'availableBytes', 'swapUsedBytes']),
196
- perDeviceCpu: rows(row(row(extra.hostTelemetry).cpu).perDevice).map(device => pick(device, ['id', 'cpuPercent', 'rssBytes', 'processes'])),
197
- processCpuMeaning: 'OS process percent may exceed 100 on multicore hosts; independent from host busy percent' },
198
- devices: rows(host.devices).map(device => ({ id: str(device.id), ...pick(device, ['state', 'sessionId', 'leaseId', 'updatedAt']), helper: {
199
- ...pick(row(device.helper), ['state', 'pid', 'lastSeenAt']), observation: observation(row(device.helper).lastSeenAt, now, fleetTtl),
200
- observationMeaning: 'last helper evidence; freshness uses fleet heartbeat threshold', error: safeFleetText(row(device.helper).error)
201
- }, lastError: safeFleetText(device.lastError) })),
202
- diagnoses: diagnosesForHost,
203
- };
204
- }
205
74
  function originalLogsUrl(value) {
206
75
  try {
207
76
  const url = new URL(String(value));
@@ -300,20 +169,25 @@ export async function inspectFleet(client, options = {}, now) {
300
169
  const fleet = await readSource(client, ROOT);
301
170
  if (fleet.state === 'unavailable' || !Array.isArray(fleet.data.hosts))
302
171
  throw new Error(`Fleet read unavailable (${fleet.reason ?? 'invalid_response'}); use an admin identity with read permission`);
303
- const [inventory, shared, diagnostics, events, history] = await Promise.all([
172
+ const [inventory, shared, diagnostics, events, history, jenkins] = await Promise.all([
304
173
  readSource(client, '/run-cloud/admin/fleet-inventory'),
305
174
  readSource(client, `${ROOT}/operator-inventory`),
306
175
  readSource(client, `${ROOT}/diagnostics?${query}`),
307
176
  readFleetEvents(client, options),
308
177
  options.hostId ? readSource(client, `${ROOT}/history?${new URLSearchParams({ range: options.range ?? '24h', hostId: options.hostId })}`) : Promise.resolve(null),
178
+ readSource(client, '/run-cloud/admin/jenkins'),
309
179
  ]);
310
- for (const [source, key] of [[inventory, 'hosts'], [shared, 'snapshots'], [diagnostics, 'diagnoses']]) {
180
+ for (const [source, key] of [[inventory, 'hosts'], [shared, 'snapshots'], [diagnostics, 'diagnoses'], [jenkins, 'nodes']]) {
311
181
  if (source.state === 'available' && !Array.isArray(source.data[key])) {
312
182
  source.state = 'unavailable';
313
183
  source.reason = 'invalid_response_shape';
314
184
  source.data = {};
315
185
  }
316
186
  }
187
+ if (jenkins.data.state === 'unavailable') {
188
+ jenkins.state = 'unavailable';
189
+ jenkins.reason = 'jenkins_collection_unavailable';
190
+ }
317
191
  if (history?.state === 'available' && !Array.isArray(history.data.points)) {
318
192
  history.state = 'unavailable';
319
193
  history.reason = 'invalid_response_shape';
@@ -326,7 +200,9 @@ export async function inspectFleet(client, options = {}, now) {
326
200
  const ids = new Set([...expected.keys(), ...live.keys(), ...supplemental.keys()]);
327
201
  if (options.hostId && !ids.has(options.hostId))
328
202
  throw new Error(`Host ${options.hostId} is absent from returned inventories`);
329
- const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => hostReport(live.get(id) ?? {}, expected.get(id) ?? {}, supplemental.get(id) ?? {}, rows(diagnostics.data.diagnoses), now, num(fleet.data.staleAfterSeconds) ?? 45, options.deep ? row(row(diagnostics.data.hostDetails)[id]) : {}));
203
+ const hosts = [...ids].filter(id => !options.hostId || id === options.hostId).sort().map(id => hostReport(live.get(id) ?? {}, expected.get(id) ?? {}, supplemental.get(id) ?? {}, rows(diagnostics.data.diagnoses), now, num(fleet.data.staleAfterSeconds) ?? 45, options.deep ? row(row(diagnostics.data.hostDetails)[id]) : {}, {
204
+ inventoryTtl: num(inventory.data.staleAfterSeconds) ?? 30, jenkins: jenkins.data,
205
+ }));
330
206
  const freshCpu = hosts.filter(host => host.cpu.state === 'fresh' && host.cpu.percent !== null);
331
207
  const codes = options.codes ?? [];
332
208
  const failureInputs = [...events.items.map(eventEvidence), ...(options.failures ?? []).map(failureEvidence)];
@@ -334,9 +210,9 @@ export async function inspectFleet(client, options = {}, now) {
334
210
  const failures = failureInputs.filter(event => selectedFailure(event, codes) && (!event.timestamp || (time(event.timestamp) >= start && time(event.timestamp) <= now)))
335
211
  .map(event => ({ ...correlate(event, rows(fleet.data.hosts), hosts, now), windowMembership: event.timestamp ? 'inside_requested_window' : 'unknown; original timestamp missing' }))
336
212
  .filter(event => !options.hostId || event.hostId === options.hostId || event.hostId === null);
337
- const sourceStatus = (source) => ({ state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt), meaning: 'response generation time; does not renew component observations' });
213
+ const sourceStatus = (source) => ({ state: source.state, reason: source.reason, generatedAt: stamp(source.data.generatedAt ?? source.data.observedAt), meaning: 'response generation time; does not renew component observations' });
338
214
  return { schemaVersion: 1, generatedAt: new Date(now).toISOString(), readOnly: true,
339
- sources: { fleet: sourceStatus(fleet), inventory: sourceStatus(inventory), shared: sourceStatus(shared), diagnostics: sourceStatus(diagnostics),
215
+ sources: { fleet: sourceStatus(fleet), inventory: sourceStatus(inventory), shared: sourceStatus(shared), diagnostics: sourceStatus(diagnostics), jenkins: sourceStatus(jenkins),
340
216
  ...(history ? { history: sourceStatus(history) } : {}) },
341
217
  summary: { hosts: hosts.length, canonicalInventoryHosts: inventory.state === 'available' ? expected.size : null, liveHosts: live.size,
342
218
  healthCounts: hosts.reduce((counts, host) => { const state = host.health.effectiveState; counts[state] = (counts[state] ?? 0) + 1; return counts; }, { ready: 0, degraded: 0, unreachable: 0, unknown: 0 }),
@@ -421,6 +297,14 @@ function printReport(report, json) {
421
297
  }
422
298
  export function registerRunCloudFleet(program) {
423
299
  const fleet = program.command('fleet').description('Read-only admin fleet telemetry, diagnosis, events and original failure evidence');
300
+ fleet.command('survey [hostId]').description('Survey all fleet signals in seconds; authenticated admin reads, no local services')
301
+ .option('--json', 'Structured metrics, diagnosis, observations and coverage')
302
+ .option('--range <range>', 'Alert/input-lag window: 1h, 6h, 24h, 7d', '24h')
303
+ .action((hostId, opts) => action(async () => {
304
+ const credentials = requireCredentials(), apiOrigin = new URL(credentials.apiUrl).origin;
305
+ const report = await surveyFleet(new ApiClient(credentials.apiUrl, credentials.token), { hostId, range: opts.range });
306
+ console.log(opts.json ? JSON.stringify({ ...report, apiOrigin }, null, 2) : `${apiOrigin}\n${surveyText(report)}`);
307
+ }));
424
308
  fleet.command('inspect [hostId]').description('Inspect whole fleet or one machine; existing admin read authorization required')
425
309
  .option('--json', 'Structured evidence with source observations and missing data')
426
310
  .option('--range <range>', 'Event/history window', '24h').option('--limit <rows>', 'Events per page (1..500)', '200')
@@ -0,0 +1,136 @@
1
+ export function processFreshness(scan, now) {
2
+ if (!scan?.observedAt)
3
+ return 'never observed';
4
+ if (scan.sourceUnavailable)
5
+ return 'source unavailable';
6
+ if (scan.error)
7
+ return 'collection failed';
8
+ const timestamp = Date.parse(scan.observedAt);
9
+ if (!Number.isFinite(timestamp) || timestamp > now + 5000 || now - timestamp > scan.staleAfterSeconds * 1000)
10
+ return 'stale';
11
+ return 'fresh';
12
+ }
13
+ export function leaseStateLabel(state) {
14
+ return { accepting: 'Lease gate open', rejecting: 'Lease gate closed', unknown: 'Lease gate unknown', held: 'Operator hold' }[state] ?? state;
15
+ }
16
+ export function operatorHeld(host) {
17
+ return [host.fence, host.requestedFence].some(fence => fence && ['experiment', 'maintenance'].includes(fence.kind));
18
+ }
19
+ export function hostLeaseState(host) {
20
+ if (operatorHeld(host))
21
+ return 'held';
22
+ return host.acceptsLeases === true ? 'accepting' : host.acceptsLeases === false ? 'rejecting' : 'unknown';
23
+ }
24
+ export function hostLeaseStatus(host) {
25
+ const gate = leaseStateLabel(host.acceptsLeases === true ? 'accepting' : host.acceptsLeases === false ? 'rejecting' : 'unknown');
26
+ return operatorHeld(host) ? `Operator hold · ${gate.toLowerCase()}` : gate;
27
+ }
28
+ export function redactSensitiveText(value) {
29
+ if (!value)
30
+ return value ?? '';
31
+ return value
32
+ .replace(/\b(?:rc_live_|rcsf_|rc_dev_)[A-Za-z0-9_.-]+/g, '[redacted]')
33
+ .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+/g, '[redacted]')
34
+ .replace(/(\bBearer\s+)[A-Za-z0-9._~+/=-]+/gi, '$1[redacted]')
35
+ .replace(/([?&](?:token|authorization|secret|credential|signature|api[_-]?key|x-amz-(?:credential|signature)|x-goog-(?:credential|signature))=)[^\s&#]+/gi, '$1[redacted]')
36
+ .replace(/((?:token|authorization|secret|credential|signature|api[_-]?key)\s*[:=]\s*)[^\s,;}]+/gi, '$1[redacted]');
37
+ }
38
+ const ready = (state) => ['clean', 'ready', 'healthy', 'serving', 'connected'].includes(state);
39
+ const responding = (state) => ready(state) || state === 'busy';
40
+ export const observationCurrent = (at, now, seconds) => {
41
+ const time = at ? Date.parse(at) : NaN;
42
+ return Number.isFinite(time) && time <= now + 5000 && now - time <= seconds * 1000;
43
+ };
44
+ export function jenkinsObservedState(node, at, now, seconds = 30, unavailable = false) {
45
+ if (unavailable || !node || !at || !Number.isFinite(Date.parse(at)) || Date.parse(at) > now + 5000)
46
+ return 'unknown';
47
+ if (!observationCurrent(at, now, seconds))
48
+ return 'stale';
49
+ return node.offline === true ? 'offline' : node.offline === false ? 'ready' : 'unknown';
50
+ }
51
+ export function machineDiagnosis(host, hosted, observedState, now, staleAfterSeconds = 120) {
52
+ const faults = [], findings = [], checks = [];
53
+ let status = ready(observedState) ? 'ready' : observedState;
54
+ const observed = observationCurrent(host.observedAt, now, staleAfterSeconds);
55
+ const serviceCurrent = !['stale', 'unknown', 'inspection-unavailable'].includes(observedState) && observed;
56
+ if (host.observedAt && !observed && status !== 'unknown')
57
+ status = 'stale';
58
+ if (!host.observedAt) {
59
+ status = 'unknown';
60
+ checks.push('service never observed');
61
+ }
62
+ else if (!serviceCurrent)
63
+ checks.push('service checks stale or unavailable; last state retained');
64
+ else if (observedState === 'unreachable')
65
+ checks.push('service unreachable from collector; cause unconfirmed');
66
+ else if (['down', 'runtime-unavailable', 'agent-running-no-health', 'offline'].includes(observedState))
67
+ faults.push(`service reports ${observedState}`);
68
+ const release = hosted?.release;
69
+ const drift = release?.matches === false && [
70
+ [release.expectedPackage, release.actualPackage], [release.expectedVersion, release.actualVersion], [release.expectedBuild, release.actualBuild],
71
+ ].some(([expected, actual]) => expected && actual && expected !== actual);
72
+ if (drift)
73
+ findings.push(`release drift: ${release.actualVersion ?? 'reported artifact'}; expected ${release.expectedVersion ?? 'configured artifact'}`);
74
+ else if (release?.matches !== true && host.role !== 'jenkins')
75
+ checks.push('release match not verified');
76
+ if (observedState === 'degraded' && !drift)
77
+ checks.push('reported degradation has no confirmed cause');
78
+ const sample = host.hostDiagnostics;
79
+ if (!sample)
80
+ checks.push('detailed service checks missing');
81
+ else if (sample.collectionError || !observationCurrent(sample.observedAt, now, sample.staleAfterSeconds))
82
+ checks.push('detailed service checks stale or unavailable');
83
+ else {
84
+ const service = sample.services.find(row => row.name === 'serve-sim');
85
+ if (host.role === 'simulator' && service && ['failed', 'stopped', 'exited', 'disabled'].includes(service.state ?? ''))
86
+ faults.push(`serve-sim is ${service.state}`);
87
+ if (host.role === 'simulator' && sample.packageMatchesService === false)
88
+ faults.push('service executable differs from installed package');
89
+ if (sample.power.thermalWarning || sample.power.performanceWarning)
90
+ findings.push('host reports thermal or performance warnings');
91
+ if ((sample.coreSimulator.orphaned ?? 0) > 0)
92
+ findings.push(`${sample.coreSimulator.orphaned} orphaned CoreSimulator bootstrap(s)`);
93
+ if ((sample.coreSimulator.externalBooted ?? 0) > 0)
94
+ findings.push(`${sample.coreSimulator.externalBooted} external booted simulator(s) consume memory`);
95
+ if ((sample.recording.failed ?? 0) > 0)
96
+ checks.push(`${sample.recording.failed} retained recording failure(s); current service scored separately`);
97
+ }
98
+ if (serviceCurrent && (host.capacity?.dirty ?? 0) > 0)
99
+ faults.push(`${host.capacity.dirty} dirty pool record(s)`);
100
+ const scan = host.processScan;
101
+ if (!scan)
102
+ checks.push('process checks missing');
103
+ else {
104
+ const freshness = processFreshness(scan, now);
105
+ const malicious = scan.iocs.some(ioc => ['present', 'active'].includes(ioc.status)) || scan.findings.some(finding => finding.reason === 'known_ioc');
106
+ if (malicious)
107
+ findings.unshift(`known malicious daemon detected${freshness === 'fresh' ? '' : ' in retained evidence; recheck required'}`);
108
+ else if (scan.status === 'critical')
109
+ findings.unshift('critical process finding requires review');
110
+ const review = scan.findings.filter(finding => ['rooted_workload', 'long_lived_workload', 'identity_changed', 'unreviewed_persistence'].includes(finding.reason));
111
+ if (review.length)
112
+ findings.push(`${review.length} process or persistence finding(s) need review`);
113
+ if (scan.iocs.some(ioc => ioc.contained))
114
+ findings.push('quarantined daemon evidence is contained');
115
+ if (freshness !== 'fresh' || scan.error)
116
+ checks.push(`process checks ${scan.error ?? freshness}`);
117
+ else if (!scan.root || scan.coverage.iocs === 'partial' || scan.iocs.some(ioc => ioc.status === 'unknown'))
118
+ checks.push('process checks incomplete; privileged visibility not established');
119
+ if (scan.baseline.state !== 'reviewed')
120
+ checks.push('process baseline not enrolled');
121
+ if (scan.status === 'warning' && !review.length && !malicious)
122
+ checks.push('process checks warning; see scanner details');
123
+ }
124
+ const actionable = findings.filter(finding => !finding.startsWith('quarantined'));
125
+ if (faults.length || actionable.length)
126
+ status = 'degraded';
127
+ const parts = [`${host.id}: ${status}.`,
128
+ faults.length ? `Service faults: ${faults.join('; ')}.` : serviceCurrent && (responding(observedState) || (observedState === 'degraded' && drift)) ? `Service responding${observedState === 'busy' ? '; capacity busy' : ''}.` : 'Current service health unverified.',
129
+ findings.length ? `Findings: ${findings.join('; ')}.` : '',
130
+ `Admission: ${hostLeaseStatus(host)}${host.fence ? `; ${host.fence.state ?? 'unconfirmed'} ${host.fence.kind} fence — ${host.fence.reason}` : ''}.`,
131
+ checks.length ? `Checks: ${checks.join('; ')}.` : 'Checks current.',
132
+ faults.length ? 'Next: inspect retained service diagnostics.' : actionable.length ? 'Next: review the release or scanner evidence.' : checks.length ? 'Next: confirm missing checks with read-only collection.' : '',
133
+ ];
134
+ return { status, summary: redactSensitiveText(parts.filter(Boolean).join(' ')),
135
+ faults: faults.map(redactSensitiveText), findings: findings.map(redactSensitiveText), checks: checks.map(redactSensitiveText) };
136
+ }
package/dist/version.js CHANGED
@@ -1 +1 @@
1
- export const CLI_VERSION = '0.2.166';
1
+ export const CLI_VERSION = '0.2.168';
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "runcloud",
3
- "version": "0.2.166",
3
+ "version": "0.2.168",
4
4
  "description": "Create and control run.cloud remote mobile simulators and cloud sandboxes",
5
5
  "license": "Apache-2.0",
6
6
  "keywords": [
Binary file
Binary file