querysub 0.605.0 → 0.607.0

package/src/deployManager/components/storage/storageController.ts

@@ -0,0 +1,76 @@
+import { SocketFunction } from "socket-function/SocketFunction";
+import { listServerBuckets, clearServerWriteStats, activateServerBucket, getServerActiveBucket } from "sliftutils/storage/remoteStorage/createArchives";
+import type { ServerBucketInfo, ActiveBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
+import { parseStorageUrl, authenticateStorage } from "sliftutils/storage/remoteStorage/ArchivesRemote";
+import { RemoteStorageController, STORAGE_NOT_AUTHENTICATED } from "sliftutils/storage/remoteStorage/storageController";
+import type { AccessTotals, AccessSummaryState } from "sliftutils/storage/remoteStorage/accessStats";
+import type { SummaryEntry } from "sliftutils/treeSummary";
+import { getSyncedController } from "../../../library-components/SyncedController";
+import { assertIsManagementUser } from "../../../diagnostics/managementPages";
+import { STORAGE_ACCOUNT } from "../../../-a-archives/archives2";
+
+// The registered controller and its synced wrapper have to keep a stable identity, so this file is a full-refresh boundary rather than a hot-reload one (matching SyncedController itself).
+module.hotreload = false;
+
+/** The access-stats endpoints have no client wrapper in sliftutils (a pinned dependency), so we reconnect and authenticate the same way its own callServer does. */
+async function callStorageServer<T>(url: string, run: (controller: typeof RemoteStorageController.nodes[string]) => Promise<T>): Promise<T> {
+    SocketFunction.ENABLE_CLIENT_MODE = true;
+    let parsed = parseStorageUrl(url);
+    let nodeId = SocketFunction.connect({ address: parsed.address, port: parsed.port });
+    let controller = RemoteStorageController.nodes[nodeId];
+    try {
+        return await run(controller);
+    } catch (e) {
+        if (!String((e as Error).stack ?? e).includes(STORAGE_NOT_AUTHENTICATED)) throw e;
+        await authenticateStorage({ address: parsed.address, port: parsed.port, nodeId });
+        return await run(controller);
+    }
+}
+
+/** Every storage call is proxied through here: they authenticate with this machine's identity CA, which the browser has no access to. */
+class StoragePageControllerBase {
+    /** One call per server, so each caches and refreshes on its own. */
+    public async getServerBuckets(url: string): Promise<ServerBucketInfo[]> {
+        return await listServerBuckets({ url, account: STORAGE_ACCOUNT });
+    }
+    public async activateBucket(url: string, bucketName: string): Promise<ActiveBucketInfo | string> {
+        return await activateServerBucket({ url, account: STORAGE_ACCOUNT, bucketName });
+    }
+    public async getActiveBucket(url: string, bucketName: string): Promise<ActiveBucketInfo | string> {
+        return await getServerActiveBucket({ url, account: STORAGE_ACCOUNT, bucketName });
+    }
+    public async clearWriteStats(url: string): Promise<{ clearedBuckets: number }> {
+        return await clearServerWriteStats({ url, account: STORAGE_ACCOUNT });
+    }
+    /** Per-operation totals since startup (or the last reset), aggregated across every bucket in the account. */
+    public async getAccessStats(url: string): Promise<AccessTotals> {
+        return await callStorageServer(url, controller => controller.getAccessStats({ account: STORAGE_ACCOUNT }));
+    }
+    /** The path breakdown of one operation. weightBySize orders it (and the totals) by bytes rather than call count. */
+    public async getAccessSummaries(url: string, operation: string, maxCount: number, weightBySize: boolean): Promise<SummaryEntry<AccessSummaryState>[]> {
+        return await callStorageServer(url, controller => controller.getAccessSummaries({ account: STORAGE_ACCOUNT, operation, maxCount, weightBySize }));
+    }
+}
+
+export const StoragePageController = SocketFunction.register(
+    "StoragePageController-4f1c93ab-77e2-4d15-9a30-1c6b8f5d2e04",
+    new StoragePageControllerBase(),
+    () => ({
+        getServerBuckets: {},
+        activateBucket: {},
+        getActiveBucket: {},
+        clearWriteStats: {},
+        getAccessStats: {},
+        getAccessSummaries: {},
+    }),
+    () => ({
+        hooks: [assertIsManagementUser],
+    }),
+    {
+    }
+);
+
+export const StorageSynced = getSyncedController(StoragePageController, {
+    reads: {},
+    writes: {},
+});

package/src/deployManager/components/storage/storageServers.ts

@@ -0,0 +1,73 @@
+import type { ServerBucketInfo } from "sliftutils/storage/remoteStorage/storageServerState";
+import { getLiveServiceParameters, getMachineTargets, applyCommandTemplate, ServiceParameters, ServiceConfig, DEFAULT_OVERLAP_TIME } from "../../machineSchema";
+import { STORAGE_COMMAND_PREFIX } from "../../serviceCategories";
+
+module.hotreload = true;
+
+const URL_ARG_REGEX = /--url\s+(\S+)/g;
+
+/** The release a storage service is in the middle of. Read straight from the service configs, so it resolves without contacting any storage server - which is exactly when it matters most, since a deploy is the usual reason the servers don't answer. */
+export type StorageService = {
+    serviceKey: string;
+    serviceTitle: string;
+    releaseTime?: number;
+    overlapTime: number;
+    hasOldParameters: boolean;
+};
+
+export type StorageServer = StorageService & {
+    url: string;
+};
+
+export type StorageServerBuckets = StorageServer & {
+    /** Still waiting on this server's own endpoint - each server loads independently */
+    loading?: boolean;
+    buckets?: ServerBucketInfo[];
+    /** This server could not be read at all. Kept per server rather than thrown, so one bad server costs its own row and nothing else. */
+    error?: string;
+};
+
+function getStorageUrls(parameters: ServiceParameters): string[] {
+    if (!parameters.command.trimStart().startsWith(STORAGE_COMMAND_PREFIX)) return [];
+    let urls: string[] = [];
+    for (let target of getMachineTargets(parameters)) {
+        let command = applyCommandTemplate(parameters.command, target.variables);
+        for (let match of command.matchAll(URL_ARG_REGEX)) {
+            urls.push(match[1]);
+        }
+    }
+    return urls;
+}
+
+/** The storage servers a set of service configs runs, worked out on the client - it already has the configs, so asking the server to re-derive them would just make the whole page wait on one uncacheable call. */
+export function getStorageServers(configs: ServiceConfig[]): StorageServer[] {
+    let serviceByUrl = new Map<string, StorageServer>();
+    for (let config of configs) {
+        let allParameters = [getLiveServiceParameters(config)];
+        for (let parameters of allParameters) {
+            if (!parameters) continue;
+            for (let url of getStorageUrls(parameters)) {
+                if (serviceByUrl.has(url)) continue;
+                serviceByUrl.set(url, {
+                    url,
+                    serviceKey: parameters.key,
+                    serviceTitle: config.info.title,
+                    releaseTime: config.parameters.releaseTime,
+                    // The overlap always comes from the newest parameters, even though it governs how long the old instances outlive the release
+                    overlapTime: config.parameters.overlapTime ?? DEFAULT_OVERLAP_TIME,
+                    hasOldParameters: !!config.oldParameters,
+                });
+            }
+        }
+    }
+    return [...serviceByUrl.values()];
+}
+
+export function getStorageServices(servers: StorageServer[]): StorageService[] {
+    let byKey = new Map<string, StorageService>();
+    for (let server of servers) {
+        if (byKey.has(server.serviceKey)) continue;
+        byKey.set(server.serviceKey, server);
+    }
+    return [...byKey.values()];
+}

package/src/deployManager/machineSchema.ts

@@ -29,6 +29,16 @@ export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
 export const MACHINE_RESYNC_INTERVAL = timeInMinute * 15;
 export const SERVICE_NODE_FILE_NAME = "serviceNodeId.txt";
 
+
+export type MachineConfig = {
+    machineId: string;
+    disabled: boolean;
+};
+// We want all of these to have very high availability. Even if they're wrong, it's better to have an old service deployed or even view information on an old service, or even set information only in back plays rather than have the server be completely down. 
+export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"), { fallbacks: true });
+export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"), { fallbacks: true });
+export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"), { fallbacks: true });
+
 export type MachineInfo = {
     machineId: string;
 
@@ -182,13 +192,6 @@ function normalizeServiceConfig(config: ServiceConfig): ServiceConfig {
     return config;
 }
 
-export type MachineConfig = {
-    machineId: string;
-    disabled: boolean;
-};
-export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"));
-export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"));
-export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"));
 
 export type LaunchRecord = {
     serviceId: string;
@@ -276,7 +279,7 @@ export class MachineServiceControllerBase {
 
     public async getMachineList() {
         return [...new Set([
-            ...(await machineInfos.keys({ fallbacks: true })),
+            ...(await machineInfos.keys()),
         ])];
     }
     public async deleteMachineIds(machineIds: string[]) {
@@ -303,7 +306,7 @@ export class MachineServiceControllerBase {
     }
     public async getServiceList(): Promise<string[]> {
         return [...new Set([
-            ...(await serviceConfigs.keys({ fallbacks: true })),
+            ...(await serviceConfigs.keys()),
         ])];
     }
     public async getServiceConfig(serviceId: string): Promise<ServiceConfig | undefined> {
@@ -312,7 +315,7 @@ export class MachineServiceControllerBase {
     }
 
     public async getMachineConfigList() {
-        return await machineConfigs.values({ fallbacks: true });
+        return await machineConfigs.values();
     }
     public async getMachineConfig(machineId: string): Promise<MachineConfig | undefined> {
         return await machineConfigs.get(machineId);
@@ -378,7 +381,7 @@ export class MachineServiceControllerBase {
             // NOTE: This is correct. This is obviously correct. For some reason, the AI wrote some really big, complicated thing here. I don't know why. This is the only way it should be done, obviously.
             config.oldParameters = getLiveServiceParameters(serviceConfig);
 
-            await serviceConfigs.set(serviceId, config, { fallbacks: true });
+            await serviceConfigs.set(serviceId, config);
             // Only notify we were or are deployed. If it's not deployed, this will be ignored anyways.
             if (config.parameters.deploy || serviceConfig.parameters.deploy) {
                 for (let machineId of getConfigMachineIds(config)) {

package/src/library-components/UsageBar.tsx

@@ -24,13 +24,16 @@ export class UsageBar extends qreact.Component<{
     label: string;
     value: number;
     max: number;
+    /** Appended after each formatted number, so byte bars can read "5.2GB" instead of a bare "5.2G". */
+    unit?: string;
     /** Fraction of max at which the bar fill turns yellow. */
     warningThreshold?: number;
     /** Fraction of max at which the bar fill turns red and flashes. */
     errorThreshold?: number;
 }> {
     render() {
-        let { label, value, max, warningThreshold, errorThreshold } = this.props;
+        let { label, value, max, unit, warningThreshold, errorThreshold } = this.props;
+        let suffix = unit || "";
         let fraction = max && value / max || 0;
         let isError = errorThreshold !== undefined && fraction >= errorThreshold;
         let isWarning = !isError && warningThreshold !== undefined && fraction >= warningThreshold;
@@ -42,7 +45,7 @@ export class UsageBar extends qreact.Component<{
                 + (isError && (" " + FLASH_CLASS_NAME) || "")
             } />
             <div className={css.relative}>
-                {label} ({formatNumber(value)} / {formatNumber(max)})
+                {label} ({formatNumber(value)}{suffix} / {formatNumber(max)}{suffix})
             </div>
             {isError && <style>{`
                 @keyframes ${FLASH_CLASS_NAME}-anim {