querysub 0.716.0 → 0.718.0

package/bin/service-configs-pull.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/serviceConfigsPullMain");

package/bin/service-configs-push.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/serviceConfigsPushMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.716.0",
+  "version": "0.718.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -56,6 +56,8 @@
     "audit-imports": "./bin/audit-imports.js",
     "audit-disk-values": "./bin/audit-disk-values.js",
     "emergency-update": "./bin/emergency-update.js",
+    "service-configs-pull": "./bin/service-configs-pull.js",
+    "service-configs-push": "./bin/service-configs-push.js",
     "mcp-indexed-logs": "./bin/mcp-indexed-logs.js",
     "autofix": "./bin/autofix.js",
     "storageserve": "./bin/storageserve.js"

package/src/-a-archives/archives2.ts

@@ -8,6 +8,10 @@ import { FindConfig } from "sliftutils/storage/IArchives";
 
 export const STORAGE_ACCOUNT = "root";
 
+const STORE_NAME_HOSTNAMES: { [hostname: string]: string | undefined } = {
+    "99-255-70-23.querysubtest.com": "99-250-124-91.querysubtest.com",
+};
+
 
 function createSourceWindows(
     overrides: Partial<RemoteConfigBase>,
@@ -60,7 +64,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
                     url: currentUrls[i],
                     route: [routeStart, routeEnd],
                     // IMPORTANT! name decides where our files are stored. We can't have two things writing to the same folder at the same time. But it also means if we reuse the name, then it has to be the same data. 
-                    name: url.hostname.replaceAll(/[^a-z0-9-]/g, "_") + "_v" + config.routeVersion + "_" + routeStart + "_" + routeEnd,
+                    name: (STORE_NAME_HOSTNAMES[url.hostname] || url.hostname).replaceAll(/[^a-z0-9-]/g, "_") + "_v" + config.routeVersion + "_" + routeStart + "_" + routeEnd,
                 });
             }
         }
@@ -91,7 +95,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 30,
+        version: 31,
         sources,
     });
 }

package/src/deployManager/serviceConfigsDisk.ts

@@ -0,0 +1,17 @@
+import { STORAGE_DIR } from "../misc/appPaths";
+import { ServiceConfig } from "./machineSchema";
+
+export const SERVICE_CONFIGS_DIR = STORAGE_DIR + "service-configs/";
+
+export function getServiceConfigFilePath(config: ServiceConfig): string {
+    let safeKey = config.parameters.key.replaceAll(/[^a-zA-Z0-9_-]/g, "_");
+    return `${SERVICE_CONFIGS_DIR}${safeKey}-${config.serviceId}.json`;
+}
+
+export function getDeployComparisonKey(config: ServiceConfig): string {
+    return JSON.stringify({
+        parameters: { ...config.parameters, releaseTime: undefined, overlapTime: undefined },
+        title: config.info.title,
+        notes: config.info.notes,
+    });
+}

package/src/deployManager/serviceConfigsPullMain.ts

@@ -0,0 +1,40 @@
+import "../inject";
+
+import fs from "fs";
+import path from "path";
+import { pathToFileURL } from "url";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { green } from "socket-function/src/formatting/logColors";
+import { MachineServiceControllerBase } from "./machineSchema";
+import { SERVICE_CONFIGS_DIR, getServiceConfigFilePath } from "./serviceConfigsDisk";
+
+async function main() {
+    disablePathAuditer();
+
+    let controller = new MachineServiceControllerBase();
+    let serviceIds = await controller.getServiceList();
+    await fs.promises.mkdir(SERVICE_CONFIGS_DIR, { recursive: true });
+
+    let existingFiles = new Set((await fs.promises.readdir(SERVICE_CONFIGS_DIR)).map(file => path.resolve(SERVICE_CONFIGS_DIR, file)));
+    let written = 0;
+    for (let serviceId of serviceIds) {
+        let config = await controller.getServiceConfig(serviceId);
+        if (!config) {
+            console.warn(`Service ${serviceId} is listed, but has no config, skipping it`);
+            continue;
+        }
+        let filePath = getServiceConfigFilePath(config);
+        existingFiles.delete(path.resolve(filePath));
+        await fs.promises.writeFile(filePath, JSON.stringify(config, undefined, 4));
+        console.log(`Wrote ${config.parameters.key} (${serviceId}) to ${pathToFileURL(filePath).href}`);
+        written++;
+    }
+    for (let staleFile of existingFiles) {
+        await fs.promises.unlink(staleFile);
+        console.log(`Removed ${pathToFileURL(staleFile).href}, as its service no longer exists`);
+    }
+    console.log(green(`Pulled ${written} service configs into ${pathToFileURL(path.resolve(SERVICE_CONFIGS_DIR)).href}. Edit them, then run service-configs-push.`));
+}
+
+main().catch(logErrors).finally(() => process.exit());

package/src/deployManager/serviceConfigsPushMain.ts

@@ -0,0 +1,63 @@
+import "../inject";
+
+import fs from "fs";
+import yargs from "yargs";
+import { pathToFileURL } from "url";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { delay } from "socket-function/src/batching";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+import { DEFAULT_OVERLAP_TIME, MachineServiceControllerBase, ServiceConfig } from "./machineSchema";
+import { SERVICE_CONFIGS_DIR, getDeployComparisonKey } from "./serviceConfigsDisk";
+
+const NOTIFY_GRACE_PERIOD = 10 * 1000;
+
+let args = yargs(process.argv)
+    .option("immediate", { type: "boolean", default: false, desc: "Deploy with no overlap, so the old instances are killed right away instead of running out the default overlap" })
+    .argv as { immediate: boolean };
+
+async function main() {
+    disablePathAuditer();
+
+    let controller = new MachineServiceControllerBase();
+    let files = (await fs.promises.readdir(SERVICE_CONFIGS_DIR)).filter(file => file.endsWith(".json"));
+    let now = Date.now();
+    let overlapTime = args.immediate ? 0 : DEFAULT_OVERLAP_TIME;
+
+    let changed: ServiceConfig[] = [];
+    for (let file of files) {
+        let filePath = SERVICE_CONFIGS_DIR + file;
+        let contents = await fs.promises.readFile(filePath, "utf8");
+        let config = JSON.parse(contents) as ServiceConfig;
+        if (!config.serviceId) {
+            throw new Error(`Expected a serviceId in ${pathToFileURL(filePath).href}, was ${JSON.stringify(config.serviceId)}. Contents: ${contents.slice(0, 500)}`);
+        }
+        let current = await controller.getServiceConfig(config.serviceId);
+        if (!current) {
+            throw new Error(`Expected service ${config.serviceId} (from ${pathToFileURL(filePath).href}) to exist, it does not. Run service-configs-pull to resync the folder.`);
+        }
+        let before = getDeployComparisonKey(current);
+        let after = getDeployComparisonKey(config);
+        if (before === after) continue;
+        console.log(magenta(`${config.parameters.key} (${config.serviceId}) changed`));
+        console.log(`    before: ${before}`);
+        console.log(`    after:  ${after}`);
+        config.parameters.releaseTime = now;
+        config.parameters.overlapTime = overlapTime;
+        changed.push(config);
+    }
+
+    if (changed.length === 0) {
+        console.log(green(`None of the ${files.length} service configs in ${pathToFileURL(SERVICE_CONFIGS_DIR).href} differ from what is deployed, nothing to push`));
+        return;
+    }
+
+    console.log(`Deploying ${changed.length} of ${files.length} service configs, releasing at ${formatDateTime(now)} with ${overlapTime ? `an overlap of ${formatTime(overlapTime)}` : "no overlap"}`);
+    await controller.setServiceConfigs(changed);
+    console.log(`Wrote ${changed.length} service configs, waiting for machine notifications to go out`);
+    await delay(NOTIFY_GRACE_PERIOD);
+    console.log(green(`Pushed ${changed.map(config => config.parameters.key).join(", ")}`));
+}
+
+main().catch(logErrors).finally(() => process.exit());