querysub 0.715.0 → 0.717.0

package/bin/service-configs-pull.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/serviceConfigsPullMain");

package/bin/service-configs-push.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/deployManager/serviceConfigsPushMain");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.715.0",
+  "version": "0.717.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -56,6 +56,8 @@
     "audit-imports": "./bin/audit-imports.js",
     "audit-disk-values": "./bin/audit-disk-values.js",
     "emergency-update": "./bin/emergency-update.js",
+    "service-configs-pull": "./bin/service-configs-pull.js",
+    "service-configs-push": "./bin/service-configs-push.js",
     "mcp-indexed-logs": "./bin/mcp-indexed-logs.js",
     "autofix": "./bin/autofix.js",
     "storageserve": "./bin/storageserve.js"

package/src/-a-archives/archives2.ts

@@ -91,7 +91,7 @@ function archiveBuilder(bucket: string, overrides: Partial<RemoteConfigBase>) {
     ]);
 
     return createArchives({
-        version: 29,
+        version: 30,
         sources,
     });
 }
@@ -180,7 +180,7 @@ export function getArchives2Public(folder: string) {
 export function getArchives2Debug(folder: string) {
     let routeVersion = 1;
     let archives = createArchives({
-        version: 20,
+        version: 21,
         sources: [
             // IMPORTANT! If we start changing around routes, it could resurrect old data. We can fix this via incrementing the route version.
             {

package/src/deployManager/serviceConfigsDisk.ts

@@ -0,0 +1,17 @@
+import { STORAGE_DIR } from "../misc/appPaths";
+import { ServiceConfig } from "./machineSchema";
+
+export const SERVICE_CONFIGS_DIR = STORAGE_DIR + "service-configs/";
+
+export function getServiceConfigFilePath(config: ServiceConfig): string {
+    let safeKey = config.parameters.key.replaceAll(/[^a-zA-Z0-9_-]/g, "_");
+    return `${SERVICE_CONFIGS_DIR}${safeKey}-${config.serviceId}.json`;
+}
+
+export function getDeployComparisonKey(config: ServiceConfig): string {
+    return JSON.stringify({
+        parameters: { ...config.parameters, releaseTime: undefined, overlapTime: undefined },
+        title: config.info.title,
+        notes: config.info.notes,
+    });
+}

package/src/deployManager/serviceConfigsPullMain.ts

@@ -0,0 +1,40 @@
+import "../inject";
+
+import fs from "fs";
+import path from "path";
+import { pathToFileURL } from "url";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { green } from "socket-function/src/formatting/logColors";
+import { MachineServiceControllerBase } from "./machineSchema";
+import { SERVICE_CONFIGS_DIR, getServiceConfigFilePath } from "./serviceConfigsDisk";
+
+async function main() {
+    disablePathAuditer();
+
+    let controller = new MachineServiceControllerBase();
+    let serviceIds = await controller.getServiceList();
+    await fs.promises.mkdir(SERVICE_CONFIGS_DIR, { recursive: true });
+
+    let existingFiles = new Set((await fs.promises.readdir(SERVICE_CONFIGS_DIR)).map(file => path.resolve(SERVICE_CONFIGS_DIR, file)));
+    let written = 0;
+    for (let serviceId of serviceIds) {
+        let config = await controller.getServiceConfig(serviceId);
+        if (!config) {
+            console.warn(`Service ${serviceId} is listed, but has no config, skipping it`);
+            continue;
+        }
+        let filePath = getServiceConfigFilePath(config);
+        existingFiles.delete(path.resolve(filePath));
+        await fs.promises.writeFile(filePath, JSON.stringify(config, undefined, 4));
+        console.log(`Wrote ${config.parameters.key} (${serviceId}) to ${pathToFileURL(filePath).href}`);
+        written++;
+    }
+    for (let staleFile of existingFiles) {
+        await fs.promises.unlink(staleFile);
+        console.log(`Removed ${pathToFileURL(staleFile).href}, as its service no longer exists`);
+    }
+    console.log(green(`Pulled ${written} service configs into ${pathToFileURL(path.resolve(SERVICE_CONFIGS_DIR)).href}. Edit them, then run service-configs-push.`));
+}
+
+main().catch(logErrors).finally(() => process.exit());

package/src/deployManager/serviceConfigsPushMain.ts

@@ -0,0 +1,63 @@
+import "../inject";
+
+import fs from "fs";
+import yargs from "yargs";
+import { pathToFileURL } from "url";
+import { logErrors } from "../errors";
+import { disablePathAuditer } from "../diagnostics/pathAuditerCallback";
+import { delay } from "socket-function/src/batching";
+import { formatDateTime, formatTime } from "socket-function/src/formatting/format";
+import { green, magenta } from "socket-function/src/formatting/logColors";
+import { DEFAULT_OVERLAP_TIME, MachineServiceControllerBase, ServiceConfig } from "./machineSchema";
+import { SERVICE_CONFIGS_DIR, getDeployComparisonKey } from "./serviceConfigsDisk";
+
+const NOTIFY_GRACE_PERIOD = 10 * 1000;
+
+let args = yargs(process.argv)
+    .option("immediate", { type: "boolean", default: false, desc: "Deploy with no overlap, so the old instances are killed right away instead of running out the default overlap" })
+    .argv as { immediate: boolean };
+
+async function main() {
+    disablePathAuditer();
+
+    let controller = new MachineServiceControllerBase();
+    let files = (await fs.promises.readdir(SERVICE_CONFIGS_DIR)).filter(file => file.endsWith(".json"));
+    let now = Date.now();
+    let overlapTime = args.immediate ? 0 : DEFAULT_OVERLAP_TIME;
+
+    let changed: ServiceConfig[] = [];
+    for (let file of files) {
+        let filePath = SERVICE_CONFIGS_DIR + file;
+        let contents = await fs.promises.readFile(filePath, "utf8");
+        let config = JSON.parse(contents) as ServiceConfig;
+        if (!config.serviceId) {
+            throw new Error(`Expected a serviceId in ${pathToFileURL(filePath).href}, was ${JSON.stringify(config.serviceId)}. Contents: ${contents.slice(0, 500)}`);
+        }
+        let current = await controller.getServiceConfig(config.serviceId);
+        if (!current) {
+            throw new Error(`Expected service ${config.serviceId} (from ${pathToFileURL(filePath).href}) to exist, it does not. Run service-configs-pull to resync the folder.`);
+        }
+        let before = getDeployComparisonKey(current);
+        let after = getDeployComparisonKey(config);
+        if (before === after) continue;
+        console.log(magenta(`${config.parameters.key} (${config.serviceId}) changed`));
+        console.log(`    before: ${before}`);
+        console.log(`    after:  ${after}`);
+        config.parameters.releaseTime = now;
+        config.parameters.overlapTime = overlapTime;
+        changed.push(config);
+    }
+
+    if (changed.length === 0) {
+        console.log(green(`None of the ${files.length} service configs in ${pathToFileURL(SERVICE_CONFIGS_DIR).href} differ from what is deployed, nothing to push`));
+        return;
+    }
+
+    console.log(`Deploying ${changed.length} of ${files.length} service configs, releasing at ${formatDateTime(now)} with ${overlapTime ? `an overlap of ${formatTime(overlapTime)}` : "no overlap"}`);
+    await controller.setServiceConfigs(changed);
+    console.log(`Wrote ${changed.length} service configs, waiting for machine notifications to go out`);
+    await delay(NOTIFY_GRACE_PERIOD);
+    console.log(green(`Pushed ${changed.map(config => config.parameters.key).join(", ")}`));
+}
+
+main().catch(logErrors).finally(() => process.exit());