querysub 0.604.0 → 0.606.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.604.0",
+  "version": "0.606.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -76,7 +76,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.83",
-    "socket-function": "^1.2.29",
+    "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-a-archives/archives2.ts

@@ -121,6 +121,7 @@ function nestBucket(folder: string, archives: ReturnType<typeof createArchives>)
             let changes = await archives.getChangesAfter2(config);
             return changes.filter(x => x.path.startsWith(folder)).map(x => ({ ...x, path: x.path.slice(folder.length) }));
         },
+        getSyncStatus: async () => await archives.getSyncStatus(),
     };
 }
 

package/src/-a-archives/archivesJSONT.ts

@@ -1,7 +1,8 @@
 import { lazy } from "socket-function/src/caching";
 import { Archives } from "./archives";
 import { green } from "socket-function/src/formatting/logColors";
-import { IArchives } from "sliftutils/storage/IArchives";
+import { FindConfig, IArchives } from "sliftutils/storage/IArchives";
+import { SetConfig } from "sliftutils/storage/IArchives";
 
 export type ArchiveT<T> = {
     get(key: string): Promise<T | undefined>;
@@ -13,7 +14,7 @@ export type ArchiveT<T> = {
     [Symbol.asyncIterator](): AsyncIterator<[string, T]>;
 };
 
-export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
+export function archiveJSONT<T>(archives: () => IArchives, config?: { fallbacks?: boolean }): ArchiveT<T> {
     archives = lazy(archives);
 
     let valuesCache = new Map<string, {
@@ -30,7 +31,7 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
     async function set(key: string, value: T) {
         let a = archives();
         console.log(`In archiveJSONT ${a.getDebugName()}, setting ${key} to ${JSON.stringify(value)}`);
-        await a.set(key, Buffer.from(JSON.stringify(value)));
+        await a.set(key, Buffer.from(JSON.stringify(value)), config);
     }
     async function deleteFnc(key: string) {
         let a = archives();
@@ -38,10 +39,10 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
         await a.del(key);
     }
     async function keys() {
-        return (await archives().find("")).map(value => value.toString());
+        return (await archives().find("", config)).map(value => value.toString());
     }
     async function values() {
-        let infos = await archives().findInfo("");
+        let infos = await archives().findInfo("", config);
 
         let needsUpdate = false;
         let currentKeys = new Set(infos.map(info => info.path));
@@ -63,7 +64,7 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
             let maxRetries = 10;
             let updated = false;
             for (let attempt = 0; attempt < maxRetries; attempt++) {
-                infos = await archives().findInfo("");
+                infos = await archives().findInfo("", config);
 
                 let newCache = new Map<string, {
                     createTime: number;
@@ -92,7 +93,7 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
 
                 if (!allFound) continue;
 
-                let newInfos = await archives().findInfo("");
+                let newInfos = await archives().findInfo("", config);
                 if (newInfos.length !== infos.length) continue;
                 function anyChanged() {
                     for (let i = 0; i < newInfos.length; i++) {

package/src/-d-trust/NetworkTrust2.ts

@@ -132,7 +132,7 @@ export const ensureWeAreTrusted = lazy(measureWrap(async () => {
     let machineKeyCert = getIdentityCA(getDomain());
     let machineId = getOwnMachineId(getDomain());
     if (!await archives().get(machineId)) {
-        await archives().set(machineId, machineKeyCert.cert);
+        await archives().set(machineId, machineKeyCert.cert, { fallbacks: true });
     }
 }));
 

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -308,7 +308,7 @@ export async function triggerNodeChange() {
 
 
 async function clearDeadThreadsFromArchives() {
-    let nodes = await archives().find("");
+    let nodes = await archives().find("", { fallbacks: true });
 
     function getPortHash(nodeId: string) {
         let obj = decodeNodeId(nodeId, getDomain());
@@ -329,7 +329,7 @@ async function clearDeadThreadsFromArchives() {
             let deadThreads = nodeIds.filter(nodeId => nodeId !== aliveNodeId);
             await Promise.all(deadThreads.map(async deadNodeId => {
                 console.log(`Removing dead thread. We contacted a node on the same port and same machine (${aliveNodeId}), which means the port has been reused by another thread, which proves that the old thread has died, as otherwise the new thread would not be able to use it.`);
-                await archives().del(deadNodeId);
+                await archives().del(deadNodeId, { fallbacks: true });
             }));
         }
     }
@@ -393,7 +393,7 @@ async function runHeartbeatAuditLoop() {
                 if (count >= DEAD_CHECK_COUNT) {
                     removedNodeIds.push(nodeId);
                     console.info(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
-                    await archives().del(nodeId);
+                    await archives().del(nodeId, { fallbacks: true });
                     deadCount.delete(nodeId);
                 } else {
                     console.info(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
@@ -497,7 +497,7 @@ async function writeHeartbeat() {
     let nodeId = getMountNodeId();
     console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
     if (!nodeId) return;
-    await archives().set(nodeId, Buffer.from(now + ""));
+    await archives().set(nodeId, Buffer.from(now + ""), { fallbacks: true });
 }
 
 async function runServerSyncLoops() {
@@ -636,7 +636,7 @@ if (isServer()) {
 
 
 export async function forceRemoveNode(nodeId: string) {
-    await archives().del(nodeId);
+    await archives().del(nodeId, { fallbacks: true });
     void tellEveryoneNodesChanges(`forceRemoveNode ${nodeId}`);
 }
 
@@ -646,7 +646,7 @@ export async function nodeDiscoveryShutdown() {
     console.log(red(`Shutting down node discovery`));
     shutdown = true;
     if (isServer()) {
-        await archives().del(getOwnNodeId());
+        await archives().del(getOwnNodeId(), { fallbacks: true });
     }
     void tellEveryoneNodesChanges("nodeDiscoveryShutdown");
 }

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -4,11 +4,11 @@ import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
-import { copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
+import { ArchivesSyncStatus, copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
-import { logErrors } from "../../errors";
+import { errorToUndefined, errorToUndefinedSilent, logErrors, timeoutToUndefined } from "../../errors";
 import { saveSnapshot } from "./archiveSnapshots";
 import { getNodeId } from "socket-function/src/nodeCache";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
@@ -35,6 +35,8 @@ const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
 const CONCURRENT_READ_COUNT = 128;
 const CONCURRENT_WRITE_COUNT = 128;
 const CONCURRENT_TRANSACTION_READS = 8;
+// When getFiles fails, how long we wait on the lock store's sync status before assuming the store is unreachable and falling back to non-authoritative (backblaze) reads.
+const FALLBACK_SYNC_STATUS_TIMEOUT = timeInSecond * 20;
 
 /** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
 async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
@@ -60,9 +62,9 @@ export function createArchiveLocker2(config: {
         debugKey,
         propagationTime: ARCHIVE_PROPAGATION_TIME,
 
-        async getKeys() {
-            let filesRawValues = await archiveValues.findInfo("", { type: "files" });
-            let filesRawLocks = await archiveLocks.findInfo("", { type: "files" });
+        async getKeys(config) {
+            let filesRawValues = await archiveValues.findInfo("", { type: "files", fallbacks: config?.fallbacks });
+            let filesRawLocks = await archiveLocks.findInfo("", { type: "files", fallbacks: config?.fallbacks });
             // Zero-byte files are deletes on new storage servers. Old storage servers still list them, so we filter here to make them behave the same.
             let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].filter(x => x.size > 0).map(x => ({
                 file: x.path,
@@ -71,14 +73,18 @@ export function createArchiveLocker2(config: {
             }));
             return files;
         },
+        async getSyncStatus() {
+            if (!archiveLocks.getSyncStatus) return undefined;
+            return archiveLocks.getSyncStatus();
+        },
 
         async setValue(key, value) {
             await getArchives(key).set(key, value);
             logNodeStats(`archives|Created TΔ`, formatNumber, 1);
         },
-        async getValue(key) {
-            // NOTE: I think we wanna not use fallbacks, Because we really want this to be atomic and safe. 
-            return getArchives(key).get(key, { noFallbacks: true });
+        async getValue(key, config) {
+            // Default to no fallbacks as we usually want this to be very safe. We don't want to apply transactions if the system isn't in a good state. 
+            return await getArchives(key).get(key, { noFallbacks: !config?.fallbacks });
         },
         async deleteKey(key) {
             let archives = await getArchives(key);
@@ -294,10 +300,12 @@ export type FileInfo = {
 type StorageType = {
     debugKey: string;
     propagationTime: number;
-    getKeys(): Promise<FileInfo[]>;
+    getKeys(config?: { fallbacks?: boolean }): Promise<FileInfo[]>;
     setValue(key: string, value: Buffer): Promise<void>;
-    getValue(key: string): Promise<Buffer | undefined>;
+    getValue(key: string, config?: { fallbacks?: boolean }): Promise<Buffer | undefined>;
     deleteKey(key: string): Promise<void>;
+    /** Sync introspection of the lock store, if it supports it - undefined when unsupported. getFiles uses this to distinguish "store is up, this was a real error" from "store is unreachable, fall back to non-authoritative reads". */
+    getSyncStatus(): Promise<ArchivesSyncStatus | undefined>;
     /** The amount of time before we have to wait until it is assumed that all reads
      *      this old will have been received.
      *      - On the local disk it could be 0, but we use a higher value so development is closer to production
@@ -473,7 +481,7 @@ class TransactionLocker {
 
     private lastFilesRead: FileInfo[] | undefined;
     private lastFilesReadTime: number | undefined;
-    private async readDataState(): Promise<{
+    private async readDataState(config?: { fallbacks?: boolean }): Promise<{
         rawDataFiles: FileInfo[];
         /** Confirmed FileInfos are === the FileInfos in rawDataFiles */
         confirmedDataFiles: FileInfo[];
@@ -487,7 +495,7 @@ class TransactionLocker {
         let bufferCache = new Map<string, Buffer>();
         const tryToRead = async () => {
             let time = Date.now();
-            let files = await this.storage.getKeys();
+            let files = await this.storage.getKeys(config);
             this.perf.checkpoint("getKeys");
             if (this.lastFilesRead && this.lastFilesReadTime) {
                 let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
@@ -516,7 +524,7 @@ class TransactionLocker {
             let missingValue = false;
             const readTransaction = async (tFile: FileInfo) => {
                 if (missingValue) return;
-                let buffer = await this.storage.getValue(tFile.file);
+                let buffer = await this.storage.getValue(tFile.file, config);
                 if (!buffer) {
                     missingValue = true;
                     return;
@@ -554,7 +562,7 @@ class TransactionLocker {
             //      but validity should/can only depend on the current state, not past event transitions
             //      (it pretty much only depends on files existing).
             {
-                let filesVerify = await this.storage.getKeys();
+                let filesVerify = await this.storage.getKeys(config);
                 this.perf.checkpoint("verify getKeys");
                 let filesVerifySet = new Set(filesVerify.map(a => a.file));
                 // If it changes while reading, read again. Otherwise, if there were no changes while reading,
@@ -790,11 +798,24 @@ class TransactionLocker {
      *      - Might run a transaction
      */
     public async getFiles(): Promise<FileInfo[]> {
-        this.perf.start();
-        let obj = await this.getFilesBase();
-        logNodeStateStats(`ArchiveLock Data File`, formatNumber, obj.dataFiles.length);
-        this.perf.logAndFlush(`getFiles (${this.storage.debugKey})`);
-        return obj.dataFiles;
+        let time = Date.now();
+        let storageAlivePromise = errorToUndefinedSilent(this.storage.getSyncStatus());
+        try {
+            this.perf.start();
+            let obj = await this.getFilesBase();
+            logNodeStateStats(`ArchiveLock Data File`, formatNumber, obj.dataFiles.length);
+            this.perf.logAndFlush(`getFiles (${this.storage.debugKey})`);
+            return obj.dataFiles;
+        } catch (e) {
+            let duration0 = Date.now() - time;
+            if (await storageAlivePromise) {
+                throw e;
+            }
+            let duration = Date.now() - time;
+            console.error(`Falling back to non-authoritative path value files after ${formatTime(duration)} (${formatTime(duration0)}). NO data will be written to the database until our storage servers are back online!`, { error: e });
+            let dataState = await this.readDataState({ fallbacks: true });
+            return dataState.confirmedDataFiles;
+        }
     }
     private transactionAppliedCount = new Map<number, number>();
     private async getFilesBase(): Promise<{

package/src/deployManager/machineSchema.ts

@@ -29,6 +29,16 @@ export const SERVICE_FOLDER = `${SERVICE_FOLDER_NAME}/`;
 export const MACHINE_RESYNC_INTERVAL = timeInMinute * 15;
 export const SERVICE_NODE_FILE_NAME = "serviceNodeId.txt";
 
+
+export type MachineConfig = {
+    machineId: string;
+    disabled: boolean;
+};
+// We want all of these to have very high availability. Even if they're wrong, it's better to have an old service deployed or even view information on an old service, or even set information only in back plays rather than have the server be completely down. 
+export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"), { fallbacks: true });
+export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"), { fallbacks: true });
+export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"), { fallbacks: true });
+
 export type MachineInfo = {
     machineId: string;
 
@@ -182,13 +192,6 @@ function normalizeServiceConfig(config: ServiceConfig): ServiceConfig {
     return config;
 }
 
-export type MachineConfig = {
-    machineId: string;
-    disabled: boolean;
-};
-export const machineInfos = archiveJSONT<MachineInfo>(() => getArchives2("machines/machine-heartbeats/"));
-export const serviceConfigs = archiveJSONT<ServiceConfig>(() => getArchives2("machines/service-configs/"));
-export const machineConfigs = archiveJSONT<MachineConfig>(() => getArchives2("machines/machine-configs/"));
 
 export type LaunchRecord = {
     serviceId: string;