querysub 0.604.0 → 0.605.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.604.0",
+  "version": "0.605.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -76,7 +76,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "sliftutils": "^1.7.83",
-    "socket-function": "^1.2.29",
+    "socket-function": "^1.2.30",
     "terser": "^5.31.0",
     "typenode": "^6.6.1",
     "typesafecss": "^0.32.0",

package/src/-a-archives/archives2.ts

@@ -121,6 +121,7 @@ function nestBucket(folder: string, archives: ReturnType<typeof createArchives>)
             let changes = await archives.getChangesAfter2(config);
             return changes.filter(x => x.path.startsWith(folder)).map(x => ({ ...x, path: x.path.slice(folder.length) }));
         },
+        getSyncStatus: async () => await archives.getSyncStatus(),
     };
 }
 

package/src/-a-archives/archivesJSONT.ts

@@ -1,14 +1,15 @@
 import { lazy } from "socket-function/src/caching";
 import { Archives } from "./archives";
 import { green } from "socket-function/src/formatting/logColors";
-import { IArchives } from "sliftutils/storage/IArchives";
+import { FindConfig, IArchives } from "sliftutils/storage/IArchives";
+import { SetConfig } from "sliftutils/storage/IArchives";
 
 export type ArchiveT<T> = {
     get(key: string): Promise<T | undefined>;
-    set(key: string, value: T): Promise<void>;
+    set(key: string, value: T, setConfig?: SetConfig): Promise<void>;
     delete(key: string): Promise<void>;
-    keys(): Promise<string[]>;
-    values(): Promise<T[]>;
+    keys(findConfig?: FindConfig): Promise<string[]>;
+    values(findConfig?: FindConfig): Promise<T[]>;
     entries(): Promise<[string, T][]>;
     [Symbol.asyncIterator](): AsyncIterator<[string, T]>;
 };
@@ -27,21 +28,21 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
         if (!buffer) return undefined;
         return JSON.parse(buffer.toString()) as T;
     }
-    async function set(key: string, value: T) {
+    async function set(key: string, value: T, setConfig?: SetConfig) {
         let a = archives();
         console.log(`In archiveJSONT ${a.getDebugName()}, setting ${key} to ${JSON.stringify(value)}`);
-        await a.set(key, Buffer.from(JSON.stringify(value)));
+        await a.set(key, Buffer.from(JSON.stringify(value)), setConfig);
     }
     async function deleteFnc(key: string) {
         let a = archives();
         console.log(`In archiveJSONT ${a.getDebugName()}, deleting ${key}`);
         await a.del(key);
     }
-    async function keys() {
-        return (await archives().find("")).map(value => value.toString());
+    async function keys(findConfig?: FindConfig) {
+        return (await archives().find("", findConfig)).map(value => value.toString());
     }
-    async function values() {
-        let infos = await archives().findInfo("");
+    async function values(findConfig?: FindConfig) {
+        let infos = await archives().findInfo("", findConfig);
 
         let needsUpdate = false;
         let currentKeys = new Set(infos.map(info => info.path));
@@ -63,7 +64,7 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
             let maxRetries = 10;
             let updated = false;
             for (let attempt = 0; attempt < maxRetries; attempt++) {
-                infos = await archives().findInfo("");
+                infos = await archives().findInfo("", findConfig);
 
                 let newCache = new Map<string, {
                     createTime: number;
@@ -92,7 +93,7 @@ export function archiveJSONT<T>(archives: () => IArchives): ArchiveT<T> {
 
                 if (!allFound) continue;
 
-                let newInfos = await archives().findInfo("");
+                let newInfos = await archives().findInfo("", findConfig);
                 if (newInfos.length !== infos.length) continue;
                 function anyChanged() {
                     for (let i = 0; i < newInfos.length; i++) {

package/src/-d-trust/NetworkTrust2.ts

@@ -132,7 +132,7 @@ export const ensureWeAreTrusted = lazy(measureWrap(async () => {
     let machineKeyCert = getIdentityCA(getDomain());
     let machineId = getOwnMachineId(getDomain());
     if (!await archives().get(machineId)) {
-        await archives().set(machineId, machineKeyCert.cert);
+        await archives().set(machineId, machineKeyCert.cert, { fallbacks: true });
     }
 }));
 

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -308,7 +308,7 @@ export async function triggerNodeChange() {
 
 
 async function clearDeadThreadsFromArchives() {
-    let nodes = await archives().find("");
+    let nodes = await archives().find("", { fallbacks: true });
 
     function getPortHash(nodeId: string) {
         let obj = decodeNodeId(nodeId, getDomain());
@@ -329,7 +329,7 @@ async function clearDeadThreadsFromArchives() {
             let deadThreads = nodeIds.filter(nodeId => nodeId !== aliveNodeId);
             await Promise.all(deadThreads.map(async deadNodeId => {
                 console.log(`Removing dead thread. We contacted a node on the same port and same machine (${aliveNodeId}), which means the port has been reused by another thread, which proves that the old thread has died, as otherwise the new thread would not be able to use it.`);
-                await archives().del(deadNodeId);
+                await archives().del(deadNodeId, { fallbacks: true });
             }));
         }
     }
@@ -393,7 +393,7 @@ async function runHeartbeatAuditLoop() {
                 if (count >= DEAD_CHECK_COUNT) {
                     removedNodeIds.push(nodeId);
                     console.info(yellow(`Node ${nodeId} was found to be dead, removing from node list. Last heartbeat at ${formatDateTime(lastTime)}, dead threshold at ${formatDateTime(deadTime)}`));
-                    await archives().del(nodeId);
+                    await archives().del(nodeId, { fallbacks: true });
                     deadCount.delete(nodeId);
                 } else {
                     console.info(yellow(`Node ${nodeId} was found to be dead, last heartbeat at ${formatDateTime(lastTime)} < dead threshold at ${formatDateTime(deadTime)}, dead count ${count}/${DEAD_CHECK_COUNT}. Total nodes seen ${nodeIds.length}`));
@@ -497,7 +497,7 @@ async function writeHeartbeat() {
     let nodeId = getMountNodeId();
     console.log(green(`Writing heartbeat ${formatDateTime(now)} for self (${nodeId})`));
     if (!nodeId) return;
-    await archives().set(nodeId, Buffer.from(now + ""));
+    await archives().set(nodeId, Buffer.from(now + ""), { fallbacks: true });
 }
 
 async function runServerSyncLoops() {
@@ -636,7 +636,7 @@ if (isServer()) {
 
 
 export async function forceRemoveNode(nodeId: string) {
-    await archives().del(nodeId);
+    await archives().del(nodeId, { fallbacks: true });
     void tellEveryoneNodesChanges(`forceRemoveNode ${nodeId}`);
 }
 
@@ -646,7 +646,7 @@ export async function nodeDiscoveryShutdown() {
     console.log(red(`Shutting down node discovery`));
     shutdown = true;
     if (isServer()) {
-        await archives().del(getOwnNodeId());
+        await archives().del(getOwnNodeId(), { fallbacks: true });
     }
     void tellEveryoneNodesChanges("nodeDiscoveryShutdown");
 }

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -4,11 +4,11 @@ import { decodeNodeId } from "sliftutils/misc/https/certs";
 import { getOwnNodeId, getOwnNodeIdAssert } from "../../-f-node-discovery/NodeDiscovery";
 import { pathValueArchives } from "../pathValueArchives";
 import { ArchiveLocker, ArchiveTransaction } from "./ArchiveLocks";
-import { copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
+import { ArchivesSyncStatus, copyArchiveFile, IArchives } from "sliftutils/storage/IArchives";
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, magenta, red } from "socket-function/src/formatting/logColors";
 import { devDebugbreak, getDomain } from "../../config";
-import { logErrors } from "../../errors";
+import { errorToUndefined, errorToUndefinedSilent, logErrors, timeoutToUndefined } from "../../errors";
 import { saveSnapshot } from "./archiveSnapshots";
 import { getNodeId } from "socket-function/src/nodeCache";
 import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
@@ -35,6 +35,8 @@ const ASSUME_CONFIRMED_BEFORE = Date.parse("2026-07-23T00:00:00.000Z");
 const CONCURRENT_READ_COUNT = 128;
 const CONCURRENT_WRITE_COUNT = 128;
 const CONCURRENT_TRANSACTION_READS = 8;
+// When getFiles fails, how long we wait on the lock store's sync status before assuming the store is unreachable and falling back to non-authoritative (backblaze) reads.
+const FALLBACK_SYNC_STATUS_TIMEOUT = timeInSecond * 20;
 
 /** Moves by copy-then-delete (IArchives has no move). copyArchiveFile stamps the target with the source's write time, so ordering-by-writeTime survives the move. A missing source means another process already moved it, which is fine. */
 async function moveArchiveFile(from: IArchives, to: IArchives, path: string): Promise<void> {
@@ -60,9 +62,9 @@ export function createArchiveLocker2(config: {
         debugKey,
         propagationTime: ARCHIVE_PROPAGATION_TIME,
 
-        async getKeys() {
-            let filesRawValues = await archiveValues.findInfo("", { type: "files" });
-            let filesRawLocks = await archiveLocks.findInfo("", { type: "files" });
+        async getKeys(config) {
+            let filesRawValues = await archiveValues.findInfo("", { type: "files", fallbacks: config?.fallbacks });
+            let filesRawLocks = await archiveLocks.findInfo("", { type: "files", fallbacks: config?.fallbacks });
             // Zero-byte files are deletes on new storage servers. Old storage servers still list them, so we filter here to make them behave the same.
             let files: FileInfo[] = [...filesRawValues, ...filesRawLocks].filter(x => x.size > 0).map(x => ({
                 file: x.path,
@@ -71,14 +73,18 @@ export function createArchiveLocker2(config: {
             }));
             return files;
         },
+        async getSyncStatus() {
+            if (!archiveLocks.getSyncStatus) return undefined;
+            return archiveLocks.getSyncStatus();
+        },
 
         async setValue(key, value) {
             await getArchives(key).set(key, value);
             logNodeStats(`archives|Created TΔ`, formatNumber, 1);
         },
-        async getValue(key) {
-            // NOTE: I think we wanna not use fallbacks, Because we really want this to be atomic and safe. 
-            return getArchives(key).get(key, { noFallbacks: true });
+        async getValue(key, config) {
+            // Default to no fallbacks as we usually want this to be very safe. We don't want to apply transactions if the system isn't in a good state. 
+            return await getArchives(key).get(key, { noFallbacks: !config?.fallbacks });
         },
         async deleteKey(key) {
             let archives = await getArchives(key);
@@ -294,10 +300,12 @@ export type FileInfo = {
 type StorageType = {
     debugKey: string;
     propagationTime: number;
-    getKeys(): Promise<FileInfo[]>;
+    getKeys(config?: { fallbacks?: boolean }): Promise<FileInfo[]>;
     setValue(key: string, value: Buffer): Promise<void>;
-    getValue(key: string): Promise<Buffer | undefined>;
+    getValue(key: string, config?: { fallbacks?: boolean }): Promise<Buffer | undefined>;
     deleteKey(key: string): Promise<void>;
+    /** Sync introspection of the lock store, if it supports it - undefined when unsupported. getFiles uses this to distinguish "store is up, this was a real error" from "store is unreachable, fall back to non-authoritative reads". */
+    getSyncStatus(): Promise<ArchivesSyncStatus | undefined>;
     /** The amount of time before we have to wait until it is assumed that all reads
      *      this old will have been received.
      *      - On the local disk it could be 0, but we use a higher value so development is closer to production
@@ -473,7 +481,7 @@ class TransactionLocker {
 
     private lastFilesRead: FileInfo[] | undefined;
     private lastFilesReadTime: number | undefined;
-    private async readDataState(): Promise<{
+    private async readDataState(config?: { fallbacks?: boolean }): Promise<{
         rawDataFiles: FileInfo[];
         /** Confirmed FileInfos are === the FileInfos in rawDataFiles */
         confirmedDataFiles: FileInfo[];
@@ -487,7 +495,7 @@ class TransactionLocker {
         let bufferCache = new Map<string, Buffer>();
         const tryToRead = async () => {
             let time = Date.now();
-            let files = await this.storage.getKeys();
+            let files = await this.storage.getKeys(config);
             this.perf.checkpoint("getKeys");
             if (this.lastFilesRead && this.lastFilesReadTime) {
                 let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
@@ -516,7 +524,7 @@ class TransactionLocker {
             let missingValue = false;
             const readTransaction = async (tFile: FileInfo) => {
                 if (missingValue) return;
-                let buffer = await this.storage.getValue(tFile.file);
+                let buffer = await this.storage.getValue(tFile.file, config);
                 if (!buffer) {
                     missingValue = true;
                     return;
@@ -554,7 +562,7 @@ class TransactionLocker {
             //      but validity should/can only depend on the current state, not past event transitions
             //      (it pretty much only depends on files existing).
             {
-                let filesVerify = await this.storage.getKeys();
+                let filesVerify = await this.storage.getKeys(config);
                 this.perf.checkpoint("verify getKeys");
                 let filesVerifySet = new Set(filesVerify.map(a => a.file));
                 // If it changes while reading, read again. Otherwise, if there were no changes while reading,
@@ -790,11 +798,24 @@ class TransactionLocker {
      *      - Might run a transaction
      */
     public async getFiles(): Promise<FileInfo[]> {
-        this.perf.start();
-        let obj = await this.getFilesBase();
-        logNodeStateStats(`ArchiveLock Data File`, formatNumber, obj.dataFiles.length);
-        this.perf.logAndFlush(`getFiles (${this.storage.debugKey})`);
-        return obj.dataFiles;
+        let time = Date.now();
+        let storageAlivePromise = errorToUndefinedSilent(this.storage.getSyncStatus());
+        try {
+            this.perf.start();
+            let obj = await this.getFilesBase();
+            logNodeStateStats(`ArchiveLock Data File`, formatNumber, obj.dataFiles.length);
+            this.perf.logAndFlush(`getFiles (${this.storage.debugKey})`);
+            return obj.dataFiles;
+        } catch (e) {
+            let duration0 = Date.now() - time;
+            if (await storageAlivePromise) {
+                throw e;
+            }
+            let duration = Date.now() - time;
+            console.error(`Falling back to non-authoritative path value files after ${formatTime(duration)} (${formatTime(duration0)}). NO data will be written to the database until our storage servers are back online!`, { error: e });
+            let dataState = await this.readDataState({ fallbacks: true });
+            return dataState.confirmedDataFiles;
+        }
     }
     private transactionAppliedCount = new Map<number, number>();
     private async getFilesBase(): Promise<{

package/src/deployManager/machineSchema.ts

@@ -276,7 +276,7 @@ export class MachineServiceControllerBase {
 
     public async getMachineList() {
         return [...new Set([
-            ...(await machineInfos.keys()),
+            ...(await machineInfos.keys({ fallbacks: true })),
         ])];
     }
     public async deleteMachineIds(machineIds: string[]) {
@@ -303,7 +303,7 @@ export class MachineServiceControllerBase {
     }
     public async getServiceList(): Promise<string[]> {
         return [...new Set([
-            ...(await serviceConfigs.keys()),
+            ...(await serviceConfigs.keys({ fallbacks: true })),
         ])];
     }
     public async getServiceConfig(serviceId: string): Promise<ServiceConfig | undefined> {
@@ -312,7 +312,7 @@ export class MachineServiceControllerBase {
     }
 
     public async getMachineConfigList() {
-        return await machineConfigs.values();
+        return await machineConfigs.values({ fallbacks: true });
     }
     public async getMachineConfig(machineId: string): Promise<MachineConfig | undefined> {
         return await machineConfigs.get(machineId);
@@ -378,7 +378,7 @@ export class MachineServiceControllerBase {
             // NOTE: This is correct. This is obviously correct. For some reason, the AI wrote some really big, complicated thing here. I don't know why. This is the only way it should be done, obviously.
             config.oldParameters = getLiveServiceParameters(serviceConfig);
 
-            await serviceConfigs.set(serviceId, config);
+            await serviceConfigs.set(serviceId, config, { fallbacks: true });
             // Only notify we were or are deployed. If it's not deployed, this will be ignored anyways.
             if (config.parameters.deploy || serviceConfig.parameters.deploy) {
                 for (let machineId of getConfigMachineIds(config)) {