querysub 0.156.0 → 0.158.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.156.0",
+  "version": "0.158.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "preact": "^10.11.3",
-    "socket-function": "^0.89.0",
+    "socket-function": "^0.90.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.6.3",
     "yaml": "^2.5.0",

package/src/-b-authorities/cloudflareHelpers.ts

@@ -3,15 +3,17 @@ import { getArchives } from "../-a-archives/archives";
 import { lazy } from "socket-function/src/caching";
 import { getStorageDir, getSubFolder } from "../fs";
 import fs from "fs";
-export const keys = getArchives("keys");
+import { isNodeTrue } from "socket-function/src/misc";
+
+export const keys = lazy(() => getArchives("keys"));
 
 export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: string }> => {
-    let credsJSON = await keys.get("cloudflare.json");
+    let credsJSON = await keys().get("cloudflare.json");
     if (!credsJSON) {
         let localPath = getStorageDir() + "cloudflare.json";
         if (fs.existsSync(localPath)) {
             credsJSON = fs.readFileSync(localPath);
-            await keys.set("cloudflare.json", credsJSON);
+            await keys().set("cloudflare.json", credsJSON);
         } else {
             throw new Error(`b2:/keys/cloudflare.json is missing. It should contain { "key": "your-key", "email": "your-email" }`);
         }

package/src/-b-authorities/dnsAuthority.ts

@@ -1,7 +1,7 @@
 import os from "os";
 import * as fs from "fs";
 import { cache, lazy } from "socket-function/src/caching";
-import { isNode } from "socket-function/src/misc";
+import { isNode, isNodeTrue } from "socket-function/src/misc";
 import { httpsRequest } from "../https";
 import { getStorageDir } from "../fs";
 import { SocketFunction } from "socket-function/SocketFunction";
@@ -16,8 +16,6 @@ const DNS_TTLSeconds = {
     "A": 60,
 };
 
-export const keys = getArchives("keys");
-
 export const hasDNSWritePermissions = lazy(async () => {
     if (!isNode()) return false;
     if (isClient()) return false;

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -243,13 +243,12 @@ export async function triggerNodeChange() {
     }));
 }
 
-let seenNodesForEvilDNS = new Set<string>();
 async function syncArchives() {
     if (isServer()) {
         // Make sure we are present
         await writeHeartbeat();
         let nodeIds = await archives().find("");
-        console.log(green(`Syncing archives`), { nodeIds });
+        console.log(green(`Syncing node ids from archives`), { nodeIds });
         setNodeIds(nodeIds);
     } else {
         if (isNoNetwork() || !isNode()) {

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -18,7 +18,6 @@ import { logNodeStateStats, logNodeStats } from "../../-0-hooks/hooks";
 /** Clean up old files after a while */
 const DEAD_CREATE_THRESHOLD = timeInHour * 12;
 const ARCHIVE_PROPAGATION_TIME = 5000;
-const LOG = false;
 // NOTE: There isn't a reason for transaction to not apply. If applying throws, we throw,
 //  and can throw infinite times, but if it doesn't throw, it SHOULD be applied. We retry
 //  a few times though, just in case the storage system was having some issues. After enough
@@ -309,9 +308,7 @@ class TransactionLocker {
     }
     public async createConfirm(key: string) {
         let path = this.getConfirmKey(key);
-        if (LOG) {
-            console.log(`Creating confirmation for ${key}`);
-        }
+        diskLog(`Creating confirmation for ${key}`);
         await this.storage.setValue(path, Buffer.from(""));
         return path;
     }
@@ -363,9 +360,10 @@ class TransactionLocker {
             delete: ellipsize(deletes.map(a => debugFileInfo(a.key)).join(","), 50),
         });
 
-        if (LOG) {
-            console.log(`Writing transaction ${transaction.seqNum} with ${transaction.ops.length} ops`);
-        }
+        diskLog(`Writing transaction`, {
+            name,
+            ops: transaction.ops.length,
+        });
 
         let key = `transaction_${name}.transaction`;
         let strippedTransaction: Transaction = { ops: transaction.ops.map(a => ({ ...a, value: undefined })) };
@@ -490,10 +488,8 @@ class TransactionLocker {
         while (true) {
             let result = await tryToRead();
             if (result) {
-                if (LOG) {
-                    let timeToRead = Date.now() - startTime;
-                    console.log(`Read data state in ${formatTime(timeToRead)}`);
-                }
+                let timeToRead = Date.now() - startTime;
+                diskLog(`Read data state in ${formatTime(timeToRead)}`);
                 return result;
             }
         }
@@ -532,20 +528,20 @@ class TransactionLocker {
         let rawLookup = new Set(Array.from(rawDataFiles).map(a => a.file));
         // If any creates are not confirmed, it must not have been applied
         if (transaction.ops.some(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))) {
-            if (LOG) {
-                console.log(`Transaction has pending confirmations of creates.`);
-                let pending = transaction.ops.filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key));
-                for (let p of pending) {
-                    console.log(`    ${p.key}`);
-                }
-            }
+            diskLog(`Transaction not applied (has pending confirmations of creates)`, {
+                keys: transaction.ops
+                    .filter(a => a.type === "create" && rawLookup.has(a.key) && !confirmedKeys.has(a.key))
+                    .map(a => a.key)
+            });
             return false;
         }
         // If any deletes still exist, it must not have been applied
         if (transaction.ops.some(a => a.type === "delete" && confirmedKeys.has(a.key))) {
-            if (LOG) {
-                console.log(`transaction has pending deletes`);
-            }
+            diskLog(`Transaction not applied (has pending deletes)`, {
+                keys: transaction.ops
+                    .filter(a => a.type === "delete" && confirmedKeys.has(a.key))
+                    .map(a => a.key)
+            });
             return false;
         }
         return true;
@@ -554,9 +550,9 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        if (LOG) {
-            console.log(`Applying transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`);
-        }
+        diskLog(`Applying transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+            transactions: transaction.ops.map(x => JSON.stringify(x)),
+        });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
         let opsRemaining = transaction.ops.slice();
         // NOTE: Order doesn't matter here. If anything is reading the values
@@ -580,9 +576,9 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        if (LOG) {
-            console.log(`Applied transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`);
-        }
+        diskLog(`Applied transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+            transactions: transaction.ops.map(x => JSON.stringify(x)),
+        });
     }
 
     /** Only returns data files (no transaction files, or confirmations).
@@ -646,9 +642,7 @@ class TransactionLocker {
         let threshold = activeT.createTime + this.storage.propagationTime;
         if (Date.now() < threshold) {
             let waitTime = threshold - Date.now();
-            if (LOG) {
-                console.log(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
-            }
+            diskLog(`Waiting ${formatTime(waitTime)} for transaction ${activeT.seqNum} to settle.`);
             await new Promise(resolve => setTimeout(resolve, waitTime));
             return this.getFilesBase();
         }
@@ -688,9 +682,7 @@ class TransactionLocker {
                     //  triggered (without this check), and deletes all of our files...
                     let unconfirmedOldFiles2 = veryOldFiles.filter(a => !doubleCheckLookup.has(a) && doubleCheckDataFiles.has(a.file));
                     console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`));
-                    if (LOG) {
-                        logNodeStats(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
-                    }
+                    diskLog(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
                     // At the point the file was very old when we started reading, not part of the active transaction.
                     for (let file of unconfirmedOldFiles2) {
                         await this.deleteDataFile(file.file, `old unconfirmed file (${getOwnNodeId()}, ${process.argv[1]})`);
@@ -706,10 +698,8 @@ class TransactionLocker {
                 let oldEnoughConfirms = dataState.rawDataFiles.filter(x => x.file.endsWith(".confirm") && x.createTime < oldThreshold);
                 let deprecatedFiles = oldEnoughConfirms.filter(a => !usedConfirmations.has(a.file));
                 if (deprecatedFiles.length > 0) {
-                    if (LOG) {
-                        console.warn(red(`Deleted ${deprecatedFiles.length} / ${oldEnoughConfirms.length} confirmations, for not having corresponding data files`));
-                        logNodeStats(`archives|TΔ Delete Deprecated Confirm`, formatNumber, deprecatedFiles.length);
-                    }
+                    console.warn(red(`Deleted ${deprecatedFiles.length} / ${oldEnoughConfirms.length} confirmations, for not having corresponding data files`));
+                    logNodeStats(`archives|TΔ Delete Deprecated Confirm`, formatNumber, deprecatedFiles.length);
                     for (let file of deprecatedFiles) {
                         await this.storage.deleteKey(file.file);
                     }
@@ -744,12 +734,10 @@ class TransactionLocker {
 
     /** If any deleted files were deleted by other transactions, then we will be rejected. */
     public async addTransaction(transaction: Transaction): Promise<"accepted" | "rejected"> {
-        if (LOG) {
-            let dels = transaction.ops.filter(a => a.type === "delete").length;
-            let creates = transaction.ops.filter(a => a.type === "create").length;
-            let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-            console.log(blue(`Starting transaction with ${creates} creates and ${dels} deletes, ${formatNumber(createBytes)}B`));
-        }
+        let dels = transaction.ops.filter(a => a.type === "delete").length;
+        let creates = transaction.ops.filter(a => a.type === "create").length;
+        let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
+        diskLog(`Starting transaction with ${creates} creates and ${dels} deletes, ${formatNumber(createBytes)}B`);
         transaction = { ...transaction, ops: transaction.ops.slice() };
         function normalizePath(path: string) {
             // Replace duplicate slashes with a single slash
@@ -775,9 +763,7 @@ class TransactionLocker {
             let beforeData = await this.getFilesBase();
             if (!this.isTransactionValid(transaction, beforeData.dataFiles, beforeData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Rejected`, formatNumber, 1);
-                if (LOG) {
-                    console.log(red(`Finished transaction with rejection, ${transaction.ops.length} ops`));
-                }
+                diskLog(`Finished transaction with rejection, ${transaction.ops.length} ops`);
                 return "rejected";
             }
 
@@ -786,9 +772,7 @@ class TransactionLocker {
             let afterData = await this.getFilesBase();
             if (this.wasTransactionApplied(transaction, afterData.dataFiles, afterData.rawDataFiles)) {
                 logNodeStats(`archives|TΔ Accepted`, formatNumber, 1);
-                if (LOG) {
-                    console.log(green(`Finished transaction with ${transaction.ops.length} ops`));
-                }
+                diskLog(`Finished transaction with ${transaction.ops.length} ops`);
                 return "accepted";
             }
         }

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -6,8 +6,9 @@ import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors"
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import { devDebugbreak } from "../../config";
 import { sort } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
 
-const snapshots = getArchives("snapshots");
+const snapshots = lazy(() => getArchives("snapshots"));
 
 export type ArchiveSnapshotOverview = {
     // If live, it isn't a real file that can be loaded
@@ -54,7 +55,7 @@ export async function saveSnapshot(config: {
 }) {
     let { files } = config;
     let overview = getSnapshotOverview(files);
-    await snapshots.set(overview.file, Buffer.from(files.join("\n")));
+    await snapshots().set(overview.file, Buffer.from(files.join("\n")));
 }
 function overviewToFileName(overview: Omit<ArchiveSnapshotOverview, "file">): string {
     return Object.entries({
@@ -84,7 +85,7 @@ function fileNameToOverview(fileName: string): ArchiveSnapshotOverview {
 
 // pathValueArchives.decodeDataPath
 export async function getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
-    let snapshotFiles = await snapshots.find("");
+    let snapshotFiles = await snapshots().find("");
 
     let overview: ArchiveSnapshotOverview[] = [];
     let locker = await pathValueArchives.getArchiveLocker();
@@ -128,7 +129,7 @@ async function getSnapshotBase(snapshotFile: string | "live") {
             files: allFiles.map(x => x.file),
         };
     }
-    let data = await snapshots.get(snapshotFile);
+    let data = await snapshots().get(snapshotFile);
     if (!data) {
         throw new Error(`Snapshot not found: ${snapshotFile}`);
     }
@@ -158,7 +159,7 @@ export async function loadSnapshot(config: {
     console.log(magenta(`Loading snapshot: ${overview.file}`));
     let locker = await pathValueArchives.getArchiveLocker();
 
-    let snapshotData = await snapshots.get(overview.file);
+    let snapshotData = await snapshots().get(overview.file);
     if (!snapshotData) {
         throw new Error(`Snapshot not found: ${overview.file}`);
     }

package/src/0-path-value-core/pathValueCore.ts

@@ -1768,13 +1768,15 @@ class WriteValidStorage {
             let authorityValue = authorityStorage.getValueExactIgnoreInvalid(write.path, write.time);
             if (authorityValue) {
                 authorityValue.valid = write.isValid;
-                if (!isCoreQuiet) {
-                    console.log(`Setting valid state of ${debugPathValuePath(authorityValue)} to ${write.isValid}`);
-                }
+                // NOTE: I think it is fine now for us to set the valid state early? Hopefully...
+                //  because it happens A LOT.
+                // if (!isCoreQuiet) {
+                //     console.log(`Setting valid state of ${debugPathValuePath(authorityValue)} to ${write.isValid}`);
+                // }
             } else {
-                if (isNode()) {
-                    console.error(`Setting valid state of ${write.path}@${debugTime(write.time)} to ${write.isValid}, but the ValuePath was not found. If the ValuePath is found later, it might not have the valid state set correctly.`);
-                }
+                // if (isNode()) {
+                //     console.error(`Setting valid state of ${write.path}@${debugTime(write.time)} to ${write.isValid}, but the ValuePath was not found. If the ValuePath is found later, it might not have the valid state set correctly.`);
+                // }
             }
         }
 

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -162,6 +162,9 @@ interface WatcherOptions<Result> {
     // Fairly self explanatory. If there are nested createWatchers, instead of forking, we just
     //  run them immediately.
     inlineNestedWatchers?: boolean;
+
+    // Temporary indicates after becoming synchronizes it will immediately dispose itself
+    temporary?: boolean;
 }
 
 let harvestableReadyLoopCount = 0;
@@ -286,6 +289,8 @@ export type SyncWatcher = {
     tag: SyncWatcherTag;
 
     hackHistory: { message: string; time: number }[];
+
+    createTime: number;
 }
 function addToHistory(watcher: SyncWatcher, message: string) {
     watcher.hackHistory.push({ message, time: Date.now() });
@@ -989,6 +994,7 @@ export class PathValueProxyWatcher {
             tag: new SyncWatcherTag(),
 
             hackHistory: [],
+            createTime: Date.now(),
         };
         const SHOULD_TRACE = this.SHOULD_TRACE(watcher);
         const proxy = this.proxy;
@@ -1205,11 +1211,13 @@ export class PathValueProxyWatcher {
                         debugger;
                         // NOTE: Using forceEqualWrites will also fix this a lot of the time, such as when
                         //  a write contains random numbers or dates.
-                        let errorMessage = `Too many attempts to sync with different values. If you are reading in a loop, make sure to read all the values, instead of aborting the loop if a value is not synced. ALSO, make sure you don't access paths with Math.random() or Date.now(). This will prevent the sync loop from ever stabilizing.`;
+                        let errorMessage = `Too many attempts (${watcher.countSinceLastFullSync}) to sync with different values. If you are reading in a loop, make sure to read all the values, instead of aborting the loop if a value is not synced. ALSO, make sure you don't access paths with Math.random() or Date.now(). This will prevent the sync loop from ever stabilizing.`;
                         if (specialPromiseUnsynced) {
                             errorMessage += ` A promise is being accessed, so it is possible triggerOnPromiseFinish is being used on a new promise every loop, which cannot work (you MUST cache MaybePromise and replace the value with a non-promise, otherwise it will never be available synchronously!)`;
                         }
+                        errorMessage += ` (${watcher.debugName}})`;
                         console.error(red(errorMessage));
+                        logUnsynced();
                         result = { error: new Error(errorMessage).stack || "" };
                         // Force the watches to be equal, so we stop looping
                         watcher.pendingWatches = watcher.lastWatches;
@@ -1648,6 +1656,7 @@ export class PathValueProxyWatcher {
         let anyWrites = false;
         this.createWatcher({
             ...options,
+            temporary: true,
             onResultUpdated: (result, writes, watcher) => {
                 if (writes?.length) {
                     anyWrites = true;
@@ -1700,6 +1709,7 @@ export class PathValueProxyWatcher {
             ...options,
             canWrite: true,
             dryRun: true,
+            temporary: true,
             onResultUpdated: (result, values, watcher) => {
                 watcher.dispose();
                 if ("error" in result) {

package/src/2-proxy/archiveMoveHarness.ts

@@ -1,7 +1,3 @@
-
-
-// yarn typenode src\framework-beta\archiveMoveBase\archiveMoveHarness.ts --nonetwork
-
 import { formatNumber, formatTime } from "socket-function/src/formatting/format";
 import { blue, green, red } from "socket-function/src/formatting/logColors";
 import { measureBlock, measureWrap } from "socket-function/src/profiling/measure";
@@ -22,6 +18,7 @@ import debugbreak from "debugbreak";
 let lastFileWrites = new Map<string, string[]>();
 
 export async function runArchiveMover(config: {
+    debugName: string;
     // Defaults to getOurAuthorities
     //  NOTE: Runs once per authority, as we want to keep data as sharded as possible
     authorities?: AuthorityPath[];
@@ -307,10 +304,11 @@ export async function runArchiveMover(config: {
             || lastResult === "aborted" && green("noop")
             || red("rejected")
         );
-        console.log(`ArchiveMover ${config.runMover.name} on ${authorityDir} in ${formatTime(time)}. Result = ${resultFormat}.`);
+        console.log(`ArchiveMover ${config.debugName ?? config.runMover.name} on ${authorityDir} in ${formatTime(time)}. Result = ${resultFormat}.`);
         if (lastResult === "accepted" && !abortedDueToInsufficientReduction) {
             console.log(
-                `    { files: ${formatNumber(inputFiles)}, values: ${formatNumber(inputValueCount)}, bytes: ${formatNumber(inputBytes)} }`
+                `ArchiveMover free committed ${config.debugName ?? config.runMover.name} on ${authorityDir}`
+                + `    { files: ${formatNumber(inputFiles)}, values: ${formatNumber(inputValueCount)}, bytes: ${formatNumber(inputBytes)} }`
                 + "\n=>\n"
                 + `    { files: ${formatNumber(outputFiles)}, values: ${formatNumber(outputValueCount)}, bytes: ${formatNumber(outputBytes)} }`
                 ,

package/src/2-proxy/garbageCollection.ts

@@ -542,6 +542,7 @@ export async function runAliveCheckerIteration(config?: {
 
             let lockWrites: PathValue[] = [];
             await runArchiveMover({
+                debugName: type,
                 outputType: type,
                 readLiveData: example.unsafeLiveReads,
                 force: config?.force,
@@ -719,7 +720,15 @@ export async function runAliveCheckerIteration(config?: {
                     }
 
                     let allFrees = new Set(freeLists.flat().flat().map(x => x.value));
+                    let allValues = values;
                     values = values.filter(x => !allFrees.has(x));
+                    if (allValues.length !== values.length) {
+                        console.log("Freeding values due to garbage collection", {
+                            before: allValues.length,
+                            after: values.length,
+                            freed: allValues.length - values.length,
+                        });
+                    }
 
                     // NOTE: We don't notify any servers about this. We will eventually have to write
                     //  audit code to periodically check against the disk to get updates.

package/src/2-proxy/schema2.ts

@@ -199,6 +199,10 @@ export class Schema2Fncs {
             }
             return { defaultValue: def.defaultValue, };
         }
+        // OR, if it is an optional object, then the default IS undefined
+        if (def?.isMaybeUndefined) {
+            return { defaultValue: undefined };
+        }
         return undefined;
     }
     public static isObject(schema: Schema2, path: string[]): boolean {
@@ -293,6 +297,7 @@ type InternalTypeDef = {
 
     isObject?: boolean;
     isObjectGC?: boolean;
+    isMaybeUndefined?: boolean;
 
     isNotAtomic?: boolean;
 
@@ -340,6 +345,9 @@ function typeDefTypeToInternalType(
         isObject = true;
         isObjectGC = false;
     }
+    if (typeDef.path.includes("optionalObject") || typeDef.path.includes("optionalObjectNoGC")) {
+        rootResult.isMaybeUndefined = true;
+    }
 
     let undoRegions = typeDef.pathWithCalls.filter(x => x.key === "undoRegionObject");
     for (let undoRegion of undoRegions) {
@@ -370,6 +378,7 @@ function typeDefTypeToInternalType(
         nestedValues.isObjectGC = isObjectGC || isLookupGC;
         nestedValues.gcDelay = Number(typeDef.pathWithCalls.find(x => x.key === "gcDelay")?.callParams?.[0]) || undefined;
 
+
         if (isLookup) {
             let undoRegions = typeDef.pathWithCalls.filter(x => x.key === "undoRegion");
             for (let undoRegion of undoRegions) {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -550,7 +550,7 @@ export class PathFunctionRunner {
                             }
                         }
                         if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
-                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}`);
+                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`);
                         }
 
                         // We NEED to depend on the function spec, so spec updates can be atomic (which is needed to deploy an update

package/src/3-path-functions/syncSchema.ts

@@ -140,14 +140,25 @@ export type PermissionsParameters = {
 };
 /** A false of false will deny read permissions, resulting in all reads being given value with a value
  *      of undefined, and a time of 0.
- *  NOTE: Permissions checks at the root (permissions: { READ_PERMISSIONS(config) { } }) will
- *      be used to verify calls are allowed as well, even if the call doesn't read or write any data
- *      that is under a permissions check.
+ 
  */
 export type PermissionsCallback = (config: PermissionsParameters) => PermissionsCheckResult;
 export type PermissionsCheckResult = boolean | { allowed: boolean; skipParentChecks?: boolean; };
 /*
     NOTE: All ancestor permissions checks are applied as well.
+        - This means if the ancestor disallows, it blocks any descendant checks even if they are allowed.
+
+    NOTE: If no permissions match, then access is disallowed by default.
+
+    NOTE: Permissions checks at the root (permissions: { READ_PERMISSIONS(config) { } }) will
+        be used to verify calls are allowed as well, even if the call doesn't read or write any data
+        that is under a permissions check.
+        - THIS MEANS that if there is no permission check at the root, calls won't be allowed by any user
+            (as we default to disallow).
+
+    NOTE: Servers watches ignore permissions, as they directly communicate with the underlying PathValueServer,
+        skipping the ProxyServer.
+        - However function calls by servers are still run by FunctionRunner, which will apply permission checks.
 
     IMPORTANT! Wildcards are tested via the "" path. This means if the "" key is provided access to a user, they can run
         Object.keys()/Object.values() on the data. In order to not accidentally provide list access, never allow "" to be

package/src/4-deploy/edgeNodes.ts

@@ -7,7 +7,7 @@ import { getArchivesBackblazePublic } from "../-a-archives/archivesBackBlaze";
 import { nestArchives } from "../-a-archives/archives";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { runInSerial, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
-import { compare, compareArray, timeInMinute } from "socket-function/src/misc";
+import { compare, compareArray, isNodeTrue, timeInMinute } from "socket-function/src/misc";
 import { cacheLimited, lazy } from "socket-function/src/caching";
 import { canHaveChildren } from "socket-function/src/types";
 import { shutdown } from "../diagnostics/periodic";
@@ -29,7 +29,7 @@ import { startEdgeNotifier } from "./edgeClientWatcher";
 const UPDATE_POLL_INTERVAL = timeInMinute;
 const DEAD_NODE_COUNT_THRESHOLD = 15;
 
-const edgeNodeStorage = nestArchives("edgenodes/", getArchivesBackblazePublic(getDomain()));
+const edgeNodeStorage = isNodeTrue() && nestArchives("edgenodes/", getArchivesBackblazePublic(getDomain()));
 const edgeNodeIndexFile = "edge-nodes-index.json";
 
 const getEdgeNodeConfig = cacheLimited(10000, async (fileName: string): Promise<EdgeNodeConfig | undefined> => {

package/src/4-dom/qreact.tsx

@@ -2033,7 +2033,7 @@ function blurFixOnMouseDownHack(event: MouseEvent) {
 
     // Looks like we are going to blur, so blur now
     if (selected instanceof HTMLElement && !selected.hasAttribute("data-no-early-blur")) {
-        console.info(`Simulating early blur to prevent unblurred inputs from existing after mousedown. You can use data-no-early-blur to opt-out of this feature`, selected);
+        console.info(`Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
         selected.blur();
     }
 }

package/src/4-querysub/Querysub.ts

@@ -32,7 +32,7 @@ import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
 import debugbreak from "debugbreak";
 import { extractPublicKey, verifyED25519 } from "../-a-auth/ed25519";
 import { registerDynamicResource, registerResource } from "../diagnostics/trackResources";
-import { registerPeriodic, shutdown } from "../diagnostics/periodic";
+import { registerPeriodic, registerPreshutdownHandler, registerShutdownHandler, shutdown } from "../diagnostics/periodic";
 import { sha256 } from "js-sha256";
 import { green, red } from "socket-function/src/formatting/logColors";
 import { minify_sync } from "terser";
@@ -43,7 +43,6 @@ import { measureBlock, measureFnc } from "socket-function/src/profiling/measure"
 import { delay } from "socket-function/src/batching";
 import { MaybePromise } from "socket-function/src/types";
 import { devDebugbreak, getDomain, isDevDebugbreak, isDynamicallyLoading, isNoNetwork, isPublic } from "../config";
-import { hookErrors } from "../diagnostics/errorLogs/hookErrors";
 import { Schema2, Schema2T, t } from "../2-proxy/schema2";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
 import { isDynamicModule } from "../3-path-functions/pathFunctionLoader";
@@ -115,6 +114,7 @@ export class Querysub {
     public static Fragment = qreact.Fragment;
     public static t = t;
     public static shutdown = () => shutdown();
+    public static onShutdown = (fnc: () => Promise<void>) => registerPreshutdownHandler(fnc);
     /**
         IMPORTANT! New schemas must be deployed by calling `yarn deploy`
 
@@ -192,7 +192,8 @@ export class Querysub {
     public static getCallId = () => Querysub.getCallerMachineId() + "_" + Querysub.getCallTime();
     // Returns a random value between 0 and 1. The value depends on the callId, and index (being identical for
     //      the same callid and index).
-    public static callRandom = (index: number) => hashRandom(Querysub.getCallId(), index);
+    public static callRandom = (index = 0) => hashRandom(Querysub.getCallId(), index);
+    public static nextId = (index = 0) => Querysub.getCallId() + "_" + index;
 
     public static configRootDiscoveryLocation = configRootDiscoveryLocation;
 
@@ -275,6 +276,11 @@ export class Querysub {
             watchFunction: fnc
         }));
     }
+    public static commitDelayed = (fnc: () => unknown) => {
+        setImmediate(() => {
+            Querysub.serviceWriteDetached(fnc);
+        });
+    };
 
     public static exit = Querysub.gracefulTerminateProcess;
     public static async gracefulTerminateProcess() {
@@ -286,6 +292,7 @@ export class Querysub {
 
     public static syncedCommit = Querysub.serviceWrite;
     public static commitSynced = Querysub.serviceWrite;
+    public static commitAsync = Querysub.serviceWrite;
     public static async serviceWrite<T>(fnc: () => T) {
         return await proxyWatcher.commitFunction({
             canWrite: true,
@@ -384,7 +391,8 @@ export class Querysub {
     public static getSelfMachineId = getOwnMachineId;
 
     public static getOwnNodeId = getOwnNodeId;
-    public static getSelfNodeId = getOwnMachineId;
+    public static getSelfNodeId = getOwnNodeId;
+    public static getOwnThreadId = getOwnNodeId;
 
     /** Set ClientWatcher.DEBUG_SOURCES to true for to be populated */
     public static getTriggerReason() {
@@ -899,7 +907,14 @@ function getSyncedTime() {
     if (call) {
         return call.runAtTime.time;
     }
+    let watcher = proxyWatcher.getTriggeredWatcherMaybeUndefined();
+    if (watcher?.options.temporary) {
+        return watcher.createTime;
+    }
     if (isNode()) {
+        if (watcher) {
+            throw new Error(`Trying to access time in a serverside non-temporary watcher. Clientside this is allowed, as infinite loops (render every frame) makes sense. Serverside this is not allowed. Did you try to run a call-type operatin in a watcher? If you manually created a watcher, you might want to set "temporary: true" if you will be immediately disposing it. You almost might want to fork your write logic with setImmediate to detach this from your watcher, so your write can access a single non-changing time.`);
+        }
         return Date.now();
     }
     initTimeLoop();
@@ -953,9 +968,12 @@ setImmediate(() => {
     });
 });
 
-setImmediate(() => {
-    hookErrors();
-});
+if (isNode()) {
+    setImmediate(async () => {
+        const { hookErrors } = await import("../diagnostics/errorLogs/hookErrors");
+        hookErrors();
+    });
+}
 
 setImmediate(async () => {
     // Import, so it registers addStatPeriodic

package/src/5-diagnostics/Table.tsx

@@ -32,6 +32,7 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
     // Line and character limits before we cut off the inner content
     lineLimit?: number;
     characterLimit?: number;
+    dark?: boolean;
 }> {
     state = {
         limit: this.props.initialLimit || 100,
@@ -47,7 +48,11 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
 
         return (
             <table class={css.borderCollapse("collapse") + this.props.class}>
-                <tr class={css.position("sticky").top(0).hsla(0, 0, 95, 0.9)}>
+                <tr class={
+                    css.position("sticky").top(0)
+                    + (this.props.dark && css.hsla(0, 0, 10, 0.9))
+                    + (!this.props.dark && css.hsla(0, 0, 95, 0.9))
+                }>
                     <th class={css.whiteSpace("nowrap")}>⧉ {allRows.length}</th>
                     {Object.entries(columns).filter(x => x[1] !== null).map(([columnName, column]) =>
                         <th class={css.pad2(8, 4) + cellClass}>{column?.title || toSpaceCase(columnName)}</th>

package/src/archiveapps/archiveMergeEntry.tsx

@@ -10,6 +10,7 @@ const MERGE_DELAY = timeInHour;
 
 async function mergeFiles() {
     await runArchiveMover({
+        debugName: "merge",
         outputType: "merge",
         async runMover(config) {
             let values = config.values;

package/src/diagnostics/errorLogs/ErrorLogController.ts

@@ -19,7 +19,6 @@ import { atomic, doAtomicWrites } from "../../2-proxy/PathValueProxyWatcher";
 import { isClient } from "../../config2";
 import { ControllerPick, SocketRegistered } from "socket-function/SocketFunctionTypes";
 
-
 const NOTIFY_HISTORY = timeInDay * 7;
 // Only error / fatal are tracked, for now... we might also add warns?
 const logIssueNotifyTypes: LogType[] = ["error", "fatal"];

package/src/diagnostics/errorLogs/LogTimeSelector.tsx

@@ -14,7 +14,6 @@ import { ButtonSelector } from "../../library-components/ButtonSelector";
 import { Icon } from "../../library-components/icons";
 import { MeasuredDiv } from "../../library-components/MeasuredDiv";
 
-
 type RangeSize = "lastYear" | "lastMonth" | "lastWeek" | "lastDay" | "lastHour";
 export let logNowTime = Date.now();
 const logFilterStart = new URLParam<number | RangeSize>("logFilterStartTime", "lastWeek");

package/src/diagnostics/managementPages.tsx

@@ -18,7 +18,6 @@ import { getNextTime } from "../0-path-value-core/pathValueCore";
 import { encodeArgs } from "../3-path-functions/PathFunctionHelpers";
 import { CallSpec, getCurrentCall, getCurrentCallAllowUndefined, overrideCurrentCall } from "../3-path-functions/PathFunctionRunner";
 import { createLazyComponent } from "../library-components/LazyComponent";
-import { LogNotify } from "./errorLogs/LogNotify";
 import { isTrusted } from "../-d-trust/NetworkTrust2";
 import { devDebugbreak, getDomain } from "../config";
 import { getCallWrites } from "../4-querysub/querysubPrediction";
@@ -27,7 +26,9 @@ import * as hooks from "../-0-hooks/hooks";
 import { addComponentUI } from "../5-diagnostics/qreactDebug";
 import { addComponentButton } from "../5-diagnostics/qreactDebug";
 import { closeAllModals } from "../5-diagnostics/Modal";
+import { delay } from "socket-function/src/batching";
 
+const LogNotify = createLazyComponent(() => import("./errorLogs/LogNotify"))("LogNotify");
 
 export const managementPageURL = new URLParam("managementpage", "");
 export const showingManagementURL = new URLParam("showingmanagement", false);
@@ -134,14 +135,21 @@ export async function registerManagementPages2(config: {
     __schema = config.schema;
 
     setImmediate(async () => {
-        if (!isNode() && !await ManagementController.nodes[getBrowserUrlNode()].isManagementUser()) return;
+        if (!isNode() &&
+            !await ManagementController.nodes[getBrowserUrlNode()].isManagementUser()
+            && new URLSearchParams(window.location.search).get("dropto") !== "user"
+        ) return;
+
+        // Wait, so the import system knows the modules are async imports
+        await delay(0);
+
+        const ComponentSyncStats = createLazyComponent(() => import("./misc-pages/ComponentSyncStats"))("ComponentSyncStats");
 
-        const { filtersURL } = await import("./misc-pages/archiveViewerShared");
-        const { watchValueType } = await import("./misc-pages/LocalWatchViewer");
-        const { ComponentSyncStats } = await import("./misc-pages/ComponentSyncStats");
         addComponentButton({
             title: "View in Path Navigator",
-            callback(component) {
+            async callback(component) {
+                const { filtersURL } = await import("./misc-pages/archiveViewerShared");
+                const { watchValueType } = await import("./misc-pages/LocalWatchViewer");
                 filtersURL.value[nextId()] = { column: "path", match: component.debugName, type: "and" };
                 managementPageURL.value = "LocalWatchViewer";
                 watchValueType.value = "component";
@@ -168,6 +176,8 @@ export async function registerManagementPages2(config: {
     }
 
     if (isServer()) {
+        // Wait, so the import system knows the modules are async imports
+        await delay(0);
         for (let page of inputPages) {
             // NOTE: If we split this into a module for component/controller, we need to make sure we
             //  import both serverside, so we can whitelist them for import clientside.

package/src/diagnostics/periodic.ts

@@ -1,6 +1,6 @@
-import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isNode, timeInMinute } from "socket-function/src/misc";
-import { logErrors } from "../errors";
+import { logErrors, timeoutToError } from "../errors";
 import debugbreak from "debugbreak";
 import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
 import { authorityStorage } from "../0-path-value-core/pathValueCore";
@@ -15,6 +15,10 @@ let shutdownHandlers: (() => Promise<void>)[] = [];
 export function registerShutdownHandler(fnc: () => Promise<void>) {
     shutdownHandlers.push(fnc);
 }
+let preshutdownHandlers: (() => Promise<void>)[] = [];
+export function registerPreshutdownHandler(fnc: () => Promise<void>) {
+    preshutdownHandlers.push(fnc);
+}
 
 function logAll() {
     for (let fnc of periodicFncs) {
@@ -24,8 +28,20 @@ function logAll() {
 
 logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
 
+let shuttingDown = false;
 export async function shutdown() {
+    if (shuttingDown) {
+        return;
+    }
+    shuttingDown = true;
     const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
+    for (let fnc of preshutdownHandlers) {
+        try {
+            await timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`));
+        } catch (e) {
+            console.log(`Error on preshutdown handler ${fnc.name}`, e);
+        }
+    }
     try {
         await authorityStorage.onShutdown();
         await nodeDiscoveryShutdown();
@@ -34,11 +50,13 @@ export async function shutdown() {
     }
     for (let fnc of shutdownHandlers) {
         try {
-            await fnc();
+            await timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`));
         } catch (e) {
             console.log(`Error on shutdown handler ${fnc.name}`, e);
         }
     }
+    // Wait to allow any logged errors to hopefully be written somewhere?
+    await delay(2000);
     process.exit();
 }
 

package/src/server.ts

@@ -14,7 +14,6 @@ import { AuthorityPath, pathValueAuthority2 } from "./0-path-value-core/NodePath
 import { Querysub } from "./4-querysub/QuerysubController";
 import { ClientWatcher } from "./1-path-client/pathValueClientWatcher";
 import { delay } from "socket-function/src/batching";
-import yargs from "yargs";
 import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
 import { green } from "socket-function/src/formatting/logColors";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -26,6 +25,7 @@ import { getOwnThreadId } from "./-f-node-discovery/NodeDiscovery";
 import { getOurAuthorities } from "./config2";
 import { devDebugbreak, isPublic } from "./config";
 
+import yargs from "yargs";
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
     .option("verbose", { type: "boolean", desc: "Log all writes and reads" })

package/src/user-implementation/LoginPage.tsx

@@ -1,7 +1,7 @@
 import preact from "preact"; import { qreact } from "../4-dom/qreact";
 import { Button } from "../library-components/Button";
 import { css } from "../4-dom/css";
-import { loginTokenURL, user_data, user_functions } from "./userData";
+import { getUserId, getUserIdAllowUndefined, getUserObj, loginTokenURL, user_data, user_functions } from "./userData";
 import { Querysub } from "../4-querysub/Querysub";
 import { isNode, list } from "socket-function/src/misc";
 import { InputLabel } from "../library-components/InputLabel";
@@ -16,7 +16,7 @@ export class LoginPage extends qreact.Component {
     render() {
         const login = () => {
             user_functions.sendLoginEmail({
-                userId: this.state.email,
+                email: this.state.email,
                 redirectURL: window.location.href,
                 loginTokenRandomness: Buffer.from(list(32).map(() => Math.random() * 256)).toString("base64"),
             });
@@ -54,6 +54,22 @@ export class LoginPage extends qreact.Component {
                         </div>
                     </div>
                 }
+                {getUserIdAllowUndefined() &&
+                    <div class={css.hbox(10).alignSelf("center").maxWidth("50vw")}>
+                        <div class={css.fontSize(20).hslcolor(110, 75, 75)}>
+                            You are logged in as {getUserObj().email}.
+                        </div>
+                        <button onClick={() => {
+                            let call = user_functions.logoutCurrent();
+                            void Querysub.onCommitPredictFinished(call).finally(async () => {
+                                await Querysub.gracefulTerminateProcess();
+                                location.reload();
+                            });
+                        }}>
+                            Logout
+                        </button>
+                    </div>
+                }
             </div>
         );
     }

package/src/user-implementation/UserPage.tsx

@@ -268,7 +268,7 @@ export class UserPage extends qreact.Component {
                         </table>
                     </div>
                     <div>
-                        <div class={css.fontSize(20)}>Latest Page Load By Machine/IP</div>
+                        <div class={css.fontSize(20)}>Latest Access by Machine/IP</div>
                         <table>
                             <tr>
                                 <th>Source</th>
@@ -293,7 +293,7 @@ export class UserPage extends qreact.Component {
                             <h2>Example Login Email</h2>
                             <div key="email" ref2={e => {
                                 e.innerHTML = renderToString(generateLoginEmail({
-                                    userId,
+                                    email: "user@example.com",
                                     loginToken: "loginToken",
                                     ip: "127.0.0.1",
                                     machineId: getOwnMachineId(),

package/src/user-implementation/addSuperUser.ts

@@ -14,12 +14,11 @@ async function main() {
     const email = yargObj.email;
     if (!email) throw new Error("No email provided." + howToCall);
     if (!email.includes("@")) throw new Error(`Invalid email ${email}.` + howToCall);
-    const userId = email;
 
     await Querysub.hostService("addSuperUser");
 
     await Querysub.commitSynced(() => {
-        scriptCreateUser({ userId });
+        scriptCreateUser({ email });
     });
 
     await pathValueCommitter.waitForValuesToCommit();

package/src/user-implementation/loginEmail.tsx

@@ -4,7 +4,7 @@ import { list } from "socket-function/src/misc";
 import { hslToRGB } from "socket-function/src/formatting/colors";
 import { joinVNodes } from "../misc";
 export function generateLoginEmail(config: {
-    userId: string;
+    email: string;
     loginToken: string;
     ip: string;
     machineId: string;
@@ -12,10 +12,10 @@ export function generateLoginEmail(config: {
     timeoutTime: number;
     noHTMLWrapper?: boolean;
 }): { subject: string; contents: preact.VNode } {
-    const { userId, loginToken, ip, machineId, redirectURL } = config;
+    const { loginToken, ip, machineId, redirectURL, email } = config;
     let url = new URL(redirectURL);
     url.searchParams.set(loginTokenURL.urlKey, loginToken);
-    const subject = `${userId} Access Request (${new Date().toLocaleString()})`;
+    const subject = `${email} Access Request (${new Date().toLocaleString()})`;
     function center(jsx: preact.ComponentChild) {
         return (
             <table style={{ width: "100%" }}>
@@ -79,7 +79,7 @@ export function generateLoginEmail(config: {
             </table>
             {line(<div style={{ color: "rgb(255, 255, 255)" }}>
                 {/* Mess up the email so it isn't turned into a link by gmail. */}
-                Greetings <b>{userId.replaceAll(".", ".​")}</b>. You have requested access to <a href={redirectURL} style={{ color: "rgb(153, 204, 255)" }}>{new URL(redirectURL).hostname}</a>.
+                Greetings <b>{email.replaceAll(".", ".​")}</b>. You have requested access to <a href={redirectURL} style={{ color: "rgb(153, 204, 255)" }}>{new URL(redirectURL).hostname}</a>.
             </div>)}
             {line(<div style={{ color: "rgb(255, 255, 255)" }}>
                 If you did not make this request, please disregard this email. Your account is safe and no action is required.
@@ -114,11 +114,11 @@ export function generateLoginEmail(config: {
                 <td {...{ "nowrap": true }} style={{ paddingLeft: "40px", }}></td>
             </tr></tbody></table>
             <table><tbody><tr><td style={{ paddingTop: "20px" }}></td></tr></tbody></table>
-            {line(<div style={{ color: "rgb(140, 140, 140)", width: "100%" }}><b>Do NOT share or forward this email with anyone.</b></div>)}
+            {line(<div style={{ color: "rgb(140, 140, 140)", width: "100%" }}><b>Do NOT share or forward this email to anyone.</b></div>)}
             <table><tbody><tr><td style={{ paddingTop: "10px" }}></td></tr></tbody></table>
             {line(<div style={{ color: "rgb(140, 140, 140)", width: "100%" }}>
-                This link will only work with the current device and browser.
-                This link can only be used one time, until {new Date(config.timeoutTime).toLocaleString()}.
+                This link will only work in the current device and browser.
+                This link can only be used one time, and will expire at {new Date(config.timeoutTime).toLocaleString()}.
             </div>)}
         </>
     );

package/src/user-implementation/userData.ts

@@ -1,7 +1,7 @@
 import { atomic, atomicObjectWrite } from "../2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { red } from "socket-function/src/formatting/logColors";
-import { isNode, timeInHour } from "socket-function/src/misc";
+import { isNode, sha256Hash, timeInHour } from "socket-function/src/misc";
 import { registerAliveChecker } from "../2-proxy/garbageCollection";
 import { generateLoginEmail } from "./loginEmail";
 import { logErrors } from "../errors";
@@ -15,6 +15,7 @@ import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
 import { delay } from "socket-function/src/batching";
 import { enableErrorNotifications } from "../library-components/errorNotifications";
 import { clamp } from "../misc";
+import { sha256 } from "js-sha256";
 
 /*
     IMPORTANT!
@@ -186,6 +187,10 @@ const { data, functions } = Querysub.syncSchema<{
         claimedDisplayNames: {
             [displayName: string]: string;
         };
+
+        emailToUserId: {
+            [email: string]: string;
+        };
     };
     machineSecure: {
         [machineId: string]: {
@@ -332,6 +337,7 @@ export function wildcard0Owner(config: PermissionsParameters) {
 }
 
 function isAllowedToBeUser(userId: string) {
+    userId = String(userId);
     if (isClient()) {
         return data().users[userId].userId === userId;
     }
@@ -532,12 +538,20 @@ function internalCreateUser(config: {
     return data().users[userId];
 }
 
+function createNewUserId() {
+    let randBuffer = Buffer.from(sha256(Querysub.nextId()), "hex");
+    const alphabet = "abcdefghijklmnopqrstuvwxyz";
+    return Array.from(randBuffer)
+        .map(x => alphabet[x % alphabet.length])
+        .join("")
+        .slice(0, 16);
+}
 function sendLoginEmail(config: {
-    userId: string;
+    email: string;
     redirectURL: string;
     loginTokenRandomness: string;
 }) {
-    const { userId, redirectURL } = config;
+    const { email, redirectURL } = config;
     {
         let redirectObj = new URL(redirectURL);
         redirectObj.port = "443";
@@ -547,19 +561,24 @@ function sendLoginEmail(config: {
         }
     }
     Querysub.ignorePermissionsChecks(() => {
-        if (!userId.includes("@")) {
-            throw new Error(`Invalid email ${userId}`);
+        if (!email.includes("@")) {
+            throw new Error(`Invalid email ${email}`);
         }
-        const email = userId;
+        let userId = atomic(data().secure.emailToUserId[email]) || createNewUserId();
 
         const signupError = `Signups are currently closed, and user doesn't presently exist. ${userId}. Add users with "yarn addsuperuser <email>"`;
         // NOTE: Using the same errors message makes it a BIT harder to know if you are blocked, so attackers are
         //  less likely to change their IP, and more likely to give up.
         throwIfBlockedIP(signupError);
-        if (!atomic(data().secure.signupsOpen) && !(userId in data().users)) {
+        if (!atomic(data().secure.signupsOpen) && (!userId || !(userId in data().users))) {
             throw new Error(signupError);
         }
 
+        // Create user if it doesn't exist
+        if (!(userId in data().users)) {
+            internalCreateUser({ userId, email });
+        }
+
         const ip = Querysub.getCallerIP();
         const machineId = Querysub.getCallerMachineId();
         const now = Querysub.getCallTime();
@@ -605,13 +624,6 @@ function sendLoginEmail(config: {
             }
         }
 
-        // Create user if it doesn't exist
-        {
-            if (!(userId in data().users)) {
-                internalCreateUser({ userId, email });
-            }
-        }
-
         let userObj = data().users[userId];
 
         // Per email/user throttle
@@ -644,7 +656,7 @@ function sendLoginEmail(config: {
         // NOTE: We don't track failed calls. Those should be tracked at a framework level and displayed on another page.
         logActivity({ type: "sendLoginEmail", fields: { userId, ip, machineId, redirectURL } });
         const { subject, contents } = generateLoginEmail({
-            userId, loginToken, ip, machineId, redirectURL,
+            email, loginToken, ip, machineId, redirectURL,
             timeoutTime,
         });
         Querysub.onCommitFinished(async () => {
@@ -756,35 +768,37 @@ function registerPageLoadTime() {
 }
 
 function inviteUser(config: { email: string }) {
-    let curUserObj = getUserObjAssert();
-    if (config.email in curUserObj.invitedUsers) {
-        console.info(`User ${config.email} already invited`);
-        return;
-    }
-    // If the user already exists, don't invite
-    const { email } = config;
-    let userId = email;
-    if (userId in data().users) {
-        console.info(`User ${userId} already exists, no need to invite`);
-        return;
-    }
+    Querysub.ignorePermissionsChecks(() => {
+        let curUserObj = getUserObjAssert();
+        if (config.email in curUserObj.invitedUsers) {
+            console.info(`User ${config.email} already invited`);
+            return;
+        }
+        // If the user already exists, don't invite
+        const { email } = config;
+        let userId = atomic(data().secure.emailToUserId[email]) || createNewUserId();
+        if (userId in data().users) {
+            console.info(`User ${userId} already exists, no need to invite`);
+            return;
+        }
 
-    const invitesRemaining = Number(curUserObj.invitesRemaining);
-    if (invitesRemaining <= 0) {
-        throw new Error("No invites remaining");
-    }
+        const invitesRemaining = Number(curUserObj.invitesRemaining);
+        if (invitesRemaining <= 0) {
+            throw new Error("No invites remaining");
+        }
 
-    curUserObj.invitedUsers[config.email] = atomicObjectWrite({
-        time: Querysub.getCallTime(),
-    });
-    curUserObj.invitesRemaining--;
+        curUserObj.invitedUsers[config.email] = atomicObjectWrite({
+            time: Querysub.getCallTime(),
+        });
+        curUserObj.invitesRemaining--;
 
-    if (!(userId in data().users)) {
-        // Create the user if they don't exist
-        let newUserObj = internalCreateUser({ userId, email });
-        newUserObj.invitedBy = curUserObj.userId;
-    }
-    logActivity({ type: "inviteUser", fields: { inviter: curUserObj.userId, userId, } });
+        if (!(userId in data().users)) {
+            // Create the user if they don't exist
+            let newUserObj = internalCreateUser({ userId, email });
+            newUserObj.invitedBy = curUserObj.userId;
+        }
+        logActivity({ type: "inviteUser", fields: { inviter: curUserObj.userId, userId, } });
+    });
 }
 
 function logoutFromMachine(config: { machineId: string; userId: string }) {
@@ -888,11 +902,11 @@ function specialSetUserType(config: { userId: string; userType: UserType; }) {
 
 
 export function scriptCreateUser(config: {
-    userId: string;
+    email: string;
 }) {
-    const { userId } = config;
-    let email = userId;
-    data().users[config.userId] = {
+    const { email } = config;
+    let userId = createNewUserId();
+    data().users[userId] = {
         userId,
         email,
         settings: { displayName: userId },
@@ -905,6 +919,7 @@ export function scriptCreateUser(config: {
         invitedUsers: {},
         invitesRemaining: 0,
     };
+    data().secure.emailToUserId[email] = userId;
 }
 
 function setPostmarkAPIKey(config: { apiKey: string; }) {