querysub 0.157.0 → 0.158.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.157.0",
+  "version": "0.158.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
     "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
     "pako": "^2.1.0",
     "preact": "^10.11.3",
-    "socket-function": "^0.89.0",
+    "socket-function": "^0.90.0",
     "terser": "^5.31.0",
     "typesafecss": "^0.6.3",
     "yaml": "^2.5.0",

package/src/-b-authorities/cloudflareHelpers.ts

@@ -4,15 +4,16 @@ import { lazy } from "socket-function/src/caching";
 import { getStorageDir, getSubFolder } from "../fs";
 import fs from "fs";
 import { isNodeTrue } from "socket-function/src/misc";
-export const keys = isNodeTrue() && getArchives("keys");
+
+export const keys = lazy(() => getArchives("keys"));
 
 export const getCloudflareCreds = lazy(async (): Promise<{ key: string; email: string }> => {
-    let credsJSON = await keys.get("cloudflare.json");
+    let credsJSON = await keys().get("cloudflare.json");
     if (!credsJSON) {
         let localPath = getStorageDir() + "cloudflare.json";
         if (fs.existsSync(localPath)) {
             credsJSON = fs.readFileSync(localPath);
-            await keys.set("cloudflare.json", credsJSON);
+            await keys().set("cloudflare.json", credsJSON);
         } else {
             throw new Error(`b2:/keys/cloudflare.json is missing. It should contain { "key": "your-key", "email": "your-email" }`);
         }

package/src/-b-authorities/dnsAuthority.ts

@@ -16,8 +16,6 @@ const DNS_TTLSeconds = {
     "A": 60,
 };
 
-export const keys = isNodeTrue() && getArchives("keys");
-
 export const hasDNSWritePermissions = lazy(async () => {
     if (!isNode()) return false;
     if (isClient()) return false;

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -6,8 +6,9 @@ import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors"
 import { green, magenta } from "socket-function/src/formatting/logColors";
 import { devDebugbreak } from "../../config";
 import { sort } from "socket-function/src/misc";
+import { lazy } from "socket-function/src/caching";
 
-const snapshots = getArchives("snapshots");
+const snapshots = lazy(() => getArchives("snapshots"));
 
 export type ArchiveSnapshotOverview = {
     // If live, it isn't a real file that can be loaded
@@ -54,7 +55,7 @@ export async function saveSnapshot(config: {
 }) {
     let { files } = config;
     let overview = getSnapshotOverview(files);
-    await snapshots.set(overview.file, Buffer.from(files.join("\n")));
+    await snapshots().set(overview.file, Buffer.from(files.join("\n")));
 }
 function overviewToFileName(overview: Omit<ArchiveSnapshotOverview, "file">): string {
     return Object.entries({
@@ -84,7 +85,7 @@ function fileNameToOverview(fileName: string): ArchiveSnapshotOverview {
 
 // pathValueArchives.decodeDataPath
 export async function getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
-    let snapshotFiles = await snapshots.find("");
+    let snapshotFiles = await snapshots().find("");
 
     let overview: ArchiveSnapshotOverview[] = [];
     let locker = await pathValueArchives.getArchiveLocker();
@@ -128,7 +129,7 @@ async function getSnapshotBase(snapshotFile: string | "live") {
             files: allFiles.map(x => x.file),
         };
     }
-    let data = await snapshots.get(snapshotFile);
+    let data = await snapshots().get(snapshotFile);
     if (!data) {
         throw new Error(`Snapshot not found: ${snapshotFile}`);
     }
@@ -158,7 +159,7 @@ export async function loadSnapshot(config: {
     console.log(magenta(`Loading snapshot: ${overview.file}`));
     let locker = await pathValueArchives.getArchiveLocker();
 
-    let snapshotData = await snapshots.get(overview.file);
+    let snapshotData = await snapshots().get(overview.file);
     if (!snapshotData) {
         throw new Error(`Snapshot not found: ${overview.file}`);
     }

package/src/0-path-value-core/pathValueCore.ts

@@ -1768,13 +1768,15 @@ class WriteValidStorage {
             let authorityValue = authorityStorage.getValueExactIgnoreInvalid(write.path, write.time);
             if (authorityValue) {
                 authorityValue.valid = write.isValid;
-                if (!isCoreQuiet) {
-                    console.log(`Setting valid state of ${debugPathValuePath(authorityValue)} to ${write.isValid}`);
-                }
+                // NOTE: I think it is fine now for us to set the valid state early? Hopefully...
+                //  because it happens A LOT.
+                // if (!isCoreQuiet) {
+                //     console.log(`Setting valid state of ${debugPathValuePath(authorityValue)} to ${write.isValid}`);
+                // }
             } else {
-                if (isNode()) {
-                    console.error(`Setting valid state of ${write.path}@${debugTime(write.time)} to ${write.isValid}, but the ValuePath was not found. If the ValuePath is found later, it might not have the valid state set correctly.`);
-                }
+                // if (isNode()) {
+                //     console.error(`Setting valid state of ${write.path}@${debugTime(write.time)} to ${write.isValid}, but the ValuePath was not found. If the ValuePath is found later, it might not have the valid state set correctly.`);
+                // }
             }
         }
 

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -162,6 +162,9 @@ interface WatcherOptions<Result> {
     // Fairly self explanatory. If there are nested createWatchers, instead of forking, we just
     //  run them immediately.
     inlineNestedWatchers?: boolean;
+
+    // Temporary indicates after becoming synchronizes it will immediately dispose itself
+    temporary?: boolean;
 }
 
 let harvestableReadyLoopCount = 0;
@@ -286,6 +289,8 @@ export type SyncWatcher = {
     tag: SyncWatcherTag;
 
     hackHistory: { message: string; time: number }[];
+
+    createTime: number;
 }
 function addToHistory(watcher: SyncWatcher, message: string) {
     watcher.hackHistory.push({ message, time: Date.now() });
@@ -989,6 +994,7 @@ export class PathValueProxyWatcher {
             tag: new SyncWatcherTag(),
 
             hackHistory: [],
+            createTime: Date.now(),
         };
         const SHOULD_TRACE = this.SHOULD_TRACE(watcher);
         const proxy = this.proxy;
@@ -1205,11 +1211,13 @@ export class PathValueProxyWatcher {
                         debugger;
                         // NOTE: Using forceEqualWrites will also fix this a lot of the time, such as when
                         //  a write contains random numbers or dates.
-                        let errorMessage = `Too many attempts to sync with different values. If you are reading in a loop, make sure to read all the values, instead of aborting the loop if a value is not synced. ALSO, make sure you don't access paths with Math.random() or Date.now(). This will prevent the sync loop from ever stabilizing.`;
+                        let errorMessage = `Too many attempts (${watcher.countSinceLastFullSync}) to sync with different values. If you are reading in a loop, make sure to read all the values, instead of aborting the loop if a value is not synced. ALSO, make sure you don't access paths with Math.random() or Date.now(). This will prevent the sync loop from ever stabilizing.`;
                         if (specialPromiseUnsynced) {
                             errorMessage += ` A promise is being accessed, so it is possible triggerOnPromiseFinish is being used on a new promise every loop, which cannot work (you MUST cache MaybePromise and replace the value with a non-promise, otherwise it will never be available synchronously!)`;
                         }
+                        errorMessage += ` (${watcher.debugName}})`;
                         console.error(red(errorMessage));
+                        logUnsynced();
                         result = { error: new Error(errorMessage).stack || "" };
                         // Force the watches to be equal, so we stop looping
                         watcher.pendingWatches = watcher.lastWatches;
@@ -1648,6 +1656,7 @@ export class PathValueProxyWatcher {
         let anyWrites = false;
         this.createWatcher({
             ...options,
+            temporary: true,
             onResultUpdated: (result, writes, watcher) => {
                 if (writes?.length) {
                     anyWrites = true;
@@ -1700,6 +1709,7 @@ export class PathValueProxyWatcher {
             ...options,
             canWrite: true,
             dryRun: true,
+            temporary: true,
             onResultUpdated: (result, values, watcher) => {
                 watcher.dispose();
                 if ("error" in result) {

package/src/2-proxy/schema2.ts

@@ -199,6 +199,10 @@ export class Schema2Fncs {
             }
             return { defaultValue: def.defaultValue, };
         }
+        // OR, if it is an optional object, then the default IS undefined
+        if (def?.isMaybeUndefined) {
+            return { defaultValue: undefined };
+        }
         return undefined;
     }
     public static isObject(schema: Schema2, path: string[]): boolean {
@@ -293,6 +297,7 @@ type InternalTypeDef = {
 
     isObject?: boolean;
     isObjectGC?: boolean;
+    isMaybeUndefined?: boolean;
 
     isNotAtomic?: boolean;
 
@@ -340,6 +345,9 @@ function typeDefTypeToInternalType(
         isObject = true;
         isObjectGC = false;
     }
+    if (typeDef.path.includes("optionalObject") || typeDef.path.includes("optionalObjectNoGC")) {
+        rootResult.isMaybeUndefined = true;
+    }
 
     let undoRegions = typeDef.pathWithCalls.filter(x => x.key === "undoRegionObject");
     for (let undoRegion of undoRegions) {
@@ -370,6 +378,7 @@ function typeDefTypeToInternalType(
         nestedValues.isObjectGC = isObjectGC || isLookupGC;
         nestedValues.gcDelay = Number(typeDef.pathWithCalls.find(x => x.key === "gcDelay")?.callParams?.[0]) || undefined;
 
+
         if (isLookup) {
             let undoRegions = typeDef.pathWithCalls.filter(x => x.key === "undoRegion");
             for (let undoRegion of undoRegions) {

package/src/3-path-functions/PathFunctionRunner.ts

@@ -550,7 +550,7 @@ export class PathFunctionRunner {
                             }
                         }
                         if (runCount > PathFunctionRunner.MAX_WATCH_LOOPS) {
-                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}`);
+                            throw new Error(`MAX_WATCH_LOOPS exceeded for ${getDebugName(callPath, functionSpec, true)}. All accesses have to be consistent. So Querysub.time() instead of Date.now() and Querysub.nextId() instead of nextId() / Math.random(). If you need multiple random numbers, keep track of an index, and pass it to Querysub.nextId() for the nth random number.`);
                         }
 
                         // We NEED to depend on the function spec, so spec updates can be atomic (which is needed to deploy an update

package/src/3-path-functions/syncSchema.ts

@@ -140,14 +140,25 @@ export type PermissionsParameters = {
 };
 /** A false of false will deny read permissions, resulting in all reads being given value with a value
  *      of undefined, and a time of 0.
- *  NOTE: Permissions checks at the root (permissions: { READ_PERMISSIONS(config) { } }) will
- *      be used to verify calls are allowed as well, even if the call doesn't read or write any data
- *      that is under a permissions check.
+ 
  */
 export type PermissionsCallback = (config: PermissionsParameters) => PermissionsCheckResult;
 export type PermissionsCheckResult = boolean | { allowed: boolean; skipParentChecks?: boolean; };
 /*
     NOTE: All ancestor permissions checks are applied as well.
+        - This means if the ancestor disallows, it blocks any descendant checks even if they are allowed.
+
+    NOTE: If no permissions match, then access is disallowed by default.
+
+    NOTE: Permissions checks at the root (permissions: { READ_PERMISSIONS(config) { } }) will
+        be used to verify calls are allowed as well, even if the call doesn't read or write any data
+        that is under a permissions check.
+        - THIS MEANS that if there is no permission check at the root, calls won't be allowed by any user
+            (as we default to disallow).
+
+    NOTE: Servers watches ignore permissions, as they directly communicate with the underlying PathValueServer,
+        skipping the ProxyServer.
+        - However function calls by servers are still run by FunctionRunner, which will apply permission checks.
 
     IMPORTANT! Wildcards are tested via the "" path. This means if the "" key is provided access to a user, they can run
         Object.keys()/Object.values() on the data. In order to not accidentally provide list access, never allow "" to be

package/src/4-dom/qreact.tsx

@@ -2033,7 +2033,7 @@ function blurFixOnMouseDownHack(event: MouseEvent) {
 
     // Looks like we are going to blur, so blur now
     if (selected instanceof HTMLElement && !selected.hasAttribute("data-no-early-blur")) {
-        console.info(`Simulating early blur to prevent unblurred inputs from existing after mousedown. You can use data-no-early-blur to opt-out of this feature`, selected);
+        console.info(`Simulating early blur to prevent blur from firing after mousedown. This solves a problem where mousedown changes the UI, and then the blur fires on the wrong element. You can use data-no-early-blur to opt-out of this feature`, selected);
         selected.blur();
     }
 }

package/src/4-querysub/Querysub.ts

@@ -32,7 +32,7 @@ import { pathValueCommitter } from "../0-path-value-core/PathValueCommitter";
 import debugbreak from "debugbreak";
 import { extractPublicKey, verifyED25519 } from "../-a-auth/ed25519";
 import { registerDynamicResource, registerResource } from "../diagnostics/trackResources";
-import { registerPeriodic, shutdown } from "../diagnostics/periodic";
+import { registerPeriodic, registerPreshutdownHandler, registerShutdownHandler, shutdown } from "../diagnostics/periodic";
 import { sha256 } from "js-sha256";
 import { green, red } from "socket-function/src/formatting/logColors";
 import { minify_sync } from "terser";
@@ -114,6 +114,7 @@ export class Querysub {
     public static Fragment = qreact.Fragment;
     public static t = t;
     public static shutdown = () => shutdown();
+    public static onShutdown = (fnc: () => Promise<void>) => registerPreshutdownHandler(fnc);
     /**
         IMPORTANT! New schemas must be deployed by calling `yarn deploy`
 
@@ -191,7 +192,8 @@ export class Querysub {
     public static getCallId = () => Querysub.getCallerMachineId() + "_" + Querysub.getCallTime();
     // Returns a random value between 0 and 1. The value depends on the callId, and index (being identical for
     //      the same callid and index).
-    public static callRandom = (index: number) => hashRandom(Querysub.getCallId(), index);
+    public static callRandom = (index = 0) => hashRandom(Querysub.getCallId(), index);
+    public static nextId = (index = 0) => Querysub.getCallId() + "_" + index;
 
     public static configRootDiscoveryLocation = configRootDiscoveryLocation;
 
@@ -274,6 +276,11 @@ export class Querysub {
             watchFunction: fnc
         }));
     }
+    public static commitDelayed = (fnc: () => unknown) => {
+        setImmediate(() => {
+            Querysub.serviceWriteDetached(fnc);
+        });
+    };
 
     public static exit = Querysub.gracefulTerminateProcess;
     public static async gracefulTerminateProcess() {
@@ -285,6 +292,7 @@ export class Querysub {
 
     public static syncedCommit = Querysub.serviceWrite;
     public static commitSynced = Querysub.serviceWrite;
+    public static commitAsync = Querysub.serviceWrite;
     public static async serviceWrite<T>(fnc: () => T) {
         return await proxyWatcher.commitFunction({
             canWrite: true,
@@ -383,7 +391,8 @@ export class Querysub {
     public static getSelfMachineId = getOwnMachineId;
 
     public static getOwnNodeId = getOwnNodeId;
-    public static getSelfNodeId = getOwnMachineId;
+    public static getSelfNodeId = getOwnNodeId;
+    public static getOwnThreadId = getOwnNodeId;
 
     /** Set ClientWatcher.DEBUG_SOURCES to true for to be populated */
     public static getTriggerReason() {
@@ -898,7 +907,14 @@ function getSyncedTime() {
     if (call) {
         return call.runAtTime.time;
     }
+    let watcher = proxyWatcher.getTriggeredWatcherMaybeUndefined();
+    if (watcher?.options.temporary) {
+        return watcher.createTime;
+    }
     if (isNode()) {
+        if (watcher) {
+            throw new Error(`Trying to access time in a serverside non-temporary watcher. Clientside this is allowed, as infinite loops (render every frame) makes sense. Serverside this is not allowed. Did you try to run a call-type operatin in a watcher? If you manually created a watcher, you might want to set "temporary: true" if you will be immediately disposing it. You almost might want to fork your write logic with setImmediate to detach this from your watcher, so your write can access a single non-changing time.`);
+        }
         return Date.now();
     }
     initTimeLoop();

package/src/5-diagnostics/Table.tsx

@@ -32,6 +32,7 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
     // Line and character limits before we cut off the inner content
     lineLimit?: number;
     characterLimit?: number;
+    dark?: boolean;
 }> {
     state = {
         limit: this.props.initialLimit || 100,
@@ -47,7 +48,11 @@ export class Table<RowT extends RowType> extends qreact.Component<TableType<RowT
 
         return (
             <table class={css.borderCollapse("collapse") + this.props.class}>
-                <tr class={css.position("sticky").top(0).hsla(0, 0, 95, 0.9)}>
+                <tr class={
+                    css.position("sticky").top(0)
+                    + (this.props.dark && css.hsla(0, 0, 10, 0.9))
+                    + (!this.props.dark && css.hsla(0, 0, 95, 0.9))
+                }>
                     <th class={css.whiteSpace("nowrap")}>⧉ {allRows.length}</th>
                     {Object.entries(columns).filter(x => x[1] !== null).map(([columnName, column]) =>
                         <th class={css.pad2(8, 4) + cellClass}>{column?.title || toSpaceCase(columnName)}</th>

package/src/diagnostics/periodic.ts

@@ -1,6 +1,6 @@
-import { runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
+import { delay, runInfinitePoll, runInfinitePollCallAtStart } from "socket-function/src/batching";
 import { isNode, timeInMinute } from "socket-function/src/misc";
-import { logErrors } from "../errors";
+import { logErrors, timeoutToError } from "../errors";
 import debugbreak from "debugbreak";
 import { nodeDiscoveryShutdown } from "../-f-node-discovery/NodeDiscovery";
 import { authorityStorage } from "../0-path-value-core/pathValueCore";
@@ -15,6 +15,10 @@ let shutdownHandlers: (() => Promise<void>)[] = [];
 export function registerShutdownHandler(fnc: () => Promise<void>) {
     shutdownHandlers.push(fnc);
 }
+let preshutdownHandlers: (() => Promise<void>)[] = [];
+export function registerPreshutdownHandler(fnc: () => Promise<void>) {
+    preshutdownHandlers.push(fnc);
+}
 
 function logAll() {
     for (let fnc of periodicFncs) {
@@ -24,8 +28,20 @@ function logAll() {
 
 logErrors(runInfinitePollCallAtStart(timeInMinute * 5, logAll));
 
+let shuttingDown = false;
 export async function shutdown() {
+    if (shuttingDown) {
+        return;
+    }
+    shuttingDown = true;
     const { authorityStorage } = await import("../0-path-value-core/pathValueCore");
+    for (let fnc of preshutdownHandlers) {
+        try {
+            await timeoutToError(timeInMinute, fnc(), () => new Error(`Preshutdown handler ${fnc.name} timed out`));
+        } catch (e) {
+            console.log(`Error on preshutdown handler ${fnc.name}`, e);
+        }
+    }
     try {
         await authorityStorage.onShutdown();
         await nodeDiscoveryShutdown();
@@ -34,11 +50,13 @@ export async function shutdown() {
     }
     for (let fnc of shutdownHandlers) {
         try {
-            await fnc();
+            await timeoutToError(timeInMinute, fnc(), () => new Error(`Shutdown handler ${fnc.name} timed out`));
         } catch (e) {
             console.log(`Error on shutdown handler ${fnc.name}`, e);
         }
     }
+    // Wait to allow any logged errors to hopefully be written somewhere?
+    await delay(2000);
     process.exit();
 }
 

package/src/server.ts

@@ -14,7 +14,6 @@ import { AuthorityPath, pathValueAuthority2 } from "./0-path-value-core/NodePath
 import { Querysub } from "./4-querysub/QuerysubController";
 import { ClientWatcher } from "./1-path-client/pathValueClientWatcher";
 import { delay } from "socket-function/src/batching";
-import yargs from "yargs";
 import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
 import { green } from "socket-function/src/formatting/logColors";
 import { formatTime } from "socket-function/src/formatting/format";
@@ -26,6 +25,7 @@ import { getOwnThreadId } from "./-f-node-discovery/NodeDiscovery";
 import { getOurAuthorities } from "./config2";
 import { devDebugbreak, isPublic } from "./config";
 
+import yargs from "yargs";
 let yargObj = isNodeTrue() && yargs(process.argv)
     .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
     .option("verbose", { type: "boolean", desc: "Log all writes and reads" })

package/src/user-implementation/LoginPage.tsx

@@ -1,7 +1,7 @@
 import preact from "preact"; import { qreact } from "../4-dom/qreact";
 import { Button } from "../library-components/Button";
 import { css } from "../4-dom/css";
-import { loginTokenURL, user_data, user_functions } from "./userData";
+import { getUserId, getUserIdAllowUndefined, getUserObj, loginTokenURL, user_data, user_functions } from "./userData";
 import { Querysub } from "../4-querysub/Querysub";
 import { isNode, list } from "socket-function/src/misc";
 import { InputLabel } from "../library-components/InputLabel";
@@ -16,7 +16,7 @@ export class LoginPage extends qreact.Component {
     render() {
         const login = () => {
             user_functions.sendLoginEmail({
-                userId: this.state.email,
+                email: this.state.email,
                 redirectURL: window.location.href,
                 loginTokenRandomness: Buffer.from(list(32).map(() => Math.random() * 256)).toString("base64"),
             });
@@ -54,6 +54,22 @@ export class LoginPage extends qreact.Component {
                         </div>
                     </div>
                 }
+                {getUserIdAllowUndefined() &&
+                    <div class={css.hbox(10).alignSelf("center").maxWidth("50vw")}>
+                        <div class={css.fontSize(20).hslcolor(110, 75, 75)}>
+                            You are logged in as {getUserObj().email}.
+                        </div>
+                        <button onClick={() => {
+                            let call = user_functions.logoutCurrent();
+                            void Querysub.onCommitPredictFinished(call).finally(async () => {
+                                await Querysub.gracefulTerminateProcess();
+                                location.reload();
+                            });
+                        }}>
+                            Logout
+                        </button>
+                    </div>
+                }
             </div>
         );
     }

package/src/user-implementation/UserPage.tsx

@@ -268,7 +268,7 @@ export class UserPage extends qreact.Component {
                         </table>
                     </div>
                     <div>
-                        <div class={css.fontSize(20)}>Latest Page Load By Machine/IP</div>
+                        <div class={css.fontSize(20)}>Latest Access by Machine/IP</div>
                         <table>
                             <tr>
                                 <th>Source</th>
@@ -293,7 +293,7 @@ export class UserPage extends qreact.Component {
                             <h2>Example Login Email</h2>
                             <div key="email" ref2={e => {
                                 e.innerHTML = renderToString(generateLoginEmail({
-                                    userId,
+                                    email: "user@example.com",
                                     loginToken: "loginToken",
                                     ip: "127.0.0.1",
                                     machineId: getOwnMachineId(),

package/src/user-implementation/addSuperUser.ts

@@ -14,12 +14,11 @@ async function main() {
     const email = yargObj.email;
     if (!email) throw new Error("No email provided." + howToCall);
     if (!email.includes("@")) throw new Error(`Invalid email ${email}.` + howToCall);
-    const userId = email;
 
     await Querysub.hostService("addSuperUser");
 
     await Querysub.commitSynced(() => {
-        scriptCreateUser({ userId });
+        scriptCreateUser({ email });
     });
 
     await pathValueCommitter.waitForValuesToCommit();

package/src/user-implementation/loginEmail.tsx

@@ -4,7 +4,7 @@ import { list } from "socket-function/src/misc";
 import { hslToRGB } from "socket-function/src/formatting/colors";
 import { joinVNodes } from "../misc";
 export function generateLoginEmail(config: {
-    userId: string;
+    email: string;
     loginToken: string;
     ip: string;
     machineId: string;
@@ -12,10 +12,10 @@ export function generateLoginEmail(config: {
     timeoutTime: number;
     noHTMLWrapper?: boolean;
 }): { subject: string; contents: preact.VNode } {
-    const { userId, loginToken, ip, machineId, redirectURL } = config;
+    const { loginToken, ip, machineId, redirectURL, email } = config;
     let url = new URL(redirectURL);
     url.searchParams.set(loginTokenURL.urlKey, loginToken);
-    const subject = `${userId} Access Request (${new Date().toLocaleString()})`;
+    const subject = `${email} Access Request (${new Date().toLocaleString()})`;
     function center(jsx: preact.ComponentChild) {
         return (
             <table style={{ width: "100%" }}>
@@ -79,7 +79,7 @@ export function generateLoginEmail(config: {
             </table>
             {line(<div style={{ color: "rgb(255, 255, 255)" }}>
                 {/* Mess up the email so it isn't turned into a link by gmail. */}
-                Greetings <b>{userId.replaceAll(".", ".​")}</b>. You have requested access to <a href={redirectURL} style={{ color: "rgb(153, 204, 255)" }}>{new URL(redirectURL).hostname}</a>.
+                Greetings <b>{email.replaceAll(".", ".​")}</b>. You have requested access to <a href={redirectURL} style={{ color: "rgb(153, 204, 255)" }}>{new URL(redirectURL).hostname}</a>.
             </div>)}
             {line(<div style={{ color: "rgb(255, 255, 255)" }}>
                 If you did not make this request, please disregard this email. Your account is safe and no action is required.

package/src/user-implementation/userData.ts

@@ -1,7 +1,7 @@
 import { atomic, atomicObjectWrite } from "../2-proxy/PathValueProxyWatcher";
 import { Querysub } from "../4-querysub/QuerysubController";
 import { red } from "socket-function/src/formatting/logColors";
-import { isNode, timeInHour } from "socket-function/src/misc";
+import { isNode, sha256Hash, timeInHour } from "socket-function/src/misc";
 import { registerAliveChecker } from "../2-proxy/garbageCollection";
 import { generateLoginEmail } from "./loginEmail";
 import { logErrors } from "../errors";
@@ -15,6 +15,7 @@ import { devDebugbreak, getDomain, isDevDebugbreak } from "../config";
 import { delay } from "socket-function/src/batching";
 import { enableErrorNotifications } from "../library-components/errorNotifications";
 import { clamp } from "../misc";
+import { sha256 } from "js-sha256";
 
 /*
     IMPORTANT!
@@ -186,6 +187,10 @@ const { data, functions } = Querysub.syncSchema<{
         claimedDisplayNames: {
             [displayName: string]: string;
         };
+
+        emailToUserId: {
+            [email: string]: string;
+        };
     };
     machineSecure: {
         [machineId: string]: {
@@ -332,6 +337,7 @@ export function wildcard0Owner(config: PermissionsParameters) {
 }
 
 function isAllowedToBeUser(userId: string) {
+    userId = String(userId);
     if (isClient()) {
         return data().users[userId].userId === userId;
     }
@@ -532,12 +538,20 @@ function internalCreateUser(config: {
     return data().users[userId];
 }
 
+function createNewUserId() {
+    let randBuffer = Buffer.from(sha256(Querysub.nextId()), "hex");
+    const alphabet = "abcdefghijklmnopqrstuvwxyz";
+    return Array.from(randBuffer)
+        .map(x => alphabet[x % alphabet.length])
+        .join("")
+        .slice(0, 16);
+}
 function sendLoginEmail(config: {
-    userId: string;
+    email: string;
     redirectURL: string;
     loginTokenRandomness: string;
 }) {
-    const { userId, redirectURL } = config;
+    const { email, redirectURL } = config;
     {
         let redirectObj = new URL(redirectURL);
         redirectObj.port = "443";
@@ -547,19 +561,24 @@ function sendLoginEmail(config: {
         }
     }
     Querysub.ignorePermissionsChecks(() => {
-        if (!userId.includes("@")) {
-            throw new Error(`Invalid email ${userId}`);
+        if (!email.includes("@")) {
+            throw new Error(`Invalid email ${email}`);
         }
-        const email = userId;
+        let userId = atomic(data().secure.emailToUserId[email]) || createNewUserId();
 
         const signupError = `Signups are currently closed, and user doesn't presently exist. ${userId}. Add users with "yarn addsuperuser <email>"`;
         // NOTE: Using the same errors message makes it a BIT harder to know if you are blocked, so attackers are
         //  less likely to change their IP, and more likely to give up.
         throwIfBlockedIP(signupError);
-        if (!atomic(data().secure.signupsOpen) && !(userId in data().users)) {
+        if (!atomic(data().secure.signupsOpen) && (!userId || !(userId in data().users))) {
             throw new Error(signupError);
         }
 
+        // Create user if it doesn't exist
+        if (!(userId in data().users)) {
+            internalCreateUser({ userId, email });
+        }
+
         const ip = Querysub.getCallerIP();
         const machineId = Querysub.getCallerMachineId();
         const now = Querysub.getCallTime();
@@ -605,13 +624,6 @@ function sendLoginEmail(config: {
             }
         }
 
-        // Create user if it doesn't exist
-        {
-            if (!(userId in data().users)) {
-                internalCreateUser({ userId, email });
-            }
-        }
-
         let userObj = data().users[userId];
 
         // Per email/user throttle
@@ -644,7 +656,7 @@ function sendLoginEmail(config: {
         // NOTE: We don't track failed calls. Those should be tracked at a framework level and displayed on another page.
         logActivity({ type: "sendLoginEmail", fields: { userId, ip, machineId, redirectURL } });
         const { subject, contents } = generateLoginEmail({
-            userId, loginToken, ip, machineId, redirectURL,
+            email, loginToken, ip, machineId, redirectURL,
             timeoutTime,
         });
         Querysub.onCommitFinished(async () => {
@@ -756,35 +768,37 @@ function registerPageLoadTime() {
 }
 
 function inviteUser(config: { email: string }) {
-    let curUserObj = getUserObjAssert();
-    if (config.email in curUserObj.invitedUsers) {
-        console.info(`User ${config.email} already invited`);
-        return;
-    }
-    // If the user already exists, don't invite
-    const { email } = config;
-    let userId = email;
-    if (userId in data().users) {
-        console.info(`User ${userId} already exists, no need to invite`);
-        return;
-    }
+    Querysub.ignorePermissionsChecks(() => {
+        let curUserObj = getUserObjAssert();
+        if (config.email in curUserObj.invitedUsers) {
+            console.info(`User ${config.email} already invited`);
+            return;
+        }
+        // If the user already exists, don't invite
+        const { email } = config;
+        let userId = atomic(data().secure.emailToUserId[email]) || createNewUserId();
+        if (userId in data().users) {
+            console.info(`User ${userId} already exists, no need to invite`);
+            return;
+        }
 
-    const invitesRemaining = Number(curUserObj.invitesRemaining);
-    if (invitesRemaining <= 0) {
-        throw new Error("No invites remaining");
-    }
+        const invitesRemaining = Number(curUserObj.invitesRemaining);
+        if (invitesRemaining <= 0) {
+            throw new Error("No invites remaining");
+        }
 
-    curUserObj.invitedUsers[config.email] = atomicObjectWrite({
-        time: Querysub.getCallTime(),
-    });
-    curUserObj.invitesRemaining--;
+        curUserObj.invitedUsers[config.email] = atomicObjectWrite({
+            time: Querysub.getCallTime(),
+        });
+        curUserObj.invitesRemaining--;
 
-    if (!(userId in data().users)) {
-        // Create the user if they don't exist
-        let newUserObj = internalCreateUser({ userId, email });
-        newUserObj.invitedBy = curUserObj.userId;
-    }
-    logActivity({ type: "inviteUser", fields: { inviter: curUserObj.userId, userId, } });
+        if (!(userId in data().users)) {
+            // Create the user if they don't exist
+            let newUserObj = internalCreateUser({ userId, email });
+            newUserObj.invitedBy = curUserObj.userId;
+        }
+        logActivity({ type: "inviteUser", fields: { inviter: curUserObj.userId, userId, } });
+    });
 }
 
 function logoutFromMachine(config: { machineId: string; userId: string }) {
@@ -888,11 +902,11 @@ function specialSetUserType(config: { userId: string; userType: UserType; }) {
 
 
 export function scriptCreateUser(config: {
-    userId: string;
+    email: string;
 }) {
-    const { userId } = config;
-    let email = userId;
-    data().users[config.userId] = {
+    const { email } = config;
+    let userId = createNewUserId();
+    data().users[userId] = {
         userId,
         email,
         settings: { displayName: userId },
@@ -905,6 +919,7 @@ export function scriptCreateUser(config: {
         invitedUsers: {},
         invitesRemaining: 0,
     };
+    data().secure.emailToUserId[email] = userId;
 }
 
 function setPostmarkAPIKey(config: { apiKey: string; }) {