querysub 0.2.0 → 0.4.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "querysub",
-    "version": "0.2.0",
+    "version": "0.4.0",
     "main": "index.js",
     "license": "MIT",
     "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -24,7 +24,7 @@
         "node-forge": "https://github.com/sliftist/forge#e618181b469b07bdc70b968b0391beb8ef5fecd6",
         "pako": "^2.1.0",
         "preact": "^10.11.3",
-        "socket-function": "^0.31.0",
+        "socket-function": "^0.32.0",
         "terser": "^5.31.0",
         "typesafecss": "^0.6.3",
         "yaml": "^2.5.0",

package/src/-a-archives/archiveCache.ts

@@ -1,24 +1,16 @@
 import { getStorageDir, getSubFolder } from "../fs";
 import { Archives } from "./archives";
-import { diskEscapeFileName, getArchivesLocal } from "./archivesDisk";
 import fs from "fs";
 
-import diskusage from "diskusage";
-import { binarySearchBasic, binarySearchIndex, list, nextId, sort, timeInHour, timeInMinute } from "socket-function/src/misc";
+import { list, nextId, timeInHour, timeInMinute } from "socket-function/src/misc";
 import { cache, lazy } from "socket-function/src/caching";
-import { delay, runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { runInSerial, runInfinitePoll } from "socket-function/src/batching";
 import { sha256 } from "js-sha256";
-import { devDebugbreak } from "../config";
 import child_process from "child_process";
-import os from "os";
-import fsext, { flock } from "fs-ext";
-import debugbreak from "debugbreak";
-import readline from "readline";
 import { getPPID } from "../misc/getParentProcessId";
 import { Args } from "socket-function/src/types";
 import { getArchivesBackblaze } from "./archivesBackBlaze";
-import { logErrors } from "../errors";
-import { formatNumber, formatTime } from "socket-function/src/formatting/format";
+import { formatNumber } from "socket-function/src/formatting/format";
 import { SizeLimiter } from "../diagnostics/SizeLimiter";
 
 const SIZE_LIMIT = new SizeLimiter({
@@ -702,6 +694,7 @@ async function isLocked(path: string, operation?: {
     return await lockRetryLoop(async function isLocked() {
         let handle = await getHandle(lockPath);
         try {
+            const { flock } = await import("fs-ext");
             await new Promise<void>((r, e) => flock(handle.fd, "ex", err => err ? e(err) : r()));
             let contents = (await handle.readFile()).toString();
             let locks = contents.toString().replaceAll("\0", "").trim().split("\n").filter(x => x);

package/src/0-path-value-core/archiveLocks/archiveSnapshots.ts

@@ -4,6 +4,8 @@ import { getAllNodeIds, getOwnThreadId } from "../../-f-node-discovery/NodeDisco
 import { pathValueArchives } from "../pathValueArchives";
 import { ignoreErrors, logErrors, timeoutToUndefinedSilent } from "../../errors";
 import { green, magenta } from "socket-function/src/formatting/logColors";
+import { devDebugbreak } from "../../config";
+import { sort } from "socket-function/src/misc";
 
 const snapshots = getArchives("snapshots");
 
@@ -94,6 +96,8 @@ export async function getSnapshotList(): Promise<ArchiveSnapshotOverview[]> {
 
     overview.push(...snapshotFiles.map(file => fileNameToOverview(file)));
 
+    sort(overview, x => -x.time);
+
     return overview;
 }
 

package/src/3-path-functions/deployMain.ts

@@ -8,7 +8,7 @@ import { getModuleRelativePath } from "./syncSchema";
 import * as child_process from "child_process";
 import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 import { green, red } from "socket-function/src/formatting/logColors";
-import { isDefined } from "../misc";
+import { isDefined, isEmpty } from "../misc";
 import fs from "fs";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { getThreadKeyCert } from "../-a-auth/certs";
@@ -23,6 +23,7 @@ import debugbreak from "debugbreak";
 import yargs from "yargs";
 import { isPublic } from "../config";
 import { deployBlock } from "./deployBlock";
+import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
 let yargObj = yargs(process.argv)
     .option("domain", { type: "string", required: true, desc: "Domain to deploy to" })
     .option("nogit", { type: "boolean", default: false, desc: `Disable git operations. Useful for debugging, when using --local` })
@@ -87,31 +88,33 @@ export async function deployMain() {
 
         if (!isAllowedToDeploy) continue;
         let schema = getSchemaObject(module);
+        if (!schema) continue;
+        // If it's local, we don't need to deploy it.
+        //  (And shouldn't?, as we don't want to run local functions by request?)
+        if (schema.domainName === LOCAL_DOMAIN) continue;
 
-        if (schema) {
-            let filePath = getModuleRelativePath(module);
-
-            let moduleId = getPathStr2(domainName, filePath);
-            let otherUsed = usedModules.get(moduleId);
-            if (otherUsed) {
-                throw new Error(`Overlapping module ids for ${domainName}.${filePath}, at ${otherUsed} and ${filePath}`);
-            }
-            usedModules.set(moduleId, filePath);
-
-            const functionIds = Object.keys(schema.rawFunctions);
-            functionIds.push(PERMISSIONS_FUNCTION_ID);
-            for (let functionId of functionIds) {
-                const exportPathStr = getExportPath(functionId);
-                currentFunctions.push({
-                    DomainName: domainName,
-                    ModuleId: schema.moduleId,
-                    FilePath: filePath,
-                    FunctionId: functionId,
-                    exportPathStr,
-                    gitURL,
-                    gitRef,
-                });
-            }
+        let filePath = getModuleRelativePath(module);
+
+        let moduleId = getPathStr2(domainName, filePath);
+        let otherUsed = usedModules.get(moduleId);
+        if (otherUsed) {
+            throw new Error(`Overlapping module ids for ${domainName}.${filePath}, at ${otherUsed} and ${filePath}`);
+        }
+        usedModules.set(moduleId, filePath);
+
+        const functionIds = Object.keys(schema.rawFunctions);
+        functionIds.push(PERMISSIONS_FUNCTION_ID);
+        for (let functionId of functionIds) {
+            const exportPathStr = getExportPath(functionId);
+            currentFunctions.push({
+                DomainName: domainName,
+                ModuleId: schema.moduleId,
+                FilePath: filePath,
+                FunctionId: functionId,
+                exportPathStr,
+                gitURL,
+                gitRef,
+            });
         }
     }
 

package/src/3-path-functions/pathFunctionLoader.ts

@@ -183,14 +183,15 @@ async function getModuleFromSpecBase(
         // NOTE: The true tells require to not warn about the async loading
         await (require as any)(path, true);
     } catch (e: any) {
-        throw new Error(`Error when loading function for ${JSON.stringify(path)}\n${e.stack}`);
+
+        throw new Error(`Error when loading function for ${JSON.stringify(path)}:${spec.FunctionId}\n${e.stack}`);
     }
     let moduleId = require.resolve(path) || path;
     let module = require.cache[moduleId];
     if (!module) {
         debugbreak(2);
         debugger;
-        throw new Error(`Module not found: ${moduleId}`);
+        throw new Error(`Module not found: ${moduleId} (for ${spec.FunctionId})`);
     }
 
     //todonext

package/src/3-path-functions/syncSchema.ts

@@ -17,6 +17,7 @@ import { getPathFromProxy, getProxyPath } from "../2-proxy/pathValueProxy";
 import { CALL_PERMISSIONS_KEY } from "../4-querysub/permissionsShared";
 import { LOCAL_DOMAIN } from "../0-path-value-core/PathController";
 import path from "path";
+import { isEmpty } from "../misc";
 
 // This is the the function id which should be used when creating the FunctionSpec (in order to load the module),
 //  to access the permissions in the schema.
@@ -45,6 +46,7 @@ export type FunctionMetadata<F = unknown> = {
 
 export interface SchemaObject<Schema = any, Functions = any> {
     data: () => Schema;
+    domainName: string;
     moduleId: string;
     functions: {
         [key in keyof Functions]: (
@@ -350,9 +352,7 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
 
         let permissionsNonWildcards = new Map<string, PermissionsCallback>();
         let permissionsWildcards: { pathParts: string[]; callback: PermissionsCallback }[] = [];
-        if (permissions) {
-            addPermissions(rootPathStr, permissions);
-        }
+        addPermissions(rootPathStr, permissions);
         function addPermissions(path: string, permissions: SchemaToPermissions<Schema> | undefined) {
             if (!permissions) return;
             for (let key of getStringKeys(permissions)) {
@@ -378,6 +378,7 @@ export function syncSchema<Schema>(schema?: Schema2): SyncSchemaResult<Schema> {
 
         let result: SchemaObject = {
             data,
+            domainName,
             moduleId,
             functions: wrappedFunctions as any,
             functionMetadata: functionMetadata as any,

package/src/4-querysub/querysubPrediction.ts

@@ -329,10 +329,10 @@ function predictCall(config: {
         };
         try {
             dryRunResult = await getCallWrites({ call, debugName, overrides: config.overrides });
-        } catch (e) {
+        } catch (e: any) {
 
             if (!pathValueSerializer.getPathValue(authorityStorage.getValueAtTime(pathResultWrite, undefined))) {
-                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e);
+                console.log(`Skipping prediction for ${debugName} due to error running predictive call. Likely just an out of order error.`, e.stack);
             } else {
                 // NOTE: This case happens a lot, because of how we handle locks. We don't receive locked values, and so
                 //  we assume all values have no locks, and only keep the latest. This is usually fine, but... if we lose
@@ -482,9 +482,9 @@ export async function getCallWrites(config: {
             //     throw new Error(`Domain not found ${call.DomainName}`);
             // }
             let domainObject = functionSchema()[call.DomainName];
-            if (!(call.ModuleId in domainObject.PathFunctionRunner)) {
-                throw new Error(`Module not found ${call.DomainName}.${call.ModuleId}`);
-            }
+            // if (!(call.ModuleId in domainObject.PathFunctionRunner)) {
+            //     throw new Error(`Module not found ${call.DomainName}.${call.ModuleId}`);
+            // }
             let moduleObject = domainObject.PathFunctionRunner[call.ModuleId];
             if (!(call.FunctionId in moduleObject.Sources)) {
                 throw new Error(`Function not found ${call.DomainName}.${call.ModuleId}.${call.FunctionId}`);
@@ -516,9 +516,9 @@ export async function getCallWrites(config: {
         overrides: config.overrides,
         nestedCalls: "inline",
         watchFunction() {
-            overrideCurrentCall(call, () => {
+            return overrideCurrentCall(call, () => {
                 let args = parseArgs(call);
-                baseFunction(...args);
+                return baseFunction(...args);
             });
         },
     });

package/src/diagnostics/managementPages.tsx

@@ -20,8 +20,9 @@ import { CallSpec, getCurrentCall, getCurrentCallAllowUndefined, overrideCurrent
 import { createLazyComponent } from "../library-components/LazyComponent";
 import { LogNotify } from "./errorLogs/LogNotify";
 import { isTrusted } from "../-d-trust/NetworkTrust2";
-import { getDomain } from "../config";
+import { devDebugbreak, getDomain } from "../config";
 import { getCallWrites } from "../4-querysub/querysubPrediction";
+import { SchemaObject } from "../3-path-functions/syncSchema";
 
 
 export const managementPageURL = new URLParam("managementpage", "");
@@ -33,25 +34,17 @@ let pages: {
     title: string;
 }[] = [];
 
-let internalIsManagementUser: () => boolean = () => {
-    throw new Error("registerManagementPages2 must be called during deploy for management functions to be exposed");
-};
-
-// NOTE: We don't store the UI in the database (here, or anywhere else, at least
-//  not yet), but we do want to store the permission function call, as we may
-//  need to update it immediately, across all servers.
-const schema = Querysub.createSchema()({
-    module,
-    moduleId: "managementPages",
-    functions: {
-        schemaIsManagementUser,
-    },
-});
-function schemaIsManagementUser() {
-    return internalIsManagementUser();
+let __schema: SchemaObject<unknown, {
+    schemaIsManagementUser: () => boolean;
+}> | undefined = undefined;
+function getSchema() {
+    if (!__schema) throw new Error(`registerManagementPages2 must be called for management functions to be accessed`);
+    return __schema;
 }
 
+let registeredModule: NodeJS.Module | undefined = undefined;
 export async function registerManagementPages2(config: {
+    module: NodeJS.Module;
     isManagementUser: () => boolean;
     pages: {
         componentName: string;
@@ -61,7 +54,11 @@ export async function registerManagementPages2(config: {
         title?: string;
     }[];
 }) {
-    baseIsManagementUser = config.isManagementUser;
+    if (registeredModule) {
+        if (registeredModule === config.module) return;
+        throw new Error(`registerManagementPages2 called with a different module than previously registered. This will break isManagementUser, and so is not allowed.`);
+    }
+    registeredModule = config.module;
     let inputPages = config.pages.slice();
 
     inputPages.push({
@@ -94,6 +91,17 @@ export async function registerManagementPages2(config: {
         getModule: () => import("./TimeDebug"),
     });
 
+    // NOTE: We don't store the UI in the database (here, or anywhere else, at least
+    //  not yet), but we do want to store the permission function call, as we may
+    //  need to update it immediately, across all servers.
+    __schema = Querysub.createSchema()({
+        module: config.module,
+        moduleId: "managementPages",
+        functions: {
+            schemaIsManagementUser: config.isManagementUser,
+        },
+    });
+
     if (isServer()) {
         for (let page of inputPages) {
             // NOTE: If we split this into a module for component/controller, we need to make sure we
@@ -124,11 +132,6 @@ export async function registerManagementPages2(config: {
     }
 }
 
-let baseIsManagementUser: () => boolean = () => {
-    // If we are clientside, all calls come from the server, and so are allowed.
-    if (isClient()) return true;
-    throw new Error("baseIsManagementUser not set");
-};
 
 export async function isManagementUser() {
     let caller = SocketFunction.getCaller();
@@ -138,6 +141,9 @@ export async function isManagementUser() {
     //  when clients are given callbacks by servers, for management apis which
     //  have every endpoint protected by isManagementUser.
     if (await isTrusted(callerMachineId)) return true;
+    let schema = getSchema();
+    let functionId = "schemaIsManagementUser" as const;
+    if (!schema.functions[functionId]) throw new Error(`Function ${functionId} not found in schema`);
 
     let testCall: CallSpec = {
         callerIP: IdentityController_getSecureIP(caller),
@@ -145,8 +151,8 @@ export async function isManagementUser() {
         argsEncoded: encodeArgs([]),
         CallId: nextId(),
         DomainName: getDomain(),
-        ModuleId: "managementPages",
-        FunctionId: "schemaIsManagementUser",
+        ModuleId: schema.moduleId,
+        FunctionId: functionId,
         runAtTime: getNextTime(),
     };