querysub 0.181.0 → 0.182.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.181.0",
+  "version": "0.182.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/spec.txt

@@ -51,10 +51,6 @@ More corruption resistance files
             - Value types will be string, float64, byte, Buffer[]
                 - and... we might as well add support for short, int, float, and uint (uint is a good way to store a guid, via storing 8 uint variables).
 
-
-
-
-
 Schema/binary PathValues accesses
 
     Base code

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -683,19 +683,19 @@ class TransactionLocker {
             {
                 let oldThreshold = readTime - DEAD_CREATE_THRESHOLD;
                 let veryOldFiles = dataState.rawDataFiles.filter(x => x.file.endsWith(".locked")).filter(a => a.createTime < oldThreshold);
-                let confirmedLookup = new Set(dataState.confirmedDataFiles.map(a => a));
-                let unconfirmedOldFiles = veryOldFiles.filter(a => !confirmedLookup.has(a));
+                let confirmedLookup = new Set(dataState.confirmedDataFiles.map(a => a.file));
+                let unconfirmedOldFiles = veryOldFiles.filter(a => !confirmedLookup.has(a.file));
 
                 if (unconfirmedOldFiles.length > 0) {
                     let doubleCheck = await this.readDataState();
-                    let doubleCheckLookup = new Set(doubleCheck.confirmedDataFiles.map(a => a));
+                    let doubleCheckLookup = new Set(doubleCheck.confirmedDataFiles.map(a => a.file));
                     let doubleCheckDataFiles = new Set(doubleCheck.rawDataFiles.map(a => a.file));
                     // IMPORTANT! Make sure they still aren't confirmed AND that the rawDataFiles exist. This helps prevent cases
                     //  where backblaze returns no files? Which it might be doing, as multiple times this code has
                     //  triggered (without this check), and deletes all of our files...
-                    let unconfirmedOldFiles2 = veryOldFiles.filter(a => !doubleCheckLookup.has(a) && doubleCheckDataFiles.has(a.file));
+                    let unconfirmedOldFiles2 = unconfirmedOldFiles.filter(a => !doubleCheckLookup.has(a.file) && doubleCheckDataFiles.has(a.file));
                     console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`), { files: unconfirmedOldFiles2.map(x => x.file) });
-                    diskLog(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
+                    logNodeStats(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
                     // At the point the file was very old when we started reading, not part of the active transaction.
                     for (let file of unconfirmedOldFiles2) {
                         await this.deleteDataFile(file.file, `old unconfirmed file (${getOwnNodeId()}, ${process.argv[1]})`);

package/src/0-path-value-core/pathValueArchives.ts

@@ -185,7 +185,9 @@ export class PathValueArchives {
             if (time < oldestTime) oldestTime = time;
         }
         let slowestFileWriteTime = oldestTime + ARCHIVE_FLUSH_LIMIT;
-        await archives().set(authorityPath + file, data);
+        let fullPath = authorityPath + file;
+        console.log(`Write archive file ${fullPath}, with size ${formatNumber(data.byteLength)}B, and count ${formatNumber(values.length)}`);
+        await archives().set(fullPath, data);
 
         let fileInfo = await archives().getInfo(authorityPath + file);
         // NOTE: If no fileInfo... then our file was merged? Which... is BAD, as it means we took

package/src/4-querysub/Querysub.ts

@@ -45,6 +45,7 @@ import { devDebugbreak, getDomain, isDynamicallyLoading, isPublic, noSyncing } f
 import { Schema2, Schema2T, t } from "../2-proxy/schema2";
 import { CALL_PERMISSIONS_KEY } from "./permissionsShared";
 import yargs from "yargs";
+import { parseArgsFactory } from "../misc/rawParams";
 
 import * as typesafecss from "typesafecss";
 typesafecss.setMeasureBlock(measureBlock);
@@ -55,7 +56,7 @@ typesafecss.setDelayFnc(async fnc => {
 
 export { t };
 
-let yargObj = isNodeTrue() && yargs(process.argv)
+let yargObj = parseArgsFactory()
     .option("verboseall", { type: "boolean", desc: "Log all everything (except network)" })
     .option("verbose", { type: "boolean", desc: "Log all writes and reads" })
     .option("verbosewrites", { type: "boolean", desc: "Log all writes" })
@@ -63,7 +64,8 @@ let yargObj = isNodeTrue() && yargs(process.argv)
     .option("verbosecalls", { type: "boolean", desc: "Log all calls" })
     .option("verbosenetwork", { type: "boolean", desc: "Log all network activity" })
     .option("verboseframework", { type: "boolean", desc: "Log internal SocketFunction framework" })
-    .argv || {}
+    .option("nodelay", { type: "boolean", desc: "Don't delay committing functions, even ones that are marked to be delayed." })
+    .argv
     ;
 setImmediate(() => {
     if (yargObj.verbose) {
@@ -92,6 +94,9 @@ setImmediate(() => {
     if (yargObj.verboseframework) {
         SocketFunction.silent = false;
     }
+    if (yargObj.nodelay) {
+        Querysub.DELAY_COMMIT_DELAY = 0;
+    }
 });
 
 if (!registerGetCompressNetwork) {

package/src/archiveapps/archiveJoinEntry.ts

@@ -27,7 +27,9 @@ async function runGenesisJoinIteration() {
         while (reread) {
             reread = false;
             let result = await locker.atomicSwapFiles({}, async (valueFiles, readFiles) => {
+                let originalCount = valueFiles.length;
                 valueFiles = valueFiles.filter(x => matchedDirs.some(y => x.file.startsWith(y)));
+                let underPathCount = valueFiles.length;
                 // Merge the newest first, so if we hit a big file, we can just ignore it,
                 //  and next time merge the smaller files after it.
                 sort(valueFiles, x => -x.createTime);
@@ -37,6 +39,7 @@ async function runGenesisJoinIteration() {
                     if (obj.sourceType !== "genesis") return false;
                     return x.createTime >= maxAge;
                 });
+                let withinTimeRangeCount = valueFiles.length;
 
                 if (valueFiles.length < 3) return [];
 
@@ -105,7 +108,7 @@ async function runGenesisJoinIteration() {
                 });
 
 
-                console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes`));
+                console.log(magenta(`Joining ${formatNumber(usedFiles.length)} files with ${formatNumber(allCombinedValues.length)} values in ${formatNumber(totalSize)} bytes. Original count: ${formatNumber(originalCount)}, under path count: ${formatNumber(underPathCount)}, within time range count: ${formatNumber(withinTimeRangeCount)}`));
 
                 return [transaction];
             });

package/src/config.ts

@@ -2,10 +2,11 @@ import { isNode, isNodeTrue } from "socket-function/src/misc";
 import yargs from "yargs";
 import debugbreak from "debugbreak";
 import { MaybePromise } from "socket-function/src/types";
+import { parseArgsFactory } from "./misc/rawParams";
 
 export const serverPort = 11748;
 
-let yargObj = isNodeTrue() && yargs(process.argv)
+let yargObj = parseArgsFactory()
     .option("nonetwork", { type: "boolean", desc: `Disables all network requirements. Reduces security, as this means we cannot use real certificates.` })
     .option("domain", { type: "string", desc: `Sets the domain` })
     .option("client", { type: "boolean", desc: `Drops permissions, acting as an unauthenticated node` })
@@ -16,7 +17,7 @@ let yargObj = isNodeTrue() && yargs(process.argv)
     .option("local", { type: "boolean", desc: `If true, uses the local directory instead of the remote git repo. Also hotreloads from disk. Determines the repo to replace through the package.json "repository" property.` })
     .option("nosyncing", { type: "boolean", desc: `Disabled any syncing code, just using HTTP. Still manages routing via backblaze though.` })
     .option("recovery", { type: "boolean", desc: `Allows any localhost connections to act like a superuser (and a trusted node), to help recover the database (as you need permission to access the snapshot page).` })
-    .argv || {}
+    .argv
     ;
 
 if (!isNode()) {

package/src/misc/rawParams.ts

@@ -0,0 +1,189 @@
+import { setFlag } from "socket-function/require/compileFlags";
+import { isNode } from "typesafecss";
+import yargs from "yargs";
+
+type Arguments<T = {}> = T & {
+    /** Non-option arguments */
+    _: Array<string | number>;
+    /** The script name or node command */
+    $0: string;
+    /** All remaining options */
+    [argName: string]: unknown;
+};
+
+interface Options {
+    desc?: string;
+    required?: boolean;
+    type?: "array" | "boolean" | "number" | "string" | undefined;
+}
+
+type InferredOptionTypeBase<O extends Options> = (
+    O extends { type: "array" } ? string[]
+    : O extends { type: "boolean" } ? boolean
+    : O extends { type: "number" } ? number
+    : O extends { type: "string" } ? string : never
+);
+
+type InferredOptionType<O extends Options> = (
+    O extends { required: true } ? InferredOptionTypeBase<O> : InferredOptionTypeBase<O> | undefined
+);
+
+interface Argv<T = {}> {
+    (): { [key in keyof Arguments<T>]: Arguments<T>[key] };
+    (args: readonly string[], cwd?: string): Argv<T>;
+
+    option<K extends string, O extends Options>(
+        key: K,
+        options: O,
+    ): Argv<T & { [key in K]: InferredOptionType<O> }>;
+
+    argv: T;
+}
+
+function getRawParams(): string[] {
+    if (isNode()) return process.argv;
+    let searchParams = new URLSearchParams(location.search);
+    let result: string[] = [];
+    for (let [key, value] of searchParams.entries()) {
+        if (value === "") {
+            result.push(`--${key}`);
+        } else {
+            result.push(`--${key}=${value}`);
+        }
+    }
+    return result;
+}
+
+export function parseArgsFactory(): Argv {
+    type OptionCall = {
+        key: string;
+        options: Options;
+    };
+    type CurrentConfig = {
+        options: OptionCall[];
+    };
+
+    function finish(config: CurrentConfig) {
+        let params = getRawParams();
+        let result: Record<string, unknown> = {};
+
+        params = params.map(param => param.replace(/^-+/, ""));
+
+        if (params.includes("help")) {
+            console.log("Available options:");
+            for (let option of config.options) {
+                let type = option.options.type || "string";
+                let required = option.options.required ? " (required)" : "";
+                let desc = option.options.desc ? ` - ${option.options.desc}` : "";
+                console.log(`  --${option.key} (${type})${required}${desc}`);
+            }
+            process.exit(0);
+        }
+
+        // Second pass: join split parameters
+        let processedParams: string[] = [];
+        for (let i = 0; i < params.length; i++) {
+            let key = params[i];
+
+            // Only treat as option if it's in the options list
+            let optionConfig = config.options.find(opt => opt.key === key);
+            if (optionConfig && optionConfig.options.type !== "boolean") {
+                let nextParam = params[i + 1]; // Use original for value
+                // Join them: x 5 becomes x=5
+                processedParams.push(`${key}=${nextParam}`);
+                i++; // Skip the next parameter
+                continue;
+            }
+
+            processedParams.push(key);
+        }
+
+        // Second pass: process the parameters
+        for (let param of processedParams) {
+            // Check if it's a key=value format
+            if (param.includes("=")) {
+                let [optionKey, ...valueParts] = param.split("=");
+                let value = valueParts.join("=");
+                let optionConfig = config.options.find(opt => opt.key === optionKey);
+
+                if (optionConfig) {
+                    let { type } = optionConfig.options;
+                    if (type === "boolean") {
+                        result[optionKey] = value === "true";
+                    } else if (type === "number") {
+                        result[optionKey] = Number(value);
+                    } else if (type === "array") {
+                        if (!result[optionKey]) result[optionKey] = [];
+                        (result[optionKey] as unknown[]).push(value);
+                    } else {
+                        result[optionKey] = value;
+                    }
+                } else {
+                    result[optionKey] = value;
+                }
+            } else {
+                // Regular option (boolean)
+                let optionConfig = config.options.find(opt => opt.key === param);
+
+                if (optionConfig) {
+                    let { type } = optionConfig.options;
+
+                    if (type === "boolean") {
+                        result[param] = true;
+                    } else {
+                        // Other types should have been joined in first pass
+                        // Set default if somehow we get here
+                        if (type === "number") {
+                            result[param] = 0;
+                        } else if (type === "array") {
+                            result[param] = [];
+                        } else {
+                            result[param] = "";
+                        }
+                    }
+                } else {
+                    // Unknown option, treat as boolean
+                    result[param] = true;
+                }
+            }
+        }
+
+        // Set default values for required options that weren't provided
+        for (let option of config.options) {
+            if (option.options.required && result[option.key] === undefined) {
+                if (option.options.type === "boolean") {
+                    result[option.key] = false;
+                } else if (option.options.type === "array") {
+                    result[option.key] = [];
+                } else if (option.options.type === "number") {
+                    result[option.key] = 0;
+                } else {
+                    result[option.key] = "";
+                }
+            }
+        }
+
+        return result;
+    }
+
+    function createCallback(config: CurrentConfig): Argv {
+        const callback = {
+            option: (key: string, options: Options) => {
+                config.options.push({ key, options });
+                return createCallback(config) as any;
+            },
+            get argv() {
+                return finish(config);
+            }
+        };
+
+        // Make the callback callable
+        const fn = (() => callback.argv) as any;
+        fn.option = callback.option;
+        fn.argv = callback.argv;
+
+        return fn;
+    }
+
+    return createCallback({ options: [] });
+}