querysub 0.2.0

package/src/buffers.ts

@@ -0,0 +1,69 @@
+import { canHaveChildren } from "socket-function/src/types";
+
+export type ArrayBufferViewTypes = Uint8Array | Int8Array | Uint16Array | Int16Array | Uint32Array | Int32Array | BigUint64Array | BigInt64Array | Float64Array | Float32Array | Uint8ClampedArray;
+export type BufferType = ArrayBuffer | SharedArrayBuffer | ArrayBufferViewTypes;
+
+export function cloneBuffer(data: Buffer): Buffer {
+    let newBuffer = Buffer.alloc(data.length);
+    data.copy(newBuffer);
+    return newBuffer;
+}
+
+
+export function asBuffer(data: BufferType): Buffer {
+    if (!data) return data;
+    if (data instanceof Buffer) return data;
+    if (!canHaveChildren(data)) return data as any;
+    if (!("buffer" in data) || !("byteOffset" in data) || !("byteLength" in data)) {
+        return Buffer.from(data);
+    }
+    let result = Buffer.from((data as any).buffer, (data as any).byteOffset, (data as any).byteLength);
+    return result;
+}
+export function asFloat64(data: Buffer): Float64Array {
+    if (data.length % 8 !== 0) {
+        throw new Error(`Data is not 8 count aligned, received length of ${data.length}`);
+    }
+    if (data.byteOffset % 8 !== 0) {
+        throw new Error(`Data is not 8 byte aligned, received offset of ${data.byteOffset}`);
+    }
+    return new Float64Array(data.buffer, data.byteOffset, Math.floor(data.length / 8));
+}
+export function asFloat32(data: Buffer): Float32Array {
+    if (data.length % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received length of ${data.length}`);
+    }
+    if (data.byteOffset % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received offset of ${data.byteOffset}`);
+    }
+    return new Float32Array(data.buffer, data.byteOffset, Math.floor(data.length / 4));
+}
+export function asUint32(data: Buffer): Uint32Array {
+    if (data.length % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received length of ${data.length}`);
+    }
+    if (data.byteOffset % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received offset of ${data.byteOffset}`);
+    }
+    return new Uint32Array(data.buffer, data.byteOffset, Math.floor(data.length / 4));
+}
+export function asInt32(data: Buffer): Int32Array {
+    if (data.length % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received length of ${data.length}`);
+    }
+    if (data.byteOffset % 4 !== 0) {
+        throw new Error(`Data is not 4 byte aligned, received offset of ${data.byteOffset}`);
+    }
+    return new Int32Array(data.buffer, data.byteOffset, Math.floor(data.length / 4));
+}
+
+
+export function asFloat64MaybeCopy(data: Buffer) {
+    if (data.length % 8 !== 0) {
+        throw new Error(`Data is not 8 count aligned, received length of ${data.length}`);
+    }
+    if (data.byteOffset % 8 !== 0) {
+        return asFloat64(cloneBuffer(data));
+    }
+    return new Float64Array(data.buffer, data.byteOffset, Math.floor(data.length / 8));
+}

package/src/config.ts

@@ -0,0 +1,53 @@
+import { isNode, isNodeTrue } from "socket-function/src/misc";
+import yargs from "yargs";
+import debugbreak from "debugbreak";
+
+export const serverPort = 11748;
+
+let yargObj = isNodeTrue() && yargs(process.argv)
+    .option("nonetwork", { type: "boolean", desc: `Disables all network requirements. Reduces security, as this means we cannot use real certificates.` })
+    .option("domain", { type: "string", desc: `Sets the domain` })
+    .option("client", { type: "boolean", desc: `Drops permissions, acting as an unauthenticated node` })
+    .option("authority", { type: "string", desc: `Defines the base paths we are an authority on (the domain is prepended to them). Either a file path to a JSON(AuthorityPath[]), or a base64 representation of the JSON(AuthorityPath[]).` })
+    .option("debugbreak", { type: "boolean", desc: "Break and show a popup when there are fatal errors" })
+    .option("nobreak", { type: "boolean", desc: "Do not break on errors. Safer to set this than to just not set debugbreak, as some places might break without checking debugbreak, but nobreak works at a level where it is always used." })
+    .option("public", { type: "boolean", desc: "Expose on public ports." })
+    .argv || {}
+    ;
+
+export function isNoNetwork() {
+    return yargObj.nonetwork;
+}
+
+export function getDomain() {
+    if (!isNode()) {
+        return location.hostname.split(".").slice(-2).join(".");
+    }
+    return yargObj.domain || "querysub.com";
+}
+
+export function baseIsClient() {
+    return !isNode() || yargObj.client;
+}
+
+export function devDebugbreak() {
+    if (!isNode()) {
+        debugger;
+        return;
+    }
+    if (!yargObj.debugbreak) return;
+    debugbreak(2);
+    debugger;
+}
+export function isDevDebugbreak() {
+    return yargObj.debugbreak;
+}
+
+
+export function authorityRaw() {
+    return yargObj.authority;
+}
+
+export function isPublic() {
+    return !!yargObj.public;
+}

package/src/config2.ts

@@ -0,0 +1,48 @@
+import { deepCloneJSON, isNode } from "socket-function/src/misc";
+import { hasArchivesPermissions } from "./-a-archives/archives";
+import { authorityRaw, baseIsClient, getDomain } from "./config";
+import { JSONLACKS } from "socket-function/src/JSONLACKS/JSONLACKS";
+import { AuthorityPath } from "./0-path-value-core/NodePathAuthorities";
+import { rootPathStr, prependToPathStr } from "./path";
+import fs from "fs";
+
+let isInCall = false;
+export function isClient() {
+    isInCall = true;
+    try {
+        return !isNode() || baseIsClient() || !isInCall && !hasArchivesPermissions();
+    } finally {
+        isInCall = false;
+    }
+}
+
+export function isServer() {
+    return !isClient();
+}
+
+export function getOurAuthorities() {
+    let authorities: AuthorityPath[] = [];
+    let authority = authorityRaw();
+
+    let baseAuthorityPaths: AuthorityPath[] = [{
+        pathPrefix: rootPathStr
+    }];
+    if (authority) {
+        if (fs.existsSync(authority)) {
+            baseAuthorityPaths = JSONLACKS.parse(fs.readFileSync(authority, "utf8"));
+        } else {
+            baseAuthorityPaths = JSONLACKS.parse(Buffer.from(authority, "base64").toString("utf8"));
+        }
+    }
+
+    let domain = getDomain();
+    for (let base of baseAuthorityPaths) {
+        base = deepCloneJSON(base);
+        base.pathPrefix = prependToPathStr(base.pathPrefix, domain);
+        if (base.excludedChildren) {
+            base.excludedChildren = base.excludedChildren.map(x => prependToPathStr(x, domain));
+        }
+        authorities.push(base);
+    }
+    return authorities;
+}

package/src/diagnostics/ActionsHistory.ts

@@ -0,0 +1,56 @@
+import { cache } from "socket-function/src/caching";
+import { blue, green, red } from "socket-function/src/formatting/logColors";
+import { getStorageDir, getSubFolder } from "../fs";
+import { ClientWatcher } from "../1-path-client/pathValueClientWatcher";
+import { PathValueProxyWatcher } from "../2-proxy/PathValueProxyWatcher";
+import { debugPathValuePath, PathValue, PathWatcherCallback, ReadLock, WatchConfig, WriteState } from "../0-path-value-core/pathValueCore";
+import fs from "fs";
+import { measureWrap } from "socket-function/src/profiling/measure";
+import { pathValueSerializer } from "../-h-path-value-serialize/PathValueSerializer";
+
+
+
+function group(expanded: boolean) {
+    return expanded ? console.group : console.groupCollapsed;
+}
+
+let epoch = Date.now();
+export class ActionsHistory {
+    public static LOG_ACTION_HISTORY = "";
+    public static OnNewWatches(config: { newPathsWatched: Set<string>; newParentsWatched: Set<string>; debugName?: string; authorityId: string; }) {
+        if (ClientWatcher.DEBUG_READS) {
+            let now = Date.now();
+            group(ClientWatcher.DEBUG_READS_EXPANDED)(`(${now - epoch}ms) New watch count ${config.newPathsWatched.size} (${config.newParentsWatched.size} parent watches) for ${JSON.stringify(config.debugName || "anonymous")}. Authority ${config.authorityId}, time ${now}`);
+            for (let path of config.newParentsWatched) {
+                console.log(blue(path));
+            }
+            for (let path of config.newPathsWatched) {
+                console.log(path);
+            }
+            console.groupEnd();
+        }
+    }
+    public static OnRead(values: PathValue[]) {
+        if (ClientWatcher.DEBUG_READS && values.length > 0) {
+            let now = Date.now();
+            let oldest = Math.min(...values.map(x => x.time.time).filter(x => x), now);
+            group(ClientWatcher.DEBUG_READS_EXPANDED)(`(${now - epoch}ms, ${now}) Read count ${values.length}, ${(Date.now() - oldest).toFixed()}ms old`);
+            for (let value of values) {
+                console.log(blue(debugPathValuePath(value)), "===", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+            }
+            console.groupEnd();
+        }
+        if (!ActionsHistory.LOG_ACTION_HISTORY) return;
+    }
+    public static OnWrite(values: PathValue[]) {
+        if (ClientWatcher.DEBUG_WRITES) {
+            let now = Date.now();
+            group(ClientWatcher.DEBUG_WRITES_EXPANDED)(green(`(${now - epoch}ms, ${now}) Committing writes ${values.length}`));
+            for (let value of values) {
+                console.log(green(debugPathValuePath(value)), "=", pathValueSerializer.getPathValue(value), `(${value.valid ? "valid" : "invalid"})`);
+            }
+            console.groupEnd();
+        }
+        if (!ActionsHistory.LOG_ACTION_HISTORY) return;
+    }
+}