querysub 0.403.0 → 0.405.0

package/.cursorrules

@@ -1,5 +1,7 @@
 The code automatically updates on save, so you do not need to ever run commands to run the site.
 
+NEVER copy and paste code. Use functions correctly, unless making it a function is more code than not making it a function.
+
 Always directly set the state (this.state.x = 1), and never use setState.
 
 Unless schema values use atomic or type they will always be returned as a value (as a proxy). Use "in" to check if a value is in a t.lookup.

package/bin/audit-imports.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+
+require("typenode");
+require("../src/diagnostics/auditImportViolations");

package/bin/join.js

@@ -1,4 +1,4 @@
 #!/usr/bin/env node
 
 require("typenode");
-require("../src/archiveapps/archiveGCEntry");
+require("../src/archiveapps/archiveJoinEntry");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.403.0",
+  "version": "0.405.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",
@@ -17,7 +17,9 @@
     "test-wat": "yarn typenode ./src/wat/watCompiler.ts",
     "error-watch": "yarn typenode ./src/diagnostics/logs/errorNotifications2/errorWatchEntry.ts",
     "error-email": "yarn typenode ./src/diagnostics/logs/errorNotifications/errorDigestEntry.tsx",
-    "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild"
+    "build-native": "cd src/diagnostics/logs/IndexedLogs && node-gyp rebuild",
+    "audit-imports": "yarn typenode ./src/diagnostics/auditImportViolations.ts",
+    "test": "yarn typenode ./test.ts"
   },
   "bin": {
     "deploy": "./bin/deploy.js",
@@ -36,7 +38,8 @@
     "movelogs": "./bin/movelogs.js",
     "addsuperuser": "./bin/addsuperuser.js",
     "error-watch": "./bin/error-watch.js",
-    "error-watch-public": "./bin/error-watch-public.js"
+    "error-watch-public": "./bin/error-watch-public.js",
+    "audit-imports": "./bin/audit-imports.js"
   },
   "dependencies": {
     "@types/fs-ext": "^2.0.3",
@@ -57,7 +60,7 @@
     "pako": "^2.1.0",
     "peggy": "^5.0.6",
     "querysub": "^0.357.0",
-    "socket-function": "^1.1.5",
+    "socket-function": "^1.1.11",
     "terser": "^5.31.0",
     "typesafecss": "^0.28.0",
     "yaml": "^2.5.0",

package/spec.txt

@@ -1,3 +1,80 @@
+
+
+Trigger values
+    1) validStateComputer.ingestValuesAndValidStates
+    1.5) authorityStorage.ingestValues
+    2) authorityStorage.ingestValues
+    3) lockWatcher2.watchValueLocks
+    4) validStateComputer.computeValidStates
+    5) lockWatcher2.getValuePathWatchers
+    6) pathWatcher.triggerValuesChanged
+    7) pathWatcher.triggerLatestWatcher
+    8) PathValueControllerBase/clientWatcher.localOnValueCallback
+
+Source of values
+    1) clientWatcher.setValues
+    2) PathValueCommitter.commitValues
+    3) (predictions) validStateComputer.ingestValuesAndValidStates
+    4) (non-predictions) PathValueControllerBase.createValues
+    5) validStateComputer.ingestValuesAndValidStates
+
+Archiving values
+    1) authorityStorage.ingestValues
+    2) pathValueArchives.archiveValues
+
+TODO: pathValueAuthority detection
+
+
+Core parts
+    - Valid evaluation
+        - Needs external value syncing, which can give it all past values, and all future values for a path
+    - Value syncer
+        - Needs someone to give it authorities, and tell it when an authority goes down (at which point it finds a new authority and syncs the new values)
+    - Authority tracker (supports registering, checking, notifying, and generally synchronizing all authority information between all authorities)
+        - No requirements
+    - Authority setuper
+        - Needs authority tracker
+        - Needs value syncer
+    - Authority (stitches together functionality)
+        - Needs authority tracker
+        - Needs authority setuper
+        - Needs value syncer
+        - Needs valid evaluation
+    - Disk value saving/retrieval
+
+FunctionRunner is funny. It wants a full history, for reading, but does not calculate the valid states of PathValues.
+
+
+Optimization / optional features list
+    Valid Evaluation
+        - Only need to sync some data, not all data. 
+        - event: sometimes we discard data
+        - isTransparent: Sometimes, even though two values are different, they can be considered equivalent. 
+        - gc: Older values won't change their valid state, so we can discard their locks.
+    Value Syncer
+        - Path watches
+        - Parent path watches
+        - Authority path (tree) watches
+        - Valid watching (authorities know this, so it's free to also sync this)
+        - Only latest/valid state watches (no history, just latest, more efficient)
+        - Special connection clobber logic (you can't store the history after you disconnect if you are not the authority)
+        - Special logic
+        - Reused for internal watches (rendering, etc)
+    Authority tracker
+        - Immutable authorities, to simplify usages
+        - Path requests => authorities might take slices from multiple authorities
+    Authority setuper
+        - Once on startup, so we don't change authorities, to simplify implementation
+        - Leverage disk snapshots to make loading faster and lighter for existing nodes
+    Authority
+        - The stitching together of all the values is reused by the function runner in order to rerun functions when they are invalidated
+    Disk value saving/retrieval
+        - Only old values with discarded locks are written, and the history is compressed occasionally 
+
+
+
+
+
 *** Goal is to get to === Full public website stack milestone === ***
 === 3 tasks to fully fledged syncing KVP database, 2023 / 25 / 2 ===
 === 1 tasks to fully fledged syncing KVP database, 2023 / 26 / 2 ===

package/src/-a-archives/archiveCache.ts

@@ -4,7 +4,7 @@ import fs from "fs";
 
 import { list, nextId, timeInHour, timeInMinute } from "socket-function/src/misc";
 import { cache, lazy } from "socket-function/src/caching";
-import { runInSerial, runInfinitePoll } from "socket-function/src/batching";
+import { runInParallel, runInSerial, runInfinitePoll } from "socket-function/src/batching";
 import { sha256 } from "js-sha256";
 import child_process from "child_process";
 import { getPPID } from "../misc/getParentProcessId";
@@ -13,6 +13,7 @@ import { getArchivesBackblaze } from "./archivesBackBlaze";
 import { formatNumber } from "socket-function/src/formatting/format";
 import { SizeLimiter } from "../diagnostics/SizeLimiter";
 import { isPublic } from "../config";
+import { measureWrap } from "socket-function/src/profiling/measure";
 
 const SIZE_LIMIT = new SizeLimiter({
     diskRoot: getStorageDir(),
@@ -56,7 +57,7 @@ const getDiskMetricsBase = async () => {
     let usedCacheBytes = 0;
     let usedCacheFiles = 0;
     let cacheFiles = await fs.promises.readdir(cacheArchives2);
-    for (let file of cacheFiles) {
+    async function processFile(file: string) {
         if (file.endsWith(LOCK_SUFFIX)) {
             let base = file.slice(0, -LOCK_SUFFIX.length);
             if (!fs.existsSync(base) && !await isLocked(cacheArchives2 + base)) {
@@ -72,7 +73,7 @@ const getDiskMetricsBase = async () => {
             try {
                 info = await fs.promises.stat(file);
             } catch { }
-            if (!info) continue;
+            if (!info) return;
             fileSizes.push({
                 path: file,
                 time: info.atimeMs,
@@ -92,6 +93,8 @@ const getDiskMetricsBase = async () => {
             }
         }
     }
+    let processFileParallel = runInParallel({ parallelCount: 32 }, processFile);
+    await Promise.all(cacheFiles.map(processFileParallel));
 
     let { remove, availableBytes, availableFiles } = await SIZE_LIMIT.limit(fileSizes);
     for (let file of remove) {
@@ -552,7 +555,9 @@ export function wrapArchivesWithCache(archives: Archives): Archives & {
             // IMPORTANT. Unwrap, otherwise move will get a get, which might cause our value to be cached again
             if (cacheArchivesSymbol in config.target) {
                 config = { ...config };
-                config.target = config.target.getBaseArchives!()!.archives;
+                let base = config.target.getBaseArchives!()!;
+                config.target = base.archives;
+                config.targetPath = base.parentPath + config.targetPath;
             }
             await archives.move(config);
         },