querysub 0.178.0 → 0.181.0

package/spec.txt

@@ -23,6 +23,305 @@ TIMING: Transactions, currently about ~1.5ms per transaction + 10us per write an
 TIMING: Function calls, about ~1ms-3ms for a trivial function, if it can be batched with similar functions
     - Slower if we need to sync new paths
 
+
+Fixed function props / function closure parsing
+    - Presently, if a parent re-renders, and it passed a lambda to a component, that component will always have changed props. HOWEVER, this is very inefficient!
+    1) Process the file to convert lambdas to provide debug information which includes all of the values they close upon (as properties set on the lambda itself). Also an id, unique to that specific piece of code.
+    2) When evaluating if props changed, if the lambda id is the same, and the closed upone variables are ===, then... it's the same thing, so consider the prop unchanged
+    3) Test if all of our "updateOperation" lambdas in SideOpConfig.tsx work with this. I THINK updateOperation is the same, but if it isn't... we can always recognize nested lambdas.
+
+Option to not use permissions checks locks
+    1) Create a way to have any code run without locks (we might already have this?)
+    2) Add a flag in schemas to either specify all functions, or certain functions should run permissions without locks.
+        - Also something to specify a function SHOULD run permissions with locks
+    - Using the ArchiveViewer verify "heartbeat" goes from 25 to 0 locks
+    - If you step into pathValueClientWatcher.ts:setValues in FunctionRunner, you can see the actual paths that are used.
+    - Benchmark a simple function to see how much faster this is, and how much less memory we use
+        - If it doesn't matter... maybe remove the feature?
+
+More corruption resistance files
+    - Add a new serialization format, via versioning in the settings
+    - Each datum will be length prefixed WITH a special sentinel AND with a checksum for the datum data
+        - The sentinel can depend on the length (but otherwise be constant)
+    - We can have a recovery mode (automatically attempted if the checksums are wrong?) where we scan the file for length+sentinel pairs (where the sentinel matches the checksum), see which ones have a valid checksum, and then use an algorithm to decide which ones to use
+        - Maybe we sort by success rate, which is [-numberOfValidChecksumsOverlapped, length], which is very likely to work EVEN if there are values which have the right checksum, but are actuall corrupted.
+            - As long as the checksums are random enough, so... we should never use any small values for checksums, etc.
+    - Record the value layout in the settings, so we can generate arbitrary values, even if we've never seen that layout before.
+        - A series of parallel object arrays, with each object having flags (which indicate which values exist), values which always exist, etc
+            - Value types will be string, float64, byte, Buffer[]
+                - and... we might as well add support for short, int, float, and uint (uint is a good way to store a guid, via storing 8 uint variables).
+
+
+
+
+
+Schema/binary PathValues accesses
+
+    Base code
+        reader
+            let viewTime = 0;
+            for (let user of global().users) {
+                viewTime = user.viewTime;
+            }
+            return viewTime;
+        writer
+            for (let [video, viewTime] of viewedVideos) {
+                global().users[userId].viewTime += viewTime;
+            }
+
+    Schema optimization
+        writer
+            let changeStream = new SchemaPath(() => global().users[wildcard].viewTime);
+            for (let [video, viewTime] of viewedVideos) {
+                changeStream.write(userId, viewTime);
+            }
+        reader
+            let schemaWatcher = new SchemaPath(() => global().users[wildcard].viewTime);
+            watch(() => {
+                let viewTime = 0;
+                for (let [value, userId] of schemaWatcher.getValues()) {
+                    viewTime += value;
+                }
+                return viewTime;
+            });
+        reader specific path
+            // I guess we have to support gets as well. It should still be faster, as the schema is pre-defined
+            let viewTimeWatcher = new SchemaPath(() => global().users[wildcard].viewTime);
+            let factorWatcher = new SchemaPath(() => global().users[wildcard].factor);
+            watch(() => {
+                let viewTime = 0;
+                for (let [value, userId] of schemaWatcher.getValues()) {
+                    let factor = factorWatcher.get(userId);
+                    viewTime += value * factor;
+                }
+                return viewTime;
+            });
+        delta reader (eventually)
+            // This can easily be extrapolated to just getting deltas
+            let deltaWatcher = new DeltaWatcher(() => global().users[wildcard].viewTime);
+            let viewTime = 0;
+            onDelta(() => {
+                for (let [newValue, prevValue, userId] of deltaWatcher.getChanges()) {
+                    viewTime += newValue - prevValue;
+                }
+                return viewTime;
+            });
+
+    - Can be binary, or not.
+    - Streams can work with non-streams, and vice versa.
+        - The streams need to be setup in a tree, so we can efficiently check for watchers of them
+    - We also need to support partial key watching. Often we will watch a few keys (and then within them, maybe all keys at another level).
+
+    1) START by supporting write streams (but NOT read schemas), as this allows us to define our schemas.
+        - This will give us a big chunk, which we will pass around (even passing around arrays of chunks).
+            - The core will break this apart somewhat, with an object per schema, and then a tree of maps for the dynamicValues inside of it (and global lookups for the locks and values)
+        - We will never create this from PathValues, instead, we will append values to specific schemas as we build it
+            - And... locks need to be kept track of as well
+        - So we need a global "captureWrites", to set the state
+            - It will return a function, which will take parameter to finish the writes?
+                - Or... something. We need to look at proxyWatcher and see what the best way to do this is. I think it might check the values before finishing them?
+        - The schema builder functions (on the object returned by defining the schema), will then internally add to a lookup in the globally capturing state (keys by the schema seqNum)
+            - Directly adding to the PathValue for that schema
+                - A lot of fields will only be set on finish
+        - I think we still need to support the helper flags (.valid). Which is actually fine, even if it's binary. It's fairly easy to flip flags in binary data...
+        - When we need to reason about PathValues independently we can provide some kind of pointer than can be used in conjuction with the chunk
+            - Although most of the time we will just provide iterators to iterate over all the PathValues?
+            - I guess eventually the pointer could be serializable too, so that we could select PathValues with a Chunk + Buffer containing pointers, so we entirely manage our own memory. But... probably not for a while, as { schemaId: number; index: number }[] should be VERY efficient to allocate and store, especially if it isn't persisted.
+        // PathValue
+        {
+            schema: ({
+                type: "constant";
+                key: string | number;
+            } | {
+                type: "dynamic";
+            })[];
+            values: {
+                dynamicValues: (string | number)[];
+                // Pointer to position in value Buffer
+                setValue: Pointer;
+                // SeqNum to ReadLockGroup (as often many values will have the same set of ReadLocks)
+                readLocks: Pointer;
+                ...more fields for various PathValue fields
+                // NOTE: ReadLocks need to reference another binary structure
+                //  - Probably via a locally unique seqNum, which is remapped upon receiving data over the network
+            }[];
+        }[]
+        // ReadLockGroups
+        {
+            seqNum: number;
+            lockSeqNums: {
+                schemaSeqNum: number;
+                seqNum: number;
+            }[];
+        }[]
+        // ReadLocks
+        {
+            seqNum: number;
+            schema: ({
+                type: "constant";
+                key: string | number;
+            } | {
+                type: "dynamic";
+            })[];
+            locks: {
+                seqNum: number;
+                dynamicValues: (string | number)[];
+                startTime, endTime, flags, ...;
+            }[];
+        }[]
+        // Values
+        {
+            seqNum: number;
+            // arbitrary binary data
+        }[]
+    2) Isolate this object structure, so it is only manipulated or read via helpers, so we can change it to use a binary format later on
+        - This means adding functions to go to/from binary, even at the start, and just having it go to PathValue and use PathValueSerializer for now.
+        - We will have unique id (a number, locally unique) per schema, so we can very quickly tell if anything might match
+        - OH! use global objects for schema defs, so equivalent schema defs share an object. This lets us store flags for them!
+            - Not the whole schema, just the def part
+            - Just a pointer as well
+            - If we have conflicts with other schemas (now or in the future), this needs to point to the shared data for the combined schema.
+        - Expose the shared objects as createdSharedAccessor<Object>(key) => { get(schema); set(schema, value) }
+    3) THEN, use this structure throughout, replacing PathValues in the entire application with lists of these chunks
+        - We'll probably store in the core in an entirely splayed manner. Maybe the schema, then a tree of dynamic values with the leaves being the set value?
+            - The set value will contain much of the PathValue, excluding the path
+        - Update the core code to store data in schemas
+            - If schemas conflict, we take the more general union of them. Annoying, but this should be rare, as schemas aren't recursive.
+                - Ex, "x.*.z" and "x.y.*" NEED to be combined to "x.*.*", at least for storage.
+                - AND, we need to store the original schemas, and have them MAP to the combined schema!
+            - So we end up with a non-schema data storage, and a bunch of schemas
+            - TWO ways to access data in schemas
+                - If you have a schema, you just directly lookup your data in the shared schema data, which is owned by the core, but every schema points to it
+                - If you don't have a schema...
+                    - We need a lookup of schemas as values with wildcard (I guess empty keys equals a wildcard? Hmm...)
+                        - I think we disallow reading empty keys? If not... maybe we should? They are already used a wildcard watches anyways...
+                    - Check each part of the path, with wildcard filling the rest "x.y.z" => ["*.*.*", "x.*.*", "x.y.*", "x.y.z"]
+                        - OH! That's not enough. Because... "x.*.z" is a valid path, which won't match any of them.
+                        - So... maybe a sorted list?
+                    - Should be reasonably fast...|
+    4) Then update it to be binary
+        - On write
+        - And on network traversal
+        - I think PathValueSerializer needs to be rewritten. Basically, it will be given schemas ahead of time, sometimes (or maybe always, as we might as well always encode in our base schema), and then encode the paths more efficiently.
+        - OH! ReadLocks need some kind of schema as well. Hmm... I guess, they can use the schema of their readers?
+    5) ALSO, create some kind of global string lookup? A lot of string keys will be the same, and it's easier to manipulate/store numbers
+        - We will need to provide definition when we send them over the wire though
+        - THIS is IMPORTANT! Without this our strings become inefficient, and the fastest code would involve mapping all strings (such as userIds), to numbers at an application level, which is a lot of work that the framework should really handle...
+        - Maybe not for ALL string keys? Hmm...
+        - OH! And... they only need to be unique per schema? Hmm...
+        
+        - Test with structures like "components.*.x" (but lots of different x values)
+            - Hmm... the big speed issue is that the component ids don't map to an array. Although, in theory, we could do that, remapping ids to indexes. And then... reuse indexes on gc, so they become direct offsets into binary memory.
+                - Then... I guess we can remember the last mapping, and so quickly go from string to index, with just a === on the strings (which should just be a pointer comparison), which... and even though the index is dynamic, that's still basically just a pointer. So... we could recover a lot of the speed
+            - Maybe we should try this out in a mini-benchmark, with a greatly simplified structure (values constant size, no extra fields, all schemas are just 3 long and have similar structure)?
+        - AH! Have the schema context wrapper (so schema reads/write get registered), give us a context object, which it gives back for the same type of access.
+            - THEN, we can have state per watcher, and... maybe per watcher+schema. Then we can use this to cache the last keys at indexes per schema, per watcher... and for component accesses... this will always be the same, making the mapping instant every time
+    6) Support reading from schemas as well
+        - If the data is already stored in the schema format, querying in this format should be very fast
+        - We will need a shorter read binary format, which has a schema and dynamic values
+    7) AUTOMATIC schema definition generation from createSchema (replacing the code), and replacing (where possible) the uses of the schema to use the schema definition
+        - Probably the hardest part, but... the most important. Without this our application code becomes too difficult to use
+            - Nested accesses become difficult, but... not impossible.
+            - Object assignment needs to become global
+                Local assignment
+                    let list = data().list;
+                    for (let datum of newData) {
+                        list[datum.key] = datum.value;
+                    }
+                Global assigment
+                    let list = data().list;
+                    for (let datum of newData) {
+                        data().list[datum.key] = datum.value;
+                    }
+                Schema based
+                    let list = data().list;
+                    for (let datum of newData) {
+                        listSchema.write(datum.key, datum.value);
+                    }
+
+                Reads are more likely to use highly nested assignment... even cross function
+                    - Cross function stuff is hard.
+                    - We might need to specialize functions?
+                        - As in, per type of possible schema input object, make a new function, and still leave a non-schema function.
+                        - Actually... this should be fairly easy. We aren't going to use === on the functions... ever? And if we do... then I guess it'll fail (or they can mark them as non-specializable, or... we can detect if === is ever used on the function).
+
+    8) Delta watchers
+        - If we store change reasons in the schema format... then we can query deltas for free
+            - Stored in the packed format.
+
+Global nested object state for latest values
+    - Can have multiple global objects, maybe one per collection / schema?
+    - We still need to register reads
+        - If we can get code to not write duplicate reads (which a lot of code can be verified to not do anyways), then tracking the reads becomes very simple with our binary schema data, basically just involving writing numbers (or even just the start/end of ranges) to an output Buffer.
+
+Code transformation for accesses
+    - Add code which can parse a function, find any schema accesses (this requires walking the type tree, and maybe checking other files), and transform the code to call data.q() functions instead (if the schema is a schema2 schema)
+        - Add support for __callerFileName__, etc so this function can determine the caller file, so it can know how to parse it.
+        - At first do this via a stand-alone function call?
+            - And then make it a parameter on the schema?
+        - OR, should we transform the code directly?
+        - If we do it at runtime, we have to figure out capturing scope again. This is hard, and breaks static variables, but... maybe that's okay?
+            - We could always warn about static variables
+            - For any non-static variables (constants, functions, etc), we will need to duplicate them, and anything they access, etc).
+                - THIS is actually REALLY nice. We can add a file name for this so it can be discoverable in devtools, or, the user can set `debugger;`, at which point, they will see a file which just has the function and anything it depends on. This makes debugging easier, as it removes everything not involved in that function (and more importantly, shows everything that IS involved, which could be surprising, as you might miss the fact that certain functions are being called, but if their definition is there you are far less likely to miss it).
+
+Code transformation to turn types into schemas?
+    - If we could do this... we could get fast behavior from normal typescript code
+    - We would need an out in case the code accesses the schema in a non-schema type way (just for accesses, the backend should already be able to handle this).
+
+Binary ValuePaths, which are never decoded
+    1) Allowing accessing a value in a way which returns the intermediate deserialized object (the Buffer[], strings lookup, etc) PLUS the id (or an object?).
+        - Call it, ValuePathBinary
+    2) Allow using ValuePathBinary as an input to SchemaPath parameters, updating the SchemaPath code to directly copy the binary data
+    3) Create a function which exclusively accesses ValuePathBinary, using it for nested sets (getting a value, using that to index another value, etc)
+        - Verify we are never actually decoding the values
+    4) Hack together something where we can NOT decode strings in the ValuePath serializer, sometimes, as our special function will never use strings directly.
+    5) Benchmark to see the benefit (probably via memory pressure), or not decoding the strings?
+
+Automatic commit wrapping of local synced writes
+    - If we write or read from local synced state... and are not in a transaction, we should start a transaction, which exists behind the end of the call, and is only stopped:
+        a) After Promise().finally()
+        b) When any non-local synced state is accessed (which will cause us to throw as well)
+        c) When any transactions are explicitly started
+    - This resolves the issue of having to constantly wrap state writes, which is just annoying
+    - We SHOULD wrap this into a singleton local watcher which we re-use for local events ("once" watchers). This will make events faster, and should make it easier to implement.
+ALSO
+    - Allow accessing local state values without being in a synced state?
+
+Server crash log protection
+    - Up to 10 times per hour, synchronously write unclassified logs to special single log blocks
+    - We will delete these special blocks when we finally write the logs
+    - This way if we get a fatal error, the console.error before we crash (which HOPEFULLY) we get, will let us know what went wrong.
+    - OR... maybe not... maybe the server manager should just watch for crashes and log the most recent stdout/stderr logs?
+
+PathValueController direct remote database support
+    --ext-remote "D:/code/other/loader.ts"
+        {
+            /** undefined means it doesn't need to be loaded remotely */
+            loadRemoteValue(path): Promise<unknown | undefined> | undefined;
+        }
+        We'll probably have a timeout, just providing epochValues if loadRemoteValue takes more than a few seconds
+    We COULD allow for core to be resolved to the same thing independent of version (by making the core register itself globally, and all cores to use the same thing).
+        - AND, we could do the same thing for a few other files, such as clientWatcher, etc
+        - This would allow clientWatcher.setValues to be called explicitly
+        - This MIGHT be useful, but... then again... it adds a lot of complexity
+
+Hot render throttle
+    - We want all inputs to be hot, BUT, to not render on each keydown. So... we need to delay rendering triggering?
+        - BUT, we run into issues if a button then runs, which expects to have the most recent data closed upon? Ugh...
+
+Auto commit mode
+    - If anything is accessed outside of synced state, enter a commit, and end that commit when:
+        a) Promise.resolve().finally
+        b) We try to end any other commit
+    - Should be a Querysub config value which is on by default
+    - VERY useful clientside
+    - We need to support "lazyClose" watcher in proxyWatcher, that DOESN'T reset the runningWatcher UNTIL it is explicitly reset, OR, until another watcher tries to run
+    - We need to detect synchronized accesses in proxyWatcher and start this special type of commit
+    - We need to know if we are in this state, and THROW if we access any non-synced paths
+        - This is mostly for non-local paths
+        - Throwing WHEN they paths are accesses is required, otherwise the error location is too annoying
+
 Watcher diff mode
     - Running stress tests (satTest.ts) with a single watcher shows that the slowest part is our function watcher, specifically all of the parts that don't operate in a delta mode (ex, setWatches).
     - If we made a watcher fully support a delta mode (which is fairly easy, as it converts all the changes to deltas anyways), the watcher would likely work MUCH faster.
@@ -126,14 +425,6 @@ Turn on and test backblaze storage
     - Test starting a server on another machine
     - After setting up the .json keys files, it... should just work?
 
-Undo support
-    - Not needed for games, but needed for every other application
-    - Probably by marking certain functions as "undoable", then showing (in the app UI), a list of "versions" (time points) at a certain path (showing all the changes under a path), and letting the user pick a point. Anything that is undoable can just be reverted, creating new writes, everything that isn't undoable... well, any non-undoable changes cause the undo to not be able to go that far
-        - ALSO, anything that depends on writes has to be undoable as well, and... also undone (ideally nothing outside the undo scope would depend on it)
-        - Alternatively, dependencies could be made "undoIgnored", which would allow them to stick around, ignoring that fact that their write has been clobbered.
-            - OR, maybe we just ignore dependencies, as... undoing adds new writes, so anything that is undoable has to be something where reversing is fine?
-            - OR, we made multiple undo modes, "safe", and "unsafe"? Or different undo flags, for failing on dependencies, or not failing? Hmm...
-
 Anti-rejection code (isn't REQUIRED to make the database useful, so we should wait. Would be pretty slick though... Ideally this can all be extensions that have no or only modular impact on the core functions, or even no or modular impact on the proxy? Although that might not be possible...)
     Summary
         Excess work (N^2) due to rejections
@@ -253,24 +544,6 @@ Anti-rejection code (isn't REQUIRED to make the database useful, so we should wa
 
     ++ Makes system a lot faster, by reducing rejections that don't change the value, and by increasing the efficiency of comparisons
 
-Explicit schema objects accesses
-    - Allow defining a strict schema, with defines the names of both the keys and values, with fixed values just being implicit to the schema, and not requiring a key or any storage.
-    - Would access values via the schema, which changes how to PathValue is interpretted
-        - ALSO, we encode the value using the schema, so that when it is both encoded and interpretted with the schema, the operation becomes VERY fast.
-    - ONLY for wire transfer, not archival storage
-        - We can probably encode the schema when transferring it, storing all values of the same schema in a row, making it highly compresssed, and efficient to decode.
-    - Could be used for function calls, which would make permission checks faster
-    - Could help permission checks too
-        - Would have to support multiple levels, so you can check the root permission check, etc (if all the levels use the schema object, this then becomes very efficient)
-    - By default key schemas would derive from the Data schema
-    - If values weren't serialized using the schema we can quickly fallback to using the path+value
-    - We need a string wrapper so we can pass PathValue and then use our schema to access it the path on it
-        - We also need to make ALL PathValue.path accesses use this helper function, even if they aren't providing a schema.
-            - Right now we have 52 references to PathValue.path, so... this is very doable. Of course this just covers compatibility, but we would also want to allow not even decoding the path, such as in cases where we are just storing it in a Map and checking for ===.
-    - A way to access paths to get a value that is not the path, but will preserve === (just threadwide is fine)
-        - Using a number would be plausible. IF the path comes from a schema we can make the first 32 bits be associated with the schema, which can make finding a new number a lot faster.
-    - AND THEN... the end goal, is to have functions which ONLY use schemas, and then transpile them to a schema-full language (such as C#, or even C).
-
 Automatic large value storage separation
     - If a value is > ~10MB, when we write it to archives, we should ALWAYS break it into another file
         - Have this decided via a flag on the PathValue, which can be dynamically set via size, or explicitly set.

package/src/-c-identity/IdentityController.ts

@@ -7,7 +7,7 @@
 import debugbreak from "debugbreak";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { CallerContext } from "socket-function/SocketFunctionTypes";
-import { cache, lazy } from "socket-function/src/caching";
+import { cache, cacheWeak, lazy } from "socket-function/src/caching";
 import { getClientNodeId, getNodeId, getNodeIdDomain, getNodeIdIP, getNodeIdLocation, isClientNodeId } from "socket-function/src/nodeCache";
 import { getCommonName, getIdentityCA, getMachineId, getPublicIdentifier, getThreadKeyCert, parseCert, sign, validateCertificate, verify } from "../-a-auth/certs";
 import { getShortNumber } from "../bits";
@@ -175,7 +175,8 @@ const IdentityController = SocketFunction.register(
     })
 );
 
-const changeIdentityOnce = cache(async function changeIdentityOnce(connectionId: { nodeId: string }) {
+// IMPORTANT! We need to cache per connection, not per nodeId, so caching based on connectionId is required!
+const changeIdentityOnce = cacheWeak(async function changeIdentityOnce(connectionId: { nodeId: string }) {
     let nodeId = connectionId.nodeId;
     let threadKeyCert = getThreadKeyCert();
     let issuer = getIdentityCA();

package/src/-d-trust/NetworkTrust2.ts

@@ -6,13 +6,14 @@ import { SocketFunctionHook } from "socket-function/SocketFunctionTypes";
 import { SocketFunction } from "socket-function/SocketFunction";
 import { IdentityController_getMachineId } from "../-c-identity/IdentityController";
 import { cache, lazy } from "socket-function/src/caching";
-import { getNodeIdDomainMaybeUndefined, getNodeIdLocation } from "socket-function/src/nodeCache";
+import { getNodeIdDomainMaybeUndefined, getNodeIdIP, getNodeIdLocation } from "socket-function/src/nodeCache";
 import { trustCertificate } from "socket-function/src/certStore";
 import { isClient, isServer } from "../config2";
 import debugbreak from "debugbreak";
-import { devDebugbreak, getDomain, isDevDebugbreak, isPublic } from "../config";
+import { devDebugbreak, getDomain, isDevDebugbreak, isPublic, isRecovery } from "../config";
 import { formatTime } from "socket-function/src/formatting/format";
 import { runInSerial } from "socket-function/src/batching";
+import { Querysub } from "../4-querysub/QuerysubController";
 
 // Cache the untrust list, to prevent bugs from causing too many backend reads (while also allowing
 //  bad servers which make request before their trust is verified from staying broken).
@@ -34,6 +35,10 @@ export const requiresNetworkTrustHook: SocketFunctionHook = async config => {
     if (isClient()) {
         return;
     }
+    let caller = SocketFunction.getCaller();
+    if (getNodeIdIP(caller.nodeId) === "127.0.0.1" && isRecovery()) {
+        return;
+    }
     let machineId = IdentityController_getMachineId(SocketFunction.getCaller());
     let trusted = await isTrusted(machineId);
     if (!trusted) {

package/src/-f-node-discovery/NodeDiscovery.ts

@@ -112,9 +112,12 @@ export async function getAllNodeIds() {
 
     // NOTE: We always wait for the time sync before returning any node ids. Because if we try to talk
     //  to remote nodes with a massively out of sync time, they will likely reject our messages!
-    await measureBlock(async () => {
-        await waitForFirstTimeSync();
-    }, "waitForFirstTimeSync");
+    let promise = waitForFirstTimeSync();
+    if (promise) {
+        await measureBlock(async () => {
+            await promise;
+        }, "waitForFirstTimeSync");
+    }
 
     if (nodeOverrides) {
         return nodeOverrides;

package/src/-h-path-value-serialize/PathValueSerializer.ts

@@ -612,7 +612,9 @@ class PathValueSerializer {
                         await delay("paintLoop");
                     }
                 }
-                await delay("paintLoop");
+                if (stringBuffers.length > 1) {
+                    await delay("paintLoop");
+                }
             }
             strings = stringArrays.flat();
         }

package/src/0-path-value-core/PathValueCommitter.ts

@@ -123,7 +123,7 @@ class PathValueCommitter {
     }
 
     private broadcastValues = batchFunction(
-        { delay: 10, throttleWindow: 500 },
+        { delay: 10, throttleWindow: 500, noMeasure: true },
         async function internal_forwardWrites(valuesBatched: Set<PathValue>[]) {
             let values = new Set(valuesBatched.flatMap(x => Array.from(x)));
 
@@ -226,7 +226,7 @@ class PathValueCommitter {
      *          at a rate faster than that will just pointlessly add lag.
      */
     public ingestRemoteValues = batchFunction(
-        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValues" },
+        { delay: 16, throttleWindow: 1000, name: "ingestRemoteValues", noMeasure: true },
         async (batched: {
             pathValues: PathValue[],
             parentsSynced?: string[];

package/src/0-path-value-core/PathValueController.ts

@@ -211,7 +211,8 @@ export const PathValueController = SocketFunction.register(
     "PathValueController-1e062e2c-81c9-497b-b414-a46d0a4c2313",
     new PathValueControllerBase(),
     () => ({
-        forwardWrites: {},
+        forwardWrites: {
+        },
 
         watchLockValid: {},
         onValidChange: {},
@@ -226,6 +227,9 @@ export const PathValueController = SocketFunction.register(
     }),
     () => ({
         hooks: [requiresNetworkTrustHook],
-    })
+    }),
+    {
+        noFunctionMeasure: !isNode(),
+    }
 );
 

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -313,6 +313,7 @@ class TransactionLocker {
         return path;
     }
     private async deleteDataFile(key: string, reason: string): Promise<void> {
+        console.log(red(`Deleting data file ${key}, because ${reason}`));
         //await this.storage.setValue(key + ".reason", Buffer.from(reason));
         // Delete file, and confirmation as well
         await this.storage.deleteKey(key);
@@ -370,6 +371,8 @@ class TransactionLocker {
         let value = Buffer.from(JSON.stringify(strippedTransaction));
         await this.storage.setValue(key, value);
     }
+
+    private lastFilesRead: FileInfo[] | undefined;
     private async readDataState(): Promise<{
         rawDataFiles: FileInfo[];
         /** Confirmed FileInfos are === the FileInfos in rawDataFiles */
@@ -384,6 +387,16 @@ class TransactionLocker {
         let bufferCache = new Map<string, Buffer>();
         const tryToRead = async () => {
             let files = await this.storage.getKeys();
+            if (this.lastFilesRead) {
+                let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
+                let newFiles = files.filter(a => !prevFiles.has(a.file));
+                let pastTime = Date.now() - ARCHIVE_PROPAGATION_TIME * 10;
+                let veryBadFiles = newFiles.filter(x => x.createTime < pastTime);
+                if (veryBadFiles.length > 0) {
+                    console.error(`Old files suddenly appeared. This isn't possible, if they are old, they should have appeared when they were created! This likely means that our getKeys() failed to actually read all of the files. This is bad and can result in us deleting seemingly broken files for missing a confirmation, when they in fact had a confirmation.`, { files: veryBadFiles.map(x => x.file) });
+                }
+            }
+            this.lastFilesRead = files;
 
             let transactions: (Transaction & {
                 seqNum: number;
@@ -550,7 +563,7 @@ class TransactionLocker {
         let createCount = transaction.ops.filter(a => a.type === "create").length;
         let deleteCount = transaction.ops.filter(a => a.type === "delete").length;
         let lockedFiles = transaction.lockedFilesMustEqual?.length;
-        diskLog(`Applying transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        diskLog(`Applying transaction with ${createCount} file creates and ${deleteCount} file deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
         logNodeStats(`archives|TΔ Apply`, formatNumber, 1);
@@ -576,7 +589,7 @@ class TransactionLocker {
         };
         await Promise.all(list(CONCURRENT_WRITE_COUNT).map(runThread));
 
-        diskLog(`Applied transaction with ${createCount} creates and ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
+        diskLog(`Applied transaction with ${createCount} file creates and file ${deleteCount} deletes. ${lockedFiles !== undefined && `Lock state depends on ${lockedFiles} files` || ""}`, {
             transactions: transaction.ops.map(x => JSON.stringify(x)),
         });
     }
@@ -681,7 +694,7 @@ class TransactionLocker {
                     //  where backblaze returns no files? Which it might be doing, as multiple times this code has
                     //  triggered (without this check), and deletes all of our files...
                     let unconfirmedOldFiles2 = veryOldFiles.filter(a => !doubleCheckLookup.has(a) && doubleCheckDataFiles.has(a.file));
-                    console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`));
+                    console.warn(red(`Deleted ${unconfirmedOldFiles2.length} very old unconfirmed files`), { files: unconfirmedOldFiles2.map(x => x.file) });
                     diskLog(`archives|TΔ Delete Old Rejected File`, formatNumber, unconfirmedOldFiles2.length);
                     // At the point the file was very old when we started reading, not part of the active transaction.
                     for (let file of unconfirmedOldFiles2) {
@@ -697,11 +710,20 @@ class TransactionLocker {
                 let usedConfirmations = new Set(dataState.confirmedDataFiles.map(a => this.getConfirmKey(a.file)));
                 let oldEnoughConfirms = dataState.rawDataFiles.filter(x => x.file.endsWith(".confirm") && x.createTime < oldThreshold);
                 let deprecatedFiles = oldEnoughConfirms.filter(a => !usedConfirmations.has(a.file));
-                if (deprecatedFiles.length > 0) {
-                    console.warn(red(`Deleted ${deprecatedFiles.length} / ${oldEnoughConfirms.length} confirmations, for not having corresponding data files`));
-                    logNodeStats(`archives|TΔ Delete Deprecated Confirm`, formatNumber, deprecatedFiles.length);
-                    for (let file of deprecatedFiles) {
-                        await this.storage.deleteKey(file.file);
+                if (deprecatedFiles.length > 0 && dataState.rawDataFiles.length > 0) {
+                    let dataState = await this.readDataState();
+
+                    let usedConfirmations = new Set(dataState.confirmedDataFiles.map(a => this.getConfirmKey(a.file)));
+                    let oldEnoughConfirms = dataState.rawDataFiles.filter(x => x.file.endsWith(".confirm") && x.createTime < oldThreshold);
+                    let doubleCheckLookup = new Set(oldEnoughConfirms.filter(a => !usedConfirmations.has(a.file)).map(a => a.file));
+
+                    deprecatedFiles = deprecatedFiles.filter(a => doubleCheckLookup.has(a.file));
+                    if (deprecatedFiles.length > 0 && dataState.rawDataFiles.length > 0) {
+                        console.warn(red(`Deleted ${deprecatedFiles.length} / ${oldEnoughConfirms.length} confirmations, for not having corresponding data files`), { files: deprecatedFiles.map(x => x.file) });
+                        logNodeStats(`archives|TΔ Delete Deprecated Confirm`, formatNumber, deprecatedFiles.length);
+                        for (let file of deprecatedFiles) {
+                            await this.storage.deleteKey(file.file);
+                        }
                     }
                 }
             }
@@ -737,7 +759,10 @@ class TransactionLocker {
         let dels = transaction.ops.filter(a => a.type === "delete").length;
         let creates = transaction.ops.filter(a => a.type === "create").length;
         let createBytes = transaction.ops.map(a => a.type === "create" && a.value?.length || 0).reduce((a, b) => a + b, 0);
-        diskLog(`Starting transaction with ${creates} creates and ${dels} deletes, ${formatNumber(createBytes)}B`);
+        diskLog(`Starting transaction with ${creates} file creates and ${dels} file deletes, ${formatNumber(createBytes)}B`, {
+            createFilesNames: transaction.ops.filter(a => a.type === "create").map(a => a.key),
+            deleteFilesNames: transaction.ops.filter(a => a.type === "delete").map(a => a.key),
+        });
         transaction = { ...transaction, ops: transaction.ops.slice() };
         function normalizePath(path: string) {
             // Replace duplicate slashes with a single slash

package/src/0-path-value-core/pathValueArchives.ts

@@ -326,12 +326,19 @@ export class PathValueArchives {
 
         let dataValues: PathValue[] = [];
 
+        let decodedObj = pathValueArchives.decodeDataPath(path);
+
         try {
             dataValues = await pathValueSerializer.deserialize([data], {
                 singleBuffer: true,
                 skipStrings: config.skipStrings,
                 skipValues: config.skipValues,
             });
+            if (dataValues.length !== decodedObj.valueCount) {
+                require("debugbreak")(2);
+                debugger;
+                console.error(`Bad archive data file at ${config.path}, Decoded count ${formatNumber(decodedObj.valueCount)} !== count in file name ${formatNumber(dataValues.length)} (${decodedObj.valueCount} !== ${dataValues.length})`);
+            }
         } catch (e: any) {
             console.log(red(`Bad archive data file at ${config.path}, error: ${e.stack}`));
         }

package/src/0-path-value-core/pathValueCore.ts

@@ -224,7 +224,7 @@ export type PathValue = {
     time: Time;
     /** @deprecated NOT deprecated, just remember when you set this ALWAYS set lockCount, otherwise
      *      valid checking breaks!
-     *  IMPORTANT! DO NOT check for locks.length === 0. Locks are lost during serialization, so this
+     *  IMPORTANT! DO NOT check for locks.length === 0. Locks are often dropped during serialization, so this
      *      will be irrelevant. lockCount on the other hand, should be accurate.
      */
     locks: ReadLock[];

package/src/1-path-client/RemoteWatcher.ts

@@ -148,7 +148,7 @@ export class RemoteWatcher {
     }
 
     private watchUnwatchSerial = runInSerial((name: string, fnc: () => Promise<unknown>) =>
-        measureBlock(fnc, name)
+        fnc()
     );
 
     /** NOTE: We dedupe duplicate watches in watchLatest. */