querysub 0.680.0 → 0.682.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "querysub",
-  "version": "0.680.0",
+  "version": "0.682.0",
   "main": "index.js",
   "license": "MIT",
   "note1": "note on node-forge fork, see https://github.com/digitalbazaar/forge/issues/744 for details",

package/src/0-path-value-core/PathRouter.ts

@@ -322,9 +322,9 @@ export class PathRouter {
     //  - If this becomes an issue we COULD filter, as we can do it quickly, but I don't think it is required, as all the present usecases prefilter anyways.
     @measureFnc
     public static async getPathIdentifierTargets(values: PathValue[], ourSpec: AuthoritySpec): Promise<Map<string, PathValue[]>> {
-        // NOTE: The file size limit is 1024 bytes. But we also have our folder, etc, so we want to add enough buffer
+        // NOTE: The identifier becomes a single directory name, which linux caps at 255 bytes (see MAX_IDENTIFIER_LENGTH), so the range numbers plus this many prefix hashes must always fit within that.
         //  - Shorter hashes means we can store more, but there's a point when the collisions make it less useful.
-        const MAX_PREFIXES_PER_FILE = 50;
+        const MAX_PREFIXES_PER_FILE = 20;
         const PREFIX_COVER_FRACTION = 0.99;
         const TARGET_VALUES_PER_SHARD_GROUP = 10 * 1000 * 1000;
         const TARGET_SHARD_SIZE = 50 * 1000;

package/src/0-path-value-core/archiveLocks/ArchiveLocks2.ts

@@ -209,7 +209,7 @@ export function createArchiveLocker2(config: {
                 readFiles: (files: FileInfo[]) => Promise<(Buffer | undefined)[]>,
             ) => Promise<ArchiveTransaction[]>
         ): Promise<"accepted" | "rejected"> {
-            let files = await locker.getFiles();
+            let files = await locker.getFiles({ forTransaction: true });
             await saveSnapshot({ files: files.map(a => a.file) });
             let readFiles = async (files: FileInfo[]) => {
                 let pendingFiles = files.slice();
@@ -494,21 +494,23 @@ class TransactionLocker {
             let time = Date.now();
             let files = await this.storage.getKeys(config);
             this.perf.checkpoint("getKeys");
-            if (this.lastFilesRead && this.lastFilesReadTime) {
-                let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
-                let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
-                let newFiles = files.filter(a => !prevFiles.has(a.file));
-                let veryBadFiles = newFiles.filter(x => x.createTime < suspiciousThreshold);
-                if (veryBadFiles.length > 0) {
-                    console.error(`Old read was not exhaustive, because we just found files which existed well before it, but that it did not read. This likely means our propagation assumptions are wrong, and propagation is much slower than anticipated. OR, getKeys is flakey and randomly misses files? Either way, this will BREAK THE DATABASE! Any node experiencing this might start deleting valid database files, which is very bad!`, {
-                        newlyAppearingOldFiles: veryBadFiles.map(x => x.file),
-                        prevFiles: this.lastFilesRead.map(x => x.file),
-                        newFiles: files.map(x => x.file),
-                    });
+            if (!config?.fallbacks) {
+                if (this.lastFilesRead && this.lastFilesReadTime) {
+                    let prevFiles = new Set(this.lastFilesRead.map(a => a.file));
+                    let suspiciousThreshold = this.lastFilesReadTime - ARCHIVE_PROPAGATION_TIME * 10;
+                    let newFiles = files.filter(a => !prevFiles.has(a.file));
+                    let veryBadFiles = newFiles.filter(x => x.createTime < suspiciousThreshold);
+                    if (veryBadFiles.length > 0) {
+                        console.error(`Old read was not exhaustive, because we just found files which existed well before it, but that it did not read. This likely means our propagation assumptions are wrong, and propagation is much slower than anticipated. OR, getKeys is flakey and randomly misses files? Either way, this will BREAK THE DATABASE! Any node experiencing this might start deleting valid database files, which is very bad!`, {
+                            newlyAppearingOldFiles: veryBadFiles.map(x => x.file),
+                            prevFiles: this.lastFilesRead.map(x => x.file),
+                            newFiles: files.map(x => x.file),
+                        });
+                    }
                 }
+                this.lastFilesRead = files;
+                this.lastFilesReadTime = time;
             }
-            this.lastFilesRead = files;
-            this.lastFilesReadTime = time;
 
             let transactions: (Transaction & {
                 seqNum: number;
@@ -794,7 +796,7 @@ class TransactionLocker {
     /** Only returns data files (no transaction files, or confirmations).
      *      - Might run a transaction
      */
-    public async getFiles(): Promise<FileInfo[]> {
+    public async getFiles(config?: { forTransaction?: boolean }): Promise<FileInfo[]> {
         let time = Date.now();
         let storageAlivePromise = errorToUndefinedSilent(this.storage.getSyncStatus());
         try {
@@ -805,7 +807,7 @@ class TransactionLocker {
             return obj.dataFiles;
         } catch (e) {
             let duration0 = Date.now() - time;
-            if (await storageAlivePromise) {
+            if (config?.forTransaction || await storageAlivePromise) {
                 throw e;
             }
             let duration = Date.now() - time;

package/src/2-proxy/PathValueProxyWatcher.ts

@@ -1383,7 +1383,7 @@ export class PathValueProxyWatcher {
                     },
                     code() {
                         return currentReadSource.temporaryOverride(options.overrides, () =>
-                            runCodeWithDatabase(proxy, baseFunction)
+                            runCodeWithDatabase(proxy, curFunction)
                         );
                     },
                 }) as Result;

package/src/5-diagnostics/Modal.tsx

@@ -83,13 +83,21 @@ export function showModal(config: {
 } {
     ensureRendering();
     let id = nextId();
-    Querysub.commit(() => {
+    if (Querysub.isInSyncedCall()) {
         data().modals[id] = atomicObjectWriteNoFreeze({
             value: config.content,
             onClose: config.onClose,
             onlyExplicitClose: config.onlyCloseExplicitly,
         });
-    });
+    } else {
+        Querysub.commit(() => {
+            data().modals[id] = atomicObjectWriteNoFreeze({
+                value: config.content,
+                onClose: config.onClose,
+                onlyExplicitClose: config.onlyCloseExplicitly,
+            });
+        });
+    }
     function close() {
         Querysub.commit(() => closeModal(id));
     }

package/src/diagnostics/FunctionCallInfoState.ts

@@ -2,57 +2,41 @@ import { Querysub } from "../4-querysub/Querysub";
 import { onPredictionFinished } from "../4-querysub/querysubPrediction";
 import { lazy } from "socket-function/src/caching";
 import { t } from "../2-proxy/schema2";
-import { StatsValue, createStatsValue, addToStatsValue } from "socket-function/src/profiling/stats";
 import { authorityStorage } from "../0-path-value-core/pathValueCore";
 import { pathWatcher } from "../0-path-value-core/PathWatcher";
 import { proxyWatcher } from "../2-proxy/PathValueProxyWatcher";
 
-interface CallStatsData {
+export interface FunctionCallStats {
     totalCalls: number;
-    totalInternalReruns: number;
-    totalFullReruns: number;
-    callsWithMultipleInternalRuns: number;
-    callsWithCascadingRuns: number;
-    totalInternalRerunsForCascading: number;
     maxInternalReruns: number;
-    callsWithMultipleFullRuns: number;
-    evalTimeStats: StatsValue;
-    timeTakenStats: StatsValue;
-    totalTimeStats: StatsValue;
+    maxFullReruns: number;
+    maxEvalTime: number;
+    maxTotalTime: number;
+}
+
+interface CallStatsData extends FunctionCallStats {
     localPathCount: number;
     remotePathCount: number;
     totalValueCount: number;
     proxyWatcherCount: number;
     callsByServer: { [creatorId: string]: number };
     perFunctionStats: {
-        [functionId: string]: {
-            totalCalls: number;
-            totalInternalReruns: number;
-            totalFullReruns: number;
-            callsWithMultipleInternalRuns: number;
-            callsWithCascadingRuns: number;
-            totalInternalRerunsForCascading: number;
-            maxInternalReruns: number;
-            callsWithMultipleFullRuns: number;
-            evalTimeStats: StatsValue;
-            timeTakenStats: StatsValue;
-            totalTimeStats: StatsValue;
-        }
+        [functionId: string]: FunctionCallStats;
+    };
+}
+
+function createFunctionCallStats(): FunctionCallStats {
+    return {
+        totalCalls: 0,
+        maxInternalReruns: 0,
+        maxFullReruns: 0,
+        maxEvalTime: 0,
+        maxTotalTime: 0,
     };
 }
 
 let statsData: CallStatsData = {
-    totalCalls: 0,
-    totalInternalReruns: 0,
-    totalFullReruns: 0,
-    callsWithMultipleInternalRuns: 0,
-    callsWithCascadingRuns: 0,
-    totalInternalRerunsForCascading: 0,
-    maxInternalReruns: 0,
-    callsWithMultipleFullRuns: 0,
-    evalTimeStats: createStatsValue(),
-    timeTakenStats: createStatsValue(),
-    totalTimeStats: createStatsValue(),
+    ...createFunctionCallStats(),
     localPathCount: 0,
     remotePathCount: 0,
     totalValueCount: 0,
@@ -76,64 +60,20 @@ export function callState(): CallStatsData {
 
 export let ensureSubscribed = lazy(() => {
     onPredictionFinished((data) => {
-        statsData.totalCalls++;
         statsData.callsByServer[data.creatorId] = (statsData.callsByServer[data.creatorId] || 0) + 1;
-        statsData.totalInternalReruns += data.result.totalInternalLoopCount - 1;
-        statsData.totalFullReruns += data.result.outerLoopCount - 1;
-        addToStatsValue(statsData.evalTimeStats, data.result.evalTime);
-        addToStatsValue(statsData.timeTakenStats, data.result.timeTaken);
-        addToStatsValue(statsData.totalTimeStats, data.result.totalTime);
-
-        if (data.result.totalInternalLoopCount > 1) {
-            statsData.callsWithMultipleInternalRuns++;
-        }
-        if (data.result.totalInternalLoopCount > 2) {
-            statsData.callsWithCascadingRuns++;
-            statsData.totalInternalRerunsForCascading += data.result.totalInternalLoopCount - 1;
-            if (data.result.totalInternalLoopCount > statsData.maxInternalReruns) {
-                statsData.maxInternalReruns = data.result.totalInternalLoopCount;
-            }
-        }
-        if (data.result.outerLoopCount > 1) {
-            statsData.callsWithMultipleFullRuns++;
-        }
-
-        if (!(data.functionId in statsData.perFunctionStats)) {
-            statsData.perFunctionStats[data.functionId] = {
-                totalCalls: 0,
-                totalInternalReruns: 0,
-                totalFullReruns: 0,
-                callsWithMultipleInternalRuns: 0,
-                callsWithCascadingRuns: 0,
-                totalInternalRerunsForCascading: 0,
-                maxInternalReruns: 0,
-                callsWithMultipleFullRuns: 0,
-                evalTimeStats: createStatsValue(),
-                timeTakenStats: createStatsValue(),
-                totalTimeStats: createStatsValue(),
-            };
-        }
 
         let fnStats = statsData.perFunctionStats[data.functionId];
-        fnStats.totalCalls++;
-        fnStats.totalInternalReruns += data.result.totalInternalLoopCount - 1;
-        fnStats.totalFullReruns += data.result.outerLoopCount - 1;
-        addToStatsValue(fnStats.evalTimeStats, data.result.evalTime);
-        addToStatsValue(fnStats.timeTakenStats, data.result.timeTaken);
-        addToStatsValue(fnStats.totalTimeStats, data.result.totalTime);
-
-        if (data.result.totalInternalLoopCount > 1) {
-            fnStats.callsWithMultipleInternalRuns++;
-        }
-        if (data.result.totalInternalLoopCount > 2) {
-            fnStats.callsWithCascadingRuns++;
-            fnStats.totalInternalRerunsForCascading += data.result.totalInternalLoopCount - 1;
-            if (data.result.totalInternalLoopCount > fnStats.maxInternalReruns) {
-                fnStats.maxInternalReruns = data.result.totalInternalLoopCount;
-            }
+        if (!fnStats) {
+            fnStats = createFunctionCallStats();
+            statsData.perFunctionStats[data.functionId] = fnStats;
         }
-        if (data.result.outerLoopCount > 1) {
-            fnStats.callsWithMultipleFullRuns++;
+
+        for (let stats of [statsData as FunctionCallStats, fnStats]) {
+            stats.totalCalls++;
+            stats.maxInternalReruns = Math.max(stats.maxInternalReruns, data.result.totalInternalLoopCount - 1);
+            stats.maxFullReruns = Math.max(stats.maxFullReruns, data.result.outerLoopCount - 1);
+            stats.maxEvalTime = Math.max(stats.maxEvalTime, data.result.evalTime);
+            stats.maxTotalTime = Math.max(stats.maxTotalTime, data.result.totalTime);
         }
 
         Querysub.commit(() => {

package/src/diagnostics/logs/errorTickets/TicketPage.tsx

@@ -10,17 +10,22 @@ import { URLParam } from "../../../library-components/URLParam";
 import { mainResets } from "../../../library-components/urlResetGroups";
 import { ATag } from "../../../library-components/ATag";
 import { Button } from "../../../library-components/Button";
+import { DropdownSelector } from "../../../library-components/DropdownSelector";
 import { InputLabel } from "../../../library-components/InputLabel";
 import { LogDatum } from "../diskLogger";
 import { managementPageURL, showingManagementURL } from "../../managementPages";
 import { TicketsController, watchTickets } from "./tickets";
-import { isTicketFinished, Ticket, TicketComment, TicketPatchFile, TicketState, TICKET_STATES } from "./ticketTypes";
+import { AUTOFIXER_MODEL_OPTIONS, isTicketFinished, Ticket, TicketComment, TicketPatchFile, TicketState, TICKET_STATES } from "./ticketTypes";
 
 export const ticketIdURL = new URLParam("ticketid", "");
+// Empty means claude's default model.
+export const aiModelURL = new URLParam("aimodel", "");
 // "unfinished" hides tickets in a final state (fixed / not-a-bug). Resets when the page changes, so the filter doesn't confusingly stick around.
 export const ticketFilterURL = new URLParam("ticketfilter", "", { reset: [mainResets] });
 
 const TITLE_MAX_LENGTH = 200;
+// Ticket change notifications already refresh the AI status on every AI step; the poll just catches transitions that don't touch the ticket (e.g. the AI server restarting) and keeps the elapsed time ticking.
+const AI_STATUS_POLL_INTERVAL = 5000;
 const COMMENT_TEXTAREA_MIN_HEIGHT = 250;
 const COMMENT_TEXTAREA_COLLAPSED_HEIGHT = 40;
 
@@ -167,6 +172,7 @@ export class TicketPage extends qreact.Component {
             let controller = getController();
             controller.getTickets.refreshAll();
             controller.getTicket.refreshAll();
+            controller.getAIStatus.refreshAll();
         }).then(unwatch => {
             this.unwatch = unwatch;
         });
@@ -221,7 +227,7 @@ class TicketList extends qreact.Component {
             <div className={css.hbox(8).alignItems("center")}>
                 <span>Run</span>
                 <code className={css.pad2(8, 4).hsl(220, 15, 15).colorhsl(120, 60, 70).fontSize(13).borderRadius(3)}>yarn autofix</code>
-                <span>to automatically investigate and fix open tickets.</span>
+                <span>to automatically investigate and fix open tickets, or open a ticket and use "Run AI Investigation" to run one on the server you are connected to.</span>
             </div>
             <div className={css.vbox(8).pad2(12).bord2(210, 50, 60).hsl(210, 50, 96).fillWidth}>
                 <strong>Create Custom Ticket</strong>
@@ -324,7 +330,10 @@ class TicketDetail extends qreact.Component<{ ticketId: string }> {
         }
 
         let sortedComments = [...ticket.comments];
-        sort(sortedComments, x => x.time);
+        sort(sortedComments, x => -x.time);
+        // Chronological, because patches on the same file build on each other in the order they were proposed.
+        let patchComments = ticket.comments.filter(c => c.kind === "patch" && c.patchFiles);
+        sort(patchComments, x => x.time);
         return <div className={css.vbox(16).pad2(16).fillBoth.maxWidth("100%").minHeight(0)}>
             <div className={css.hbox(16).alignItems("center")}>
                 <ATag values={[ticketIdURL.getOverride("")]}>← All Tickets</ATag>
@@ -368,6 +377,8 @@ class TicketDetail extends qreact.Component<{ ticketId: string }> {
                 </Button>
             </div>
 
+            <AIRunPanel ticketId={this.props.ticketId} />
+
             <div className={css.vbox(8).pad2(12).bord2(0, 50, 70).hsl(0, 30, 96).fillWidth}>
                 <div
                     className={css.hbox(8).button}
@@ -398,7 +409,24 @@ class TicketDetail extends qreact.Component<{ ticketId: string }> {
             </div>
 
             <div className={css.vbox(12).fillWidth.flexGrow(1).minHeight(0).overflowAuto}>
-                <h3>Comments ({sortedComments.length})</h3>
+                <div className={css.hbox(16).alignItems("center")}>
+                    <h3 className={css.margin(0)}>Comments ({sortedComments.length})</h3>
+                    {patchComments.length > 0 && (
+                        <Button
+                            hue={120}
+                            onClick={() => {
+                                let ticketId = this.props.ticketId;
+                                let commentIds = patchComments.map(c => c.id);
+                                Querysub.onCommitFinished(async () => {
+                                    await getController().applyPatches.promise(ticketId, commentIds);
+                                    resetTicketData();
+                                });
+                            }}
+                        >
+                            Apply All Patches ({patchComments.length})
+                        </Button>
+                    )}
+                </div>
                 {sortedComments.map(comment => (
                     <TicketCommentItem key={comment.id} ticketId={this.props.ticketId} comment={comment} />
                 ))}
@@ -444,6 +472,99 @@ class TicketDetail extends qreact.Component<{ ticketId: string }> {
     }
 }
 
+// Controls the AI investigation running on the server this browser is connected to (in debug mode, the local dev server). Progress arrives live through the normal ticket change notifications, since every AI step is recorded as a ticket comment.
+class AIRunPanel extends qreact.Component<{ ticketId: string }> {
+    pollInterval: ReturnType<typeof setInterval> | undefined = undefined;
+
+    componentDidMount() {
+        this.pollInterval = setInterval(() => {
+            getController().getAIStatus.refreshAll();
+        }, AI_STATUS_POLL_INTERVAL);
+    }
+
+    componentWillUnmount() {
+        if (this.pollInterval !== undefined) {
+            clearInterval(this.pollInterval);
+        }
+    }
+
+    render() {
+        let ticketId = this.props.ticketId;
+        let status = getController().getAIStatus();
+
+        let stop = () => {
+            Querysub.onCommitFinished(async () => {
+                await getController().stopAIInvestigation.promise(ticketId);
+                getController().getAIStatus.refreshAll();
+            });
+        };
+
+        let contents: qreact.VNode;
+        if (!status) {
+            contents = <span className={css.colorhsl(0, 0, 40)}>Checking AI status...</span>;
+        } else if (status.runningTicketId === ticketId) {
+            contents = <>
+                <span className={css.colorhsl(280, 70, 35).boldStyle}>
+                    🤖 Investigating this ticket for {formatTime(Date.now() - (status.runStartTime ?? Date.now()))}{status.runningModel && ` (${status.runningModel})` || ""}
+                </span>
+                {status.stopRequested && (
+                    <span className={css.colorhsl(0, 70, 40).boldStyle}>Stopping after the current step...</span>
+                ) || (
+                    <Button hue={0} onClick={stop}>
+                        Stop (finishes current step)
+                    </Button>
+                )}
+            </>;
+        } else if (status.queuedTicketIds.includes(ticketId)) {
+            contents = <>
+                <span className={css.colorhsl(280, 70, 35).boldStyle}>
+                    Queued{status.runningTicketId && ` behind "${status.runningTicketTitle}"` || ""}
+                </span>
+                <Button hue={0} onClick={stop}>
+                    Cancel
+                </Button>
+            </>;
+        } else {
+            contents = <>
+                <DropdownSelector
+                    title="Model"
+                    value={aiModelURL.value}
+                    onChange={value => {
+                        aiModelURL.value = value;
+                    }}
+                    options={[
+                        { value: "", label: `Default${status.defaultModel && ` (${status.defaultModel})` || ""}` },
+                        ...AUTOFIXER_MODEL_OPTIONS.map(m => ({ value: m, label: m })),
+                    ]}
+                />
+                <Button
+                    hue={280}
+                    onClick={() => {
+                        let model = aiModelURL.value || undefined;
+                        Querysub.onCommitFinished(async () => {
+                            await getController().startAIInvestigation.promise(ticketId, model);
+                            getController().getAIStatus.refreshAll();
+                            resetTicketData();
+                        });
+                    }}
+                >
+                    Run AI Investigation
+                </Button>
+                {status.runningTicketId && (
+                    <span className={css.colorhsl(0, 0, 40).fontSize(12)}>
+                        The AI is currently working on "{status.runningTicketTitle}" — this run will queue behind it.
+                    </span>
+                )}
+            </>;
+        }
+
+        return <div className={css.hbox(12).alignItems("center").wrap.pad2(12).bord2(280, 50, 60).hsl(280, 40, 96).fillWidth}>
+            <strong>AI Investigation</strong>
+            {contents}
+        </div>;
+    }
+}
+
 // Pulls the interesting parts out of a searchLogs tool-call comment so they can be shown inline. startTime/endTime may be epoch ms or strings — either way the Date constructor parses them.
 function parseSearchLogsInfo(comment: TicketComment): {
     rangeText: string;
@@ -624,35 +745,33 @@ class TicketCommentItem extends qreact.Component<{
                         <PatchFileView key={idx} patchFile={patchFile} />
                     ))}
                     <div className={css.hbox(12).alignItems("center")}>
+                        <Button
+                            hue={120}
+                            onClick={() => {
+                                let ticketId = this.props.ticketId;
+                                let commentId = comment.id;
+                                Querysub.onCommitFinished(async () => {
+                                    await getController().applyPatches.promise(ticketId, [commentId]);
+                                    resetTicketData();
+                                });
+                            }}
+                        >
+                            Apply Patch
+                        </Button>
                         {comment.patchStatus === "pending" && (
-                            <>
-                                <Button
-                                    hue={120}
-                                    onClick={() => {
-                                        let ticketId = this.props.ticketId;
-                                        let commentId = comment.id;
-                                        Querysub.onCommitFinished(async () => {
-                                            await getController().applyPatch.promise(ticketId, commentId);
-                                            resetTicketData();
-                                        });
-                                    }}
-                                >
-                                    Apply Patch
-                                </Button>
-                                <Button
-                                    hue={0}
-                                    onClick={() => {
-                                        let ticketId = this.props.ticketId;
-                                        let commentId = comment.id;
-                                        Querysub.onCommitFinished(async () => {
-                                            await getController().setPatchStatus.promise(ticketId, commentId, "rejected");
-                                            resetTicketData();
-                                        });
-                                    }}
-                                >
-                                    Reject Patch
-                                </Button>
-                            </>
+                            <Button
+                                hue={0}
+                                onClick={() => {
+                                    let ticketId = this.props.ticketId;
+                                    let commentId = comment.id;
+                                    Querysub.onCommitFinished(async () => {
+                                        await getController().setPatchStatuses.promise(ticketId, [commentId], "rejected");
+                                        resetTicketData();
+                                    });
+                                }}
+                            >
+                                Reject Patch
+                            </Button>
                         )}
                         {comment.patchStatus === "applied" && (
                             <span className={css.colorhsl(120, 80, 25).boldStyle}>