pi-provider-cursor-ask 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/LICENSE +21 -0
- package/README.md +87 -0
- package/README.zh-CN.md +87 -0
- package/UPSTREAM_CHANGELOG.md +368 -0
- package/UPSTREAM_SOURCE.md +23 -0
- package/dist/index.js +54 -0
- package/package.json +97 -0
- package/src/auth/cli-credentials.ts +275 -0
- package/src/auth/consent.ts +25 -0
- package/src/auth/index.ts +23 -0
- package/src/auth/oauth.ts +282 -0
- package/src/auth/refresh-guard.ts +93 -0
- package/src/client/bridge.ts +673 -0
- package/src/client/cursor-wire.ts +213 -0
- package/src/client/h2-unary.ts +142 -0
- package/src/client/index.ts +18 -0
- package/src/config/index.ts +69 -0
- package/src/diagnostics/diagnostics.ts +116 -0
- package/src/diagnostics/index.ts +1 -0
- package/src/extension/auth.ts +99 -0
- package/src/extension/commands.ts +163 -0
- package/src/extension/compaction-guard.ts +86 -0
- package/src/extension/debug-hooks.ts +359 -0
- package/src/extension/index.ts +8 -0
- package/src/extension/provider.ts +277 -0
- package/src/extension/quota-adapter.ts +175 -0
- package/src/extension/report-dashboard.ts +133 -0
- package/src/identity.ts +16 -0
- package/src/index.ts +186 -0
- package/src/models/ask-catalog.ts +384 -0
- package/src/models/catalog.json +1163 -0
- package/src/models/cost.ts +126 -0
- package/src/models/index.ts +6 -0
- package/src/models/limits.ts +36 -0
- package/src/models/parameterized.ts +416 -0
- package/src/models/processing.ts +313 -0
- package/src/proto/agent_pb.ts +14577 -0
- package/src/stream/bridge-session.ts +215 -0
- package/src/stream/client-transcript.ts +51 -0
- package/src/stream/config.ts +5 -0
- package/src/stream/context-normalize.ts +308 -0
- package/src/stream/context-usage.ts +168 -0
- package/src/stream/debug-log.ts +316 -0
- package/src/stream/drift.ts +122 -0
- package/src/stream/images.ts +201 -0
- package/src/stream/index.ts +68 -0
- package/src/stream/interaction-query.ts +369 -0
- package/src/stream/message-parsing.ts +402 -0
- package/src/stream/model-cache.ts +100 -0
- package/src/stream/model-discovery.ts +242 -0
- package/src/stream/model-routing.ts +100 -0
- package/src/stream/native-core.ts +2121 -0
- package/src/stream/pi-adapter.ts +414 -0
- package/src/stream/protocol.ts +63 -0
- package/src/stream/recovery.ts +494 -0
- package/src/stream/request-build.ts +668 -0
- package/src/stream/root-prompt.ts +184 -0
- package/src/stream/run-journal.ts +474 -0
- package/src/stream/run-usage.ts +107 -0
- package/src/stream/server-messages.ts +777 -0
- package/src/stream/session-state.ts +499 -0
- package/src/stream/stream-writer.ts +211 -0
- package/src/stream/thinking-filter.ts +63 -0
- package/src/stream/tool-schema.ts +185 -0
- package/src/stream/transport-errors.ts +150 -0
- package/src/stream/tuning.ts +250 -0
- package/src/stream/types.ts +330 -0
- package/src/types/enums.ts +103 -0
- package/src/types/index.ts +4 -0
- package/src/usage.ts +262 -0
- package/src/utils/cache-dir.ts +39 -0
- package/src/utils/index.ts +2 -0
- package/src/utils/security.ts +68 -0
- package/src/utils/util.ts +43 -0
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Builds the model-facing prompt messages Cursor actually reads.
|
|
3
|
+
*
|
|
4
|
+
* Cursor's agent server assembles the prompt it sends to the model from
|
|
5
|
+
* `ConversationStateStructure.root_prompt_messages_json` — a list of blob ids,
|
|
6
|
+
* each holding one JSON message in the AI-SDK "model message" shape. Everything
|
|
7
|
+
* else the client sends is conversation *state*: `turns` drive Cursor's own UI
|
|
8
|
+
* and checkpointing, and the server never renders them back into prompt
|
|
9
|
+
* messages. A request that carries history only as `turns` therefore reaches
|
|
10
|
+
* the model as a single fresh user question, which is what made a resumed or
|
|
11
|
+
* rebuilt conversation lose every earlier turn.
|
|
12
|
+
*
|
|
13
|
+
* Two shapes matter here and are verified against captured server checkpoints:
|
|
14
|
+
*
|
|
15
|
+
* - The server drops `{"role":"system",...}` entries and uses its own system
|
|
16
|
+
* prompt, so Pi's system prompt has to ride a *user* message. Cursor does
|
|
17
|
+
* the same thing with its own `<rules>` block.
|
|
18
|
+
* - Assistant tool calls / tool results replay as `tool-call` and
|
|
19
|
+
* `tool-result` content parts, with MCP tool names in Cursor's
|
|
20
|
+
* `mcp_<provider>_<tool>` form.
|
|
21
|
+
*
|
|
22
|
+
* Only used when a request is built without an upstream checkpoint. With a
|
|
23
|
+
* checkpoint the server already holds the rendered history and appends to it.
|
|
24
|
+
*/
|
|
25
|
+
import { createHash } from "node:crypto";
|
|
26
|
+
|
|
27
|
+
import type { ParsedTurn, ParsedTurnStep, ParsedToolCallStep } from "./types.js";
|
|
28
|
+
|
|
29
|
+
/** Provider identifier used when registering Pi's tools as Cursor MCP tools. */
|
|
30
|
+
const MCP_PROVIDER_IDENTIFIER = "pi";
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Replayed tool results are already bounded by `normalizeToolResultForTransport`
|
|
34
|
+
* on the turn-structure path. Bound them again here so a long history cannot
|
|
35
|
+
* blow the prompt on its own.
|
|
36
|
+
*/
|
|
37
|
+
export const MAX_REPLAYED_TOOL_RESULT_CHARS = 20_000;
|
|
38
|
+
|
|
39
|
+
export interface RootPromptTextPart {
|
|
40
|
+
type: "text";
|
|
41
|
+
text: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface RootPromptToolCallPart {
|
|
45
|
+
type: "tool-call";
|
|
46
|
+
toolCallId: string;
|
|
47
|
+
toolName: string;
|
|
48
|
+
args: Record<string, unknown>;
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export interface RootPromptToolResultPart {
|
|
52
|
+
type: "tool-result";
|
|
53
|
+
toolCallId: string;
|
|
54
|
+
toolName: string;
|
|
55
|
+
result: string;
|
|
56
|
+
isError?: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export type RootPromptMessage =
|
|
60
|
+
| { role: "user"; content: RootPromptTextPart[] }
|
|
61
|
+
| { role: "assistant"; content: Array<RootPromptTextPart | RootPromptToolCallPart> }
|
|
62
|
+
| { role: "tool"; content: RootPromptToolResultPart[] };
|
|
63
|
+
|
|
64
|
+
/** Cursor namespaces MCP tools as `mcp_<providerIdentifier>_<toolName>`. */
|
|
65
|
+
export function cursorMcpToolName(toolName: string): string {
|
|
66
|
+
const name = toolName.trim();
|
|
67
|
+
if (!name) return `mcp_${MCP_PROVIDER_IDENTIFIER}_tool`;
|
|
68
|
+
if (name.startsWith(`mcp_${MCP_PROVIDER_IDENTIFIER}_`)) return name;
|
|
69
|
+
return `mcp_${MCP_PROVIDER_IDENTIFIER}_${name}`;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function truncateReplayedResult(text: string): string {
|
|
73
|
+
if (text.length <= MAX_REPLAYED_TOOL_RESULT_CHARS) return text;
|
|
74
|
+
return `${text.slice(0, MAX_REPLAYED_TOOL_RESULT_CHARS)}\n\n[pi-cursor truncated this replayed tool result.]`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Pi's system prompt, framed the way Cursor frames its own instructions.
|
|
79
|
+
* A `system` role entry here is discarded by the server.
|
|
80
|
+
*/
|
|
81
|
+
export function systemPromptRootMessage(systemPrompt: string): RootPromptMessage {
|
|
82
|
+
return {
|
|
83
|
+
role: "user",
|
|
84
|
+
content: [{ type: "text", text: `<rules>\n${systemPrompt}\n</rules>` }],
|
|
85
|
+
};
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function isToolCallStep(step: ParsedTurnStep): step is ParsedToolCallStep {
|
|
89
|
+
return step.kind === "toolCall";
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/** Render one completed turn as the user / assistant / tool messages Cursor renders. */
|
|
93
|
+
export function turnRootMessages(turn: ParsedTurn): RootPromptMessage[] {
|
|
94
|
+
const messages: RootPromptMessage[] = [];
|
|
95
|
+
const userText = turn.userText.trim();
|
|
96
|
+
const imageNote = turn.userImages?.length
|
|
97
|
+
? `\n\n[${turn.userImages.length} image attachment(s) from this earlier turn are not replayed.]`
|
|
98
|
+
: "";
|
|
99
|
+
if (userText || imageNote) {
|
|
100
|
+
messages.push({
|
|
101
|
+
role: "user",
|
|
102
|
+
content: [{ type: "text", text: `<user_query>\n${userText}${imageNote}\n</user_query>` }],
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
const assistantContent: Array<RootPromptTextPart | RootPromptToolCallPart> = [];
|
|
107
|
+
const pendingResults: RootPromptToolResultPart[] = [];
|
|
108
|
+
const flushAssistant = (): void => {
|
|
109
|
+
if (assistantContent.length > 0) {
|
|
110
|
+
messages.push({ role: "assistant", content: [...assistantContent] });
|
|
111
|
+
assistantContent.length = 0;
|
|
112
|
+
}
|
|
113
|
+
if (pendingResults.length > 0) {
|
|
114
|
+
messages.push({ role: "tool", content: [...pendingResults] });
|
|
115
|
+
pendingResults.length = 0;
|
|
116
|
+
}
|
|
117
|
+
};
|
|
118
|
+
|
|
119
|
+
for (const step of turn.steps) {
|
|
120
|
+
// Reasoning is not replayed: Cursor re-derives it, and a provider-signed
|
|
121
|
+
// reasoning block from an earlier turn is not portable across requests.
|
|
122
|
+
if (step.kind === "thinking") continue;
|
|
123
|
+
if (step.kind === "assistantText") {
|
|
124
|
+
if (!step.text) continue;
|
|
125
|
+
// A new assistant text block after tool results starts a new message pair.
|
|
126
|
+
if (pendingResults.length > 0) flushAssistant();
|
|
127
|
+
assistantContent.push({ type: "text", text: step.text });
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (!isToolCallStep(step)) continue;
|
|
131
|
+
const toolName = cursorMcpToolName(step.toolName);
|
|
132
|
+
assistantContent.push({
|
|
133
|
+
type: "tool-call",
|
|
134
|
+
toolCallId: step.toolCallId,
|
|
135
|
+
toolName,
|
|
136
|
+
args: step.arguments,
|
|
137
|
+
});
|
|
138
|
+
if (step.result) {
|
|
139
|
+
const imageSuffix = step.result.images?.length
|
|
140
|
+
? `\n\n[${step.result.images.length} image(s) in this earlier tool result are not replayed.]`
|
|
141
|
+
: "";
|
|
142
|
+
pendingResults.push({
|
|
143
|
+
type: "tool-result",
|
|
144
|
+
toolCallId: step.toolCallId,
|
|
145
|
+
toolName,
|
|
146
|
+
result: truncateReplayedResult(`${step.result.content}${imageSuffix}`),
|
|
147
|
+
...(step.result.isError ? { isError: true } : {}),
|
|
148
|
+
});
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
flushAssistant();
|
|
152
|
+
|
|
153
|
+
return messages;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Full prompt history for a request built without an upstream checkpoint:
|
|
158
|
+
* Pi's system prompt followed by every completed turn.
|
|
159
|
+
*/
|
|
160
|
+
export function buildRootPromptMessages(
|
|
161
|
+
systemPrompt: string,
|
|
162
|
+
turns: ParsedTurn[],
|
|
163
|
+
): RootPromptMessage[] {
|
|
164
|
+
const messages: RootPromptMessage[] = [];
|
|
165
|
+
if (systemPrompt.trim()) messages.push(systemPromptRootMessage(systemPrompt));
|
|
166
|
+
for (const turn of turns) messages.push(...turnRootMessages(turn));
|
|
167
|
+
return messages;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
export function encodeRootPromptMessage(message: RootPromptMessage): Uint8Array {
|
|
171
|
+
return new TextEncoder().encode(JSON.stringify(message));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/** Env escape hatch: `PI_CURSOR_PROMPT_HISTORY=0` restores the pre-fix behavior. */
|
|
175
|
+
export function isPromptHistoryEnabled(envValue = process.env.PI_CURSOR_PROMPT_HISTORY): boolean {
|
|
176
|
+
const raw = envValue?.trim().toLowerCase();
|
|
177
|
+
if (!raw) return true;
|
|
178
|
+
return raw !== "0" && raw !== "false" && raw !== "off" && raw !== "no";
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/** Identity of the system prompt currently published to a Cursor conversation. */
|
|
182
|
+
export function hashSystemPrompt(systemPrompt: string): string {
|
|
183
|
+
return createHash("sha256").update(systemPrompt).digest("hex").slice(0, 32);
|
|
184
|
+
}
|
|
@@ -0,0 +1,474 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Disk-backed journal for conversation recovery state.
|
|
3
|
+
*
|
|
4
|
+
* In-memory maps alone die with the process and evaporate across long sessions
|
|
5
|
+
* when the bridge is lost. The journal stores the minimum needed to resume a
|
|
6
|
+
* parked tool turn or continue from the latest checkpoint after transport loss.
|
|
7
|
+
*
|
|
8
|
+
* Storage is best-effort: if the cache dir is unusable we stay memory-only.
|
|
9
|
+
*/
|
|
10
|
+
import {
|
|
11
|
+
closeSync,
|
|
12
|
+
fstatSync,
|
|
13
|
+
mkdirSync,
|
|
14
|
+
openSync,
|
|
15
|
+
readFileSync,
|
|
16
|
+
readSync,
|
|
17
|
+
renameSync,
|
|
18
|
+
unlinkSync,
|
|
19
|
+
writeFileSync,
|
|
20
|
+
readdirSync,
|
|
21
|
+
} from "node:fs";
|
|
22
|
+
import { join as pathJoin } from "node:path";
|
|
23
|
+
|
|
24
|
+
import { getCacheDir } from "../utils/cache-dir.js";
|
|
25
|
+
import { debugLog } from "./debug-log.js";
|
|
26
|
+
import { MAX_ACTIVE_BLOB_ENTRIES, MAX_INDIVIDUAL_BLOB_BYTES } from "./tuning.js";
|
|
27
|
+
import type { StoredConversation } from "./types.js";
|
|
28
|
+
|
|
29
|
+
const JOURNAL_VERSION = 1 as const;
|
|
30
|
+
const JOURNAL_SUBDIR = "run-journal";
|
|
31
|
+
const JOURNAL_TTL_MS = 24 * 60 * 60 * 1000;
|
|
32
|
+
const MAX_JOURNAL_FILE_BYTES = 192 * 1024 * 1024;
|
|
33
|
+
const MAX_CHECKPOINT_BYTES = 64 * 1024 * 1024;
|
|
34
|
+
const MAX_ENCODED_BLOB_CHARS = Math.ceil((MAX_INDIVIDUAL_BLOB_BYTES * 4) / 3) + 4;
|
|
35
|
+
/** Headroom inside MAX_JOURNAL_FILE_BYTES for the record's non-blob fields. */
|
|
36
|
+
const JOURNAL_METADATA_SLACK_CHARS = 64 * 1024;
|
|
37
|
+
|
|
38
|
+
interface JournalBlobEntry {
|
|
39
|
+
id: string;
|
|
40
|
+
/** base64 */
|
|
41
|
+
data: string;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface DurableJournalRecord {
|
|
45
|
+
version: typeof JOURNAL_VERSION;
|
|
46
|
+
convKey: string;
|
|
47
|
+
savedAt: number;
|
|
48
|
+
conversationId: string;
|
|
49
|
+
sessionScoped: boolean;
|
|
50
|
+
sessionId?: string;
|
|
51
|
+
/** base64 checkpoint bytes */
|
|
52
|
+
checkpoint: string | null;
|
|
53
|
+
checkpointSource?: "upstream" | "absent";
|
|
54
|
+
checkpointTurnCount?: number;
|
|
55
|
+
checkpointHistoryFingerprint?: string;
|
|
56
|
+
midPausePendingToolCalls?: Array<{ toolCallId: string; toolName: string }>;
|
|
57
|
+
midPauseTurnCount?: number;
|
|
58
|
+
midPauseHistoryFingerprint?: string;
|
|
59
|
+
midPauseRecordedAtMs?: number;
|
|
60
|
+
/** Hash of the system prompt last published to Cursor for this conversation. */
|
|
61
|
+
systemPromptHash?: string;
|
|
62
|
+
/** Content-addressed blobs referenced by checkpoints / history. */
|
|
63
|
+
blobs: JournalBlobEntry[];
|
|
64
|
+
/**
|
|
65
|
+
* Blobs the byte budget could not fit. Non-zero means the stored checkpoint
|
|
66
|
+
* references content this record cannot supply, so the reader drops the
|
|
67
|
+
* checkpoint rather than resuming a conversation with blank history.
|
|
68
|
+
*/
|
|
69
|
+
blobsOmitted?: number;
|
|
70
|
+
lastAccessMs: number;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function journalDir(): string | undefined {
|
|
74
|
+
const base = getCacheDir();
|
|
75
|
+
if (!base) return undefined;
|
|
76
|
+
const dir = pathJoin(base, JOURNAL_SUBDIR);
|
|
77
|
+
try {
|
|
78
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
79
|
+
return dir;
|
|
80
|
+
} catch {
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function journalPath(convKey: string): string | undefined {
|
|
86
|
+
const dir = journalDir();
|
|
87
|
+
if (!dir) return undefined;
|
|
88
|
+
// convKey is already a short hex hash; keep the filename boring.
|
|
89
|
+
const safe = convKey.replace(/[^a-zA-Z0-9._-]/g, "_").slice(0, 64);
|
|
90
|
+
return pathJoin(dir, `${safe}.json`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Enough to cover `{"version":1,"convKey":"<=64 chars>","savedAt":<ms>` with room to spare.
|
|
95
|
+
* `convKey` is sanitized to `[a-zA-Z0-9._-]`, so it can never contain a quoted key itself —
|
|
96
|
+
* the first `"savedAt":` in the prefix is always the real field.
|
|
97
|
+
*/
|
|
98
|
+
const SAVED_AT_PROBE_BYTES = 512;
|
|
99
|
+
const SAVED_AT_PATTERN = /"savedAt":\s*(\d+)/;
|
|
100
|
+
|
|
101
|
+
interface JournalStat {
|
|
102
|
+
size: number;
|
|
103
|
+
/** `savedAt` when it was readable from the file's head, else undefined. */
|
|
104
|
+
savedAt?: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Stat a journal and recover its `savedAt` without reading the whole file.
|
|
109
|
+
*
|
|
110
|
+
* Blobs are base64 and sit at the tail of the record, so a journal is routinely megabytes
|
|
111
|
+
* while the timestamp the TTL sweep needs lives in the first hundred bytes. Reading a fixed
|
|
112
|
+
* prefix keeps `evictStaleJournals` proportional to the number of journals rather than to
|
|
113
|
+
* their total size.
|
|
114
|
+
*/
|
|
115
|
+
function statJournalHead(path: string): JournalStat | undefined {
|
|
116
|
+
let fd: number | undefined;
|
|
117
|
+
try {
|
|
118
|
+
fd = openSync(path, "r");
|
|
119
|
+
const size = fstatSync(fd).size;
|
|
120
|
+
const head = Buffer.allocUnsafe(Math.min(SAVED_AT_PROBE_BYTES, size));
|
|
121
|
+
const read = head.length > 0 ? readSync(fd, head, 0, head.length, 0) : 0;
|
|
122
|
+
const savedAt = Number(SAVED_AT_PATTERN.exec(head.toString("utf8", 0, read))?.[1]);
|
|
123
|
+
return { size, ...(Number.isFinite(savedAt) && savedAt > 0 ? { savedAt } : {}) };
|
|
124
|
+
} catch {
|
|
125
|
+
return undefined;
|
|
126
|
+
} finally {
|
|
127
|
+
if (fd !== undefined) {
|
|
128
|
+
try {
|
|
129
|
+
closeSync(fd);
|
|
130
|
+
} catch {
|
|
131
|
+
// Already closed / never opened.
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
// A journal is rewritten on every checkpoint commit and every tool-call pause, but a
|
|
138
|
+
// conversation's blobs are overwhelmingly carried over unchanged from the previous write.
|
|
139
|
+
// Blob bytes are immutable once stored, so cache each buffer's encoding by identity rather
|
|
140
|
+
// than re-encoding megabytes of base64 per pause.
|
|
141
|
+
const blobBase64Cache = new WeakMap<Uint8Array, string>();
|
|
142
|
+
|
|
143
|
+
function encodeBlob(data: Uint8Array): string {
|
|
144
|
+
let encoded = blobBase64Cache.get(data);
|
|
145
|
+
if (encoded === undefined) {
|
|
146
|
+
encoded = Buffer.from(data).toString("base64");
|
|
147
|
+
blobBase64Cache.set(data, encoded);
|
|
148
|
+
}
|
|
149
|
+
return encoded;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/**
|
|
153
|
+
* Encode the whole blob store, newest-first eviction only when the byte budget
|
|
154
|
+
* genuinely cannot hold it.
|
|
155
|
+
*
|
|
156
|
+
* A checkpoint's `turns` are blob *references*, so any blob left out here is a
|
|
157
|
+
* turn that comes back blank on the next launch. The live store is already
|
|
158
|
+
* bounded (MAX_ACTIVE_BLOB_ENTRIES / MAX_CONVERSATION_BLOB_BYTES), so the
|
|
159
|
+
* common case fits and `omitted` stays zero.
|
|
160
|
+
*/
|
|
161
|
+
function encodeBlobs(
|
|
162
|
+
blobStore: Map<string, Uint8Array>,
|
|
163
|
+
budgetChars: number,
|
|
164
|
+
): { entries: JournalBlobEntry[]; omitted: number } {
|
|
165
|
+
// Map iteration is insertion order; keep the newest tail when something must go.
|
|
166
|
+
const all = [...blobStore.entries()];
|
|
167
|
+
const kept: JournalBlobEntry[] = [];
|
|
168
|
+
let usedChars = 0;
|
|
169
|
+
let omitted = 0;
|
|
170
|
+
for (let i = all.length - 1; i >= 0; i--) {
|
|
171
|
+
const [id, data] = all[i]!;
|
|
172
|
+
const encoded = encodeBlob(data);
|
|
173
|
+
// `id.length + encoded.length` under-counts the JSON punctuation around each
|
|
174
|
+
// entry; the slack constant covers that and the rest of the record.
|
|
175
|
+
if (kept.length >= MAX_ACTIVE_BLOB_ENTRIES || usedChars + encoded.length > budgetChars) {
|
|
176
|
+
omitted = i + 1;
|
|
177
|
+
break;
|
|
178
|
+
}
|
|
179
|
+
usedChars += encoded.length + id.length;
|
|
180
|
+
kept.push({ id, data: encoded });
|
|
181
|
+
}
|
|
182
|
+
kept.reverse();
|
|
183
|
+
return { entries: kept, omitted };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function decodeBlobs(entries: JournalBlobEntry[] | undefined): {
|
|
187
|
+
map: Map<string, Uint8Array>;
|
|
188
|
+
skipped: number;
|
|
189
|
+
} {
|
|
190
|
+
const map = new Map<string, Uint8Array>();
|
|
191
|
+
if (!Array.isArray(entries)) return { map, skipped: 0 };
|
|
192
|
+
let skipped = 0;
|
|
193
|
+
for (const entry of entries.slice(0, MAX_ACTIVE_BLOB_ENTRIES)) {
|
|
194
|
+
try {
|
|
195
|
+
if (
|
|
196
|
+
!entry ||
|
|
197
|
+
typeof entry.id !== "string" ||
|
|
198
|
+
!entry.id ||
|
|
199
|
+
typeof entry.data !== "string" ||
|
|
200
|
+
entry.data.length > MAX_ENCODED_BLOB_CHARS
|
|
201
|
+
) {
|
|
202
|
+
skipped += 1;
|
|
203
|
+
continue;
|
|
204
|
+
}
|
|
205
|
+
const data = Buffer.from(entry.data, "base64");
|
|
206
|
+
if (data.length > MAX_INDIVIDUAL_BLOB_BYTES) {
|
|
207
|
+
skipped += 1;
|
|
208
|
+
continue;
|
|
209
|
+
}
|
|
210
|
+
map.set(entry.id, new Uint8Array(data));
|
|
211
|
+
} catch {
|
|
212
|
+
// A corrupt entry is still a blob the checkpoint may reference.
|
|
213
|
+
skipped += 1;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
return { map, skipped: skipped + Math.max(0, entries.length - MAX_ACTIVE_BLOB_ENTRIES) };
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export function serializeConversationJournal(
|
|
220
|
+
convKey: string,
|
|
221
|
+
stored: StoredConversation,
|
|
222
|
+
now = Date.now(),
|
|
223
|
+
): DurableJournalRecord {
|
|
224
|
+
const checkpoint = stored.checkpoint ? Buffer.from(stored.checkpoint).toString("base64") : null;
|
|
225
|
+
const budgetChars = Math.max(
|
|
226
|
+
0,
|
|
227
|
+
MAX_JOURNAL_FILE_BYTES - (checkpoint?.length ?? 0) - JOURNAL_METADATA_SLACK_CHARS,
|
|
228
|
+
);
|
|
229
|
+
const { entries, omitted } = encodeBlobs(stored.blobStore, budgetChars);
|
|
230
|
+
if (omitted > 0) {
|
|
231
|
+
debugLog("journal.blobs_omitted", { convKey, omitted, kept: entries.length });
|
|
232
|
+
}
|
|
233
|
+
return {
|
|
234
|
+
version: JOURNAL_VERSION,
|
|
235
|
+
convKey,
|
|
236
|
+
savedAt: now,
|
|
237
|
+
conversationId: stored.conversationId,
|
|
238
|
+
sessionScoped: stored.sessionScoped,
|
|
239
|
+
...(stored.sessionId ? { sessionId: stored.sessionId } : {}),
|
|
240
|
+
checkpoint,
|
|
241
|
+
...(stored.checkpointSource ? { checkpointSource: stored.checkpointSource } : {}),
|
|
242
|
+
...(stored.checkpointTurnCount !== undefined
|
|
243
|
+
? { checkpointTurnCount: stored.checkpointTurnCount }
|
|
244
|
+
: {}),
|
|
245
|
+
...(stored.checkpointHistoryFingerprint
|
|
246
|
+
? { checkpointHistoryFingerprint: stored.checkpointHistoryFingerprint }
|
|
247
|
+
: {}),
|
|
248
|
+
...(stored.systemPromptHash ? { systemPromptHash: stored.systemPromptHash } : {}),
|
|
249
|
+
...(stored.midPausePendingToolCalls
|
|
250
|
+
? { midPausePendingToolCalls: stored.midPausePendingToolCalls }
|
|
251
|
+
: {}),
|
|
252
|
+
...(stored.midPauseTurnCount !== undefined
|
|
253
|
+
? { midPauseTurnCount: stored.midPauseTurnCount }
|
|
254
|
+
: {}),
|
|
255
|
+
...(stored.midPauseHistoryFingerprint
|
|
256
|
+
? { midPauseHistoryFingerprint: stored.midPauseHistoryFingerprint }
|
|
257
|
+
: {}),
|
|
258
|
+
...(stored.midPauseRecordedAtMs !== undefined
|
|
259
|
+
? { midPauseRecordedAtMs: stored.midPauseRecordedAtMs }
|
|
260
|
+
: {}),
|
|
261
|
+
blobs: entries,
|
|
262
|
+
...(omitted > 0 ? { blobsOmitted: omitted } : {}),
|
|
263
|
+
lastAccessMs: stored.lastAccessMs || now,
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export function deserializeConversationJournal(
|
|
268
|
+
record: DurableJournalRecord,
|
|
269
|
+
): StoredConversation | undefined {
|
|
270
|
+
if (!record || typeof record !== "object") return undefined;
|
|
271
|
+
if (record.version !== JOURNAL_VERSION) return undefined;
|
|
272
|
+
if (typeof record.conversationId !== "string" || typeof record.convKey !== "string") {
|
|
273
|
+
return undefined;
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
let checkpoint: Uint8Array | null = null;
|
|
277
|
+
if (record.checkpoint) {
|
|
278
|
+
try {
|
|
279
|
+
if (typeof record.checkpoint !== "string") return undefined;
|
|
280
|
+
const decoded = Buffer.from(record.checkpoint, "base64");
|
|
281
|
+
if (decoded.length > MAX_CHECKPOINT_BYTES) return undefined;
|
|
282
|
+
checkpoint = new Uint8Array(decoded);
|
|
283
|
+
} catch {
|
|
284
|
+
checkpoint = null;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
const { map: blobStore, skipped } = decodeBlobs(record.blobs);
|
|
289
|
+
|
|
290
|
+
// A checkpoint addresses its history by blob id. Replaying one whose blobs are
|
|
291
|
+
// missing does not fail — Cursor asks for them, gets an empty result, and the
|
|
292
|
+
// conversation comes back structurally intact with its older turns blank.
|
|
293
|
+
// Dropping the checkpoint instead costs one full-history rebuild from pi's
|
|
294
|
+
// transcript, which is always correct.
|
|
295
|
+
const blobsIncomplete = (record.blobsOmitted ?? 0) > 0 || skipped > 0;
|
|
296
|
+
if (checkpoint && blobsIncomplete) {
|
|
297
|
+
debugLog("journal.checkpoint_dropped_incomplete_blobs", {
|
|
298
|
+
convKey: record.convKey,
|
|
299
|
+
omitted: record.blobsOmitted ?? 0,
|
|
300
|
+
skipped,
|
|
301
|
+
kept: blobStore.size,
|
|
302
|
+
});
|
|
303
|
+
checkpoint = null;
|
|
304
|
+
}
|
|
305
|
+
const keepCheckpointMetadata = !!checkpoint;
|
|
306
|
+
|
|
307
|
+
return {
|
|
308
|
+
conversationId: record.conversationId,
|
|
309
|
+
checkpoint,
|
|
310
|
+
...(keepCheckpointMetadata && record.checkpointSource
|
|
311
|
+
? { checkpointSource: record.checkpointSource }
|
|
312
|
+
: {}),
|
|
313
|
+
...(keepCheckpointMetadata && record.checkpointTurnCount !== undefined
|
|
314
|
+
? { checkpointTurnCount: record.checkpointTurnCount }
|
|
315
|
+
: {}),
|
|
316
|
+
...(keepCheckpointMetadata && record.checkpointHistoryFingerprint
|
|
317
|
+
? { checkpointHistoryFingerprint: record.checkpointHistoryFingerprint }
|
|
318
|
+
: {}),
|
|
319
|
+
...(keepCheckpointMetadata && record.systemPromptHash
|
|
320
|
+
? { systemPromptHash: record.systemPromptHash }
|
|
321
|
+
: {}),
|
|
322
|
+
...(record.midPausePendingToolCalls
|
|
323
|
+
? { midPausePendingToolCalls: record.midPausePendingToolCalls }
|
|
324
|
+
: {}),
|
|
325
|
+
...(record.midPauseTurnCount !== undefined
|
|
326
|
+
? { midPauseTurnCount: record.midPauseTurnCount }
|
|
327
|
+
: {}),
|
|
328
|
+
...(record.midPauseHistoryFingerprint
|
|
329
|
+
? { midPauseHistoryFingerprint: record.midPauseHistoryFingerprint }
|
|
330
|
+
: {}),
|
|
331
|
+
...(record.midPauseRecordedAtMs !== undefined
|
|
332
|
+
? { midPauseRecordedAtMs: record.midPauseRecordedAtMs }
|
|
333
|
+
: {}),
|
|
334
|
+
sessionScoped: !!record.sessionScoped,
|
|
335
|
+
...(record.sessionId ? { sessionId: record.sessionId } : {}),
|
|
336
|
+
blobStore,
|
|
337
|
+
lastAccessMs: record.lastAccessMs || record.savedAt || Date.now(),
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
|
|
341
|
+
/** Persist a conversation snapshot. Best-effort; never throws to callers. */
|
|
342
|
+
export function writeConversationJournal(convKey: string, stored: StoredConversation): boolean {
|
|
343
|
+
const path = journalPath(convKey);
|
|
344
|
+
if (!path) return false;
|
|
345
|
+
try {
|
|
346
|
+
const record = serializeConversationJournal(convKey, stored);
|
|
347
|
+
const tmp = `${path}.${process.pid}.${Date.now()}.tmp`;
|
|
348
|
+
writeFileSync(tmp, JSON.stringify(record), { encoding: "utf8", mode: 0o600 });
|
|
349
|
+
renameSync(tmp, path);
|
|
350
|
+
debugLog("journal.write", {
|
|
351
|
+
convKey,
|
|
352
|
+
hasCheckpoint: !!stored.checkpoint,
|
|
353
|
+
midPause: !!stored.midPausePendingToolCalls?.length,
|
|
354
|
+
blobCount: stored.blobStore.size,
|
|
355
|
+
});
|
|
356
|
+
return true;
|
|
357
|
+
} catch (error) {
|
|
358
|
+
debugLog("journal.write_failed", {
|
|
359
|
+
convKey,
|
|
360
|
+
message: error instanceof Error ? error.message : String(error),
|
|
361
|
+
});
|
|
362
|
+
return false;
|
|
363
|
+
}
|
|
364
|
+
}
|
|
365
|
+
|
|
366
|
+
/** Load a conversation snapshot from disk, or undefined when missing/stale/corrupt. */
|
|
367
|
+
export function readConversationJournal(
|
|
368
|
+
convKey: string,
|
|
369
|
+
options?: { maxAgeMs?: number; now?: number },
|
|
370
|
+
): StoredConversation | undefined {
|
|
371
|
+
const path = journalPath(convKey);
|
|
372
|
+
if (!path) return undefined;
|
|
373
|
+
try {
|
|
374
|
+
const head = statJournalHead(path);
|
|
375
|
+
if (!head) return undefined;
|
|
376
|
+
if (head.size > MAX_JOURNAL_FILE_BYTES) {
|
|
377
|
+
debugLog("journal.oversized", { convKey });
|
|
378
|
+
return undefined;
|
|
379
|
+
}
|
|
380
|
+
const now = options?.now ?? Date.now();
|
|
381
|
+
const maxAge = options?.maxAgeMs ?? JOURNAL_TTL_MS;
|
|
382
|
+
// Reject a stale journal from its head rather than decoding megabytes of blobs first.
|
|
383
|
+
if (head.savedAt !== undefined) {
|
|
384
|
+
const headAge = now - head.savedAt;
|
|
385
|
+
if (headAge < 0 || headAge > maxAge) {
|
|
386
|
+
debugLog("journal.stale", { convKey, age: headAge });
|
|
387
|
+
return undefined;
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
const raw = readFileSync(path, "utf8");
|
|
391
|
+
const record = JSON.parse(raw) as DurableJournalRecord;
|
|
392
|
+
const age = now - (record.savedAt || record.lastAccessMs || 0);
|
|
393
|
+
if (!Number.isFinite(age) || age < 0 || age > maxAge) {
|
|
394
|
+
debugLog("journal.stale", { convKey, age });
|
|
395
|
+
return undefined;
|
|
396
|
+
}
|
|
397
|
+
const stored = deserializeConversationJournal(record);
|
|
398
|
+
if (!stored) return undefined;
|
|
399
|
+
debugLog("journal.read", {
|
|
400
|
+
convKey,
|
|
401
|
+
hasCheckpoint: !!stored.checkpoint,
|
|
402
|
+
midPause: !!stored.midPausePendingToolCalls?.length,
|
|
403
|
+
});
|
|
404
|
+
return stored;
|
|
405
|
+
} catch {
|
|
406
|
+
return undefined;
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
export function deleteConversationJournal(convKey: string): void {
|
|
411
|
+
const path = journalPath(convKey);
|
|
412
|
+
if (!path) return;
|
|
413
|
+
try {
|
|
414
|
+
unlinkSync(path);
|
|
415
|
+
} catch {
|
|
416
|
+
// absent is fine
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
/** Drop journals older than TTL. Called opportunistically on write/load paths. */
|
|
421
|
+
export function evictStaleJournals(now = Date.now(), maxAgeMs = JOURNAL_TTL_MS): number {
|
|
422
|
+
const dir = journalDir();
|
|
423
|
+
if (!dir) return 0;
|
|
424
|
+
let removed = 0;
|
|
425
|
+
try {
|
|
426
|
+
for (const name of readdirSync(dir)) {
|
|
427
|
+
if (!name.endsWith(".json")) continue;
|
|
428
|
+
const full = pathJoin(dir, name);
|
|
429
|
+
try {
|
|
430
|
+
const head = statJournalHead(full);
|
|
431
|
+
if (!head) throw new Error("unreadable journal");
|
|
432
|
+
if (head.size > MAX_JOURNAL_FILE_BYTES) {
|
|
433
|
+
unlinkSync(full);
|
|
434
|
+
removed += 1;
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
// The head carries `savedAt` for every journal this build writes, so the common
|
|
438
|
+
// sweep never reads a blob payload. Anything older, hand-edited, or truncated
|
|
439
|
+
// falls through to the full parse below.
|
|
440
|
+
let age: number;
|
|
441
|
+
if (head.savedAt !== undefined) {
|
|
442
|
+
age = now - head.savedAt;
|
|
443
|
+
} else {
|
|
444
|
+
const record = JSON.parse(readFileSync(full, "utf8")) as DurableJournalRecord;
|
|
445
|
+
age = now - (record.savedAt || record.lastAccessMs || 0);
|
|
446
|
+
}
|
|
447
|
+
if (!Number.isFinite(age) || age > maxAgeMs) {
|
|
448
|
+
unlinkSync(full);
|
|
449
|
+
removed += 1;
|
|
450
|
+
}
|
|
451
|
+
} catch {
|
|
452
|
+
try {
|
|
453
|
+
unlinkSync(full);
|
|
454
|
+
removed += 1;
|
|
455
|
+
} catch {
|
|
456
|
+
// ignore
|
|
457
|
+
}
|
|
458
|
+
}
|
|
459
|
+
}
|
|
460
|
+
} catch {
|
|
461
|
+
return removed;
|
|
462
|
+
}
|
|
463
|
+
if (removed > 0) debugLog("journal.evict", { removed });
|
|
464
|
+
return removed;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
export const __testInternals = {
|
|
468
|
+
JOURNAL_VERSION,
|
|
469
|
+
JOURNAL_TTL_MS,
|
|
470
|
+
MAX_JOURNAL_FILE_BYTES,
|
|
471
|
+
serializeConversationJournal,
|
|
472
|
+
deserializeConversationJournal,
|
|
473
|
+
journalPath,
|
|
474
|
+
};
|