rovecode 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +662 -0
- package/README.md +737 -0
- package/THIRD_PARTY_NOTICES.md +268 -0
- package/bin/rovecode.ts +21 -0
- package/package.json +56 -0
- package/src/acp/server.ts +374 -0
- package/src/cli/auth-login.ts +122 -0
- package/src/cli/connect.ts +244 -0
- package/src/cli/context-cmd.ts +199 -0
- package/src/cli/dispatch.ts +82 -0
- package/src/cli/doctor.ts +362 -0
- package/src/cli/export.ts +276 -0
- package/src/cli/help.ts +293 -0
- package/src/cli/is-tui-invocation.ts +8 -0
- package/src/cli/main.ts +583 -0
- package/src/cli/market-cmd.ts +658 -0
- package/src/cli/mcp-login.ts +141 -0
- package/src/cli/mcp-market-cmd.ts +302 -0
- package/src/cli/output.ts +382 -0
- package/src/cli/repl.ts +250 -0
- package/src/cli/repomap-root.ts +14 -0
- package/src/cli/resume.ts +57 -0
- package/src/cli/run-flags.ts +43 -0
- package/src/cli/run-limits.ts +78 -0
- package/src/cli/runtime.ts +931 -0
- package/src/cli/session-arg.ts +30 -0
- package/src/cli/sessions-cmd.ts +145 -0
- package/src/cli/setup.ts +153 -0
- package/src/cli/skills-cmd.ts +194 -0
- package/src/cli/start-chat.ts +65 -0
- package/src/cli/trust-cmd.ts +52 -0
- package/src/coding/bash.ts +148 -0
- package/src/coding/checkpoints.ts +327 -0
- package/src/coding/diff.ts +138 -0
- package/src/coding/files.ts +341 -0
- package/src/coding/hashline.ts +274 -0
- package/src/coding/lsp-gate.ts +254 -0
- package/src/coding/lsp-servers.ts +147 -0
- package/src/coding/lsp.ts +283 -0
- package/src/coding/repomap-cache.ts +99 -0
- package/src/coding/repomap-files.ts +192 -0
- package/src/coding/repomap.ts +481 -0
- package/src/core/agents.ts +255 -0
- package/src/core/compaction.ts +259 -0
- package/src/core/config.ts +289 -0
- package/src/core/context-report.ts +228 -0
- package/src/core/context.ts +60 -0
- package/src/core/count-remote.ts +107 -0
- package/src/core/execpolicy-rules.ts +196 -0
- package/src/core/execpolicy.ts +385 -0
- package/src/core/executor.ts +454 -0
- package/src/core/guardrails.ts +400 -0
- package/src/core/hooks.ts +411 -0
- package/src/core/images.ts +230 -0
- package/src/core/intro.ts +266 -0
- package/src/core/loop.ts +567 -0
- package/src/core/modes.ts +372 -0
- package/src/core/orchestrator.ts +245 -0
- package/src/core/proc-group.ts +48 -0
- package/src/core/project-trust.ts +98 -0
- package/src/core/reflection.ts +165 -0
- package/src/core/sandbox-config.ts +186 -0
- package/src/core/session-id.ts +24 -0
- package/src/core/session-images.ts +73 -0
- package/src/core/session-ops.ts +183 -0
- package/src/core/session-text.ts +29 -0
- package/src/core/session.ts +469 -0
- package/src/core/settings.ts +170 -0
- package/src/core/tasks.ts +646 -0
- package/src/core/token-scale.ts +108 -0
- package/src/core/tools.ts +309 -0
- package/src/core/trust.ts +104 -0
- package/src/core/types.ts +330 -0
- package/src/core/update-check.ts +171 -0
- package/src/core/usage.ts +204 -0
- package/src/core/validate.ts +121 -0
- package/src/core/verify-gate.ts +159 -0
- package/src/core/verify.ts +236 -0
- package/src/core/voice.ts +158 -0
- package/src/core/win-job.ts +183 -0
- package/src/core/workspace.ts +184 -0
- package/src/design/audit.ts +797 -0
- package/src/design/direction.ts +190 -0
- package/src/design/rules.ts +157 -0
- package/src/eval/bench.ts +150 -0
- package/src/eval/gauntlet-runner.ts +215 -0
- package/src/eval/gauntlet-support.ts +84 -0
- package/src/eval/gauntlet-wave3.ts +269 -0
- package/src/eval/gauntlet-wave4.ts +217 -0
- package/src/eval/gauntlet.ts +253 -0
- package/src/index.ts +17 -0
- package/src/lanes/agy.ts +95 -0
- package/src/lanes/approval.ts +24 -0
- package/src/lanes/claude.ts +129 -0
- package/src/lanes/codex.ts +127 -0
- package/src/lanes/events.ts +130 -0
- package/src/lanes/job.ts +142 -0
- package/src/lanes/opencode.ts +122 -0
- package/src/lanes/process.ts +184 -0
- package/src/lanes/progress.ts +183 -0
- package/src/lanes/registry.ts +178 -0
- package/src/lanes/runner.ts +124 -0
- package/src/lanes/types.ts +112 -0
- package/src/market/catalogs/mcp-docs.json +111 -0
- package/src/market/catalogs/plugins.json +111 -0
- package/src/market/catalogs/skills.json +478 -0
- package/src/market/clone.ts +72 -0
- package/src/market/context-cost.ts +121 -0
- package/src/market/digest.ts +106 -0
- package/src/market/index.ts +22 -0
- package/src/market/install.ts +578 -0
- package/src/market/manifest.ts +187 -0
- package/src/market/prereq.ts +145 -0
- package/src/market/registry.ts +363 -0
- package/src/market/resolve.ts +111 -0
- package/src/market/types.ts +236 -0
- package/src/market/validate.ts +227 -0
- package/src/mcp/client.ts +449 -0
- package/src/mcp/config.ts +252 -0
- package/src/mcp/local-package.ts +211 -0
- package/src/mcp/market-catalog.ts +84 -0
- package/src/mcp/market-install.ts +289 -0
- package/src/mcp/market.ts +362 -0
- package/src/mcp/oauth.ts +251 -0
- package/src/mcp/prompts-resources.ts +249 -0
- package/src/mcp/shared.ts +149 -0
- package/src/mcp/status.ts +67 -0
- package/src/mcp/tools.ts +275 -0
- package/src/mcp/transport.ts +122 -0
- package/src/mcp/trust.ts +25 -0
- package/src/memory/blocks.ts +278 -0
- package/src/memory/recall.ts +355 -0
- package/src/memory/scope.ts +182 -0
- package/src/memory/store.ts +105 -0
- package/src/memory/tools.ts +99 -0
- package/src/plugins/cli.ts +119 -0
- package/src/plugins/discover.ts +108 -0
- package/src/plugins/index.ts +50 -0
- package/src/plugins/install.ts +184 -0
- package/src/plugins/load.ts +124 -0
- package/src/plugins/manifest.ts +92 -0
- package/src/plugins/state.ts +83 -0
- package/src/providers/auth.ts +408 -0
- package/src/providers/cache.ts +223 -0
- package/src/providers/catalog-local.ts +160 -0
- package/src/providers/catalog.ts +421 -0
- package/src/providers/middleware-context.ts +86 -0
- package/src/providers/middleware.ts +373 -0
- package/src/providers/model-list.ts +23 -0
- package/src/providers/models-index.json +1 -0
- package/src/providers/oauth/common.ts +105 -0
- package/src/providers/oauth/device-code.ts +107 -0
- package/src/providers/oauth/github-copilot.ts +146 -0
- package/src/providers/oauth/loopback.ts +158 -0
- package/src/providers/oauth/openai.ts +163 -0
- package/src/providers/oauth/openrouter.ts +89 -0
- package/src/providers/oauth/pkce.ts +45 -0
- package/src/providers/oauth/registry.ts +39 -0
- package/src/providers/oauth/seam.ts +89 -0
- package/src/providers/profile-glm53.ts +111 -0
- package/src/providers/profile-sonnet5-persona.ts +65 -0
- package/src/providers/profile-sonnet5-voice.ts +23 -0
- package/src/providers/profiles.ts +156 -0
- package/src/providers/provider-config.ts +311 -0
- package/src/providers/registry.ts +333 -0
- package/src/providers/responses.ts +209 -0
- package/src/providers/retry.ts +234 -0
- package/src/providers/router.ts +294 -0
- package/src/providers/sse.ts +26 -0
- package/src/providers/stream-errors.ts +117 -0
- package/src/providers/stream.ts +566 -0
- package/src/providers/thinking.ts +189 -0
- package/src/providers/wire-messages.ts +129 -0
- package/src/providers/wire-responses.ts +79 -0
- package/src/providers/wire-select.ts +53 -0
- package/src/server/http.ts +291 -0
- package/src/server/openapi.ts +246 -0
- package/src/sextant/card-hits.ts +102 -0
- package/src/sextant/card-keys.ts +55 -0
- package/src/sextant/context-source.ts +157 -0
- package/src/sextant/crew-cards.ts +350 -0
- package/src/sextant/draw-agents.ts +273 -0
- package/src/sextant/draw-code.ts +388 -0
- package/src/sextant/draw-context.ts +222 -0
- package/src/sextant/draw-frame.ts +164 -0
- package/src/sextant/draw-market.ts +573 -0
- package/src/sextant/draw-messages.ts +386 -0
- package/src/sextant/draw-pet.ts +230 -0
- package/src/sextant/draw-plan.ts +187 -0
- package/src/sextant/draw-tabs.ts +85 -0
- package/src/sextant/draw-util.ts +65 -0
- package/src/sextant/draw-wizard.ts +378 -0
- package/src/sextant/engine.ts +230 -0
- package/src/sextant/frame-hits.ts +25 -0
- package/src/sextant/frame.ts +101 -0
- package/src/sextant/git-status.ts +197 -0
- package/src/sextant/grid.ts +59 -0
- package/src/sextant/input.ts +119 -0
- package/src/sextant/keys.ts +521 -0
- package/src/sextant/layout.ts +86 -0
- package/src/sextant/local-commands.ts +169 -0
- package/src/sextant/market-source.ts +287 -0
- package/src/sextant/mentions.ts +200 -0
- package/src/sextant/message-hits.ts +26 -0
- package/src/sextant/model.ts +387 -0
- package/src/sextant/overlays.ts +456 -0
- package/src/sextant/panel-hits.ts +38 -0
- package/src/sextant/pet.ts +399 -0
- package/src/sextant/screen.ts +324 -0
- package/src/sextant/scroll-hits.ts +66 -0
- package/src/sextant/scrollbar.ts +82 -0
- package/src/sextant/sextant-bridge.ts +174 -0
- package/src/sextant/sextant-cards.ts +142 -0
- package/src/sextant/sextant-diff-base.ts +63 -0
- package/src/sextant/sextant-files.ts +154 -0
- package/src/sextant/sextant-frame-loop.ts +335 -0
- package/src/sextant/sextant-renderer.ts +574 -0
- package/src/sextant/sextant-repo.ts +140 -0
- package/src/sextant/theme.ts +66 -0
- package/src/sextant/tool-rows.ts +189 -0
- package/src/sextant/types.ts +493 -0
- package/src/skills/index.ts +387 -0
- package/src/skills/pack.ts +220 -0
- package/src/skills/spec.ts +162 -0
- package/src/skills/tools.ts +69 -0
- package/src/skills/versioned.ts +227 -0
- package/src/telemetry/otel-export.ts +122 -0
- package/src/telemetry/otel-lanes.ts +89 -0
- package/src/telemetry/otel-logs.ts +131 -0
- package/src/telemetry/otel-metrics.ts +136 -0
- package/src/telemetry/otel.ts +397 -0
- package/src/telemetry/otlp.ts +76 -0
- package/src/tools/ask-user.ts +156 -0
- package/src/tools/bash-bg.ts +94 -0
- package/src/tools/bash-jobs.ts +237 -0
- package/src/tools/design.ts +151 -0
- package/src/tools/evalcell.ts +338 -0
- package/src/tools/html-text.ts +139 -0
- package/src/tools/provider.ts +149 -0
- package/src/tools/task.ts +250 -0
- package/src/tools/todo.ts +320 -0
- package/src/tools/webfetch.ts +332 -0
- package/src/tools/websearch.ts +359 -0
- package/src/tui/agents-cmd.ts +41 -0
- package/src/tui/app.ts +749 -0
- package/src/tui/attach.ts +127 -0
- package/src/tui/boot-notes.ts +41 -0
- package/src/tui/builtin-prompts.ts +59 -0
- package/src/tui/checkpoints-cmd.ts +70 -0
- package/src/tui/clipboard-image.ts +81 -0
- package/src/tui/clipboard.ts +78 -0
- package/src/tui/commands.ts +283 -0
- package/src/tui/config-view.ts +53 -0
- package/src/tui/context-cmds.ts +282 -0
- package/src/tui/cost.ts +108 -0
- package/src/tui/crash-guard.ts +173 -0
- package/src/tui/focus-terminal.ts +34 -0
- package/src/tui/git-cmds.ts +273 -0
- package/src/tui/git-plain.ts +58 -0
- package/src/tui/info-cmd.ts +150 -0
- package/src/tui/input-plain.ts +76 -0
- package/src/tui/mcp-cmd.ts +128 -0
- package/src/tui/memory-note.ts +77 -0
- package/src/tui/modes-cmd.ts +45 -0
- package/src/tui/notify-seq.ts +100 -0
- package/src/tui/notify.ts +318 -0
- package/src/tui/overlays.ts +97 -0
- package/src/tui/pi-renderer.ts +428 -0
- package/src/tui/providers-cmd.ts +377 -0
- package/src/tui/reasoning-view.ts +56 -0
- package/src/tui/renderer.ts +128 -0
- package/src/tui/replay-marker.ts +29 -0
- package/src/tui/session-cmd.ts +148 -0
- package/src/tui/session-manage.ts +95 -0
- package/src/tui/sextant-attach.ts +102 -0
- package/src/tui/sextant-io.ts +202 -0
- package/src/tui/sextant-smoke.ts +110 -0
- package/src/tui/shell-cmd.ts +158 -0
- package/src/tui/smoke.ts +72 -0
- package/src/tui/staged-terminal.ts +50 -0
- package/src/tui/startup.ts +12 -0
- package/src/tui/theme.ts +59 -0
- package/src/tui/todo-label.ts +7 -0
- package/src/tui/trust-card.ts +107 -0
- package/src/tui/tui-commands.ts +87 -0
- package/tsconfig.json +30 -0
- package/vendor/pi-tui/LICENSE +21 -0
- package/vendor/pi-tui/PATCHES.md +12 -0
- package/vendor/pi-tui/PROVENANCE.md +12 -0
- package/vendor/pi-tui/README.upstream.md +854 -0
- package/vendor/pi-tui/native/win32/prebuilds/win32-arm64/win32-console-mode.node +0 -0
- package/vendor/pi-tui/native/win32/prebuilds/win32-x64/win32-console-mode.node +0 -0
- package/vendor/pi-tui/src/alt-screen-search.ts +158 -0
- package/vendor/pi-tui/src/autocomplete.ts +827 -0
- package/vendor/pi-tui/src/components/alt-screen-flash.ts +52 -0
- package/vendor/pi-tui/src/components/box.ts +138 -0
- package/vendor/pi-tui/src/components/cancellable-loader.ts +41 -0
- package/vendor/pi-tui/src/components/editor.ts +2364 -0
- package/vendor/pi-tui/src/components/h-stack.ts +45 -0
- package/vendor/pi-tui/src/components/image.ts +128 -0
- package/vendor/pi-tui/src/components/input.ts +448 -0
- package/vendor/pi-tui/src/components/loader.ts +93 -0
- package/vendor/pi-tui/src/components/markdown.ts +1016 -0
- package/vendor/pi-tui/src/components/scroll-view.ts +217 -0
- package/vendor/pi-tui/src/components/select-list.ts +230 -0
- package/vendor/pi-tui/src/components/settings-list.ts +277 -0
- package/vendor/pi-tui/src/components/spacer.ts +29 -0
- package/vendor/pi-tui/src/components/stack.ts +155 -0
- package/vendor/pi-tui/src/components/text.ts +108 -0
- package/vendor/pi-tui/src/components/truncated-text.ts +66 -0
- package/vendor/pi-tui/src/components/v-stack.ts +34 -0
- package/vendor/pi-tui/src/editor-component.ts +75 -0
- package/vendor/pi-tui/src/fuzzy.ts +138 -0
- package/vendor/pi-tui/src/index.ts +149 -0
- package/vendor/pi-tui/src/keybindings.ts +321 -0
- package/vendor/pi-tui/src/keys.ts +1402 -0
- package/vendor/pi-tui/src/kill-ring.ts +47 -0
- package/vendor/pi-tui/src/latex.ts +1381 -0
- package/vendor/pi-tui/src/layout-node.ts +52 -0
- package/vendor/pi-tui/src/layout.ts +411 -0
- package/vendor/pi-tui/src/native-modifiers.ts +60 -0
- package/vendor/pi-tui/src/native-module-path.ts +32 -0
- package/vendor/pi-tui/src/stdin-buffer.ts +445 -0
- package/vendor/pi-tui/src/terminal-colors.ts +74 -0
- package/vendor/pi-tui/src/terminal-image.ts +701 -0
- package/vendor/pi-tui/src/terminal.ts +554 -0
- package/vendor/pi-tui/src/tui-alt-screen.ts +1379 -0
- package/vendor/pi-tui/src/tui-main-screen.ts +655 -0
- package/vendor/pi-tui/src/tui.ts +1264 -0
- package/vendor/pi-tui/src/undo-stack.ts +29 -0
- package/vendor/pi-tui/src/utils.ts +1327 -0
- package/vendor/pi-tui/src/word-navigation.ts +118 -0
- package/vendor/pi-tui/test/test-themes.ts +39 -0
- package/vendor/pi-tui/test/virtual-terminal.ts +219 -0
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
/** Bounded markdown memory blocks (hermes pattern): MEMORY.md + USER.md with hard char caps.
|
|
2
|
+
* The snapshot is frozen at boot (constructor) so the system prompt stays cache-stable and
|
|
3
|
+
* byte-identical across the turns of a run; tools mutate live state + disk only — an edit is in the
|
|
4
|
+
* prompt from the NEXT run. A threat-scan neutralizes injection lines in the rendered view — raw text
|
|
5
|
+
* on disk is never rewritten by the scan.
|
|
6
|
+
*
|
|
7
|
+
* Scoping (2026-09-07): the two blocks may live in DIFFERENT directories (`BlockDirs` — MEMORY under
|
|
8
|
+
* the project's .rovecode, USER under the user home; memory/scope.ts resolves them), because a USER
|
|
9
|
+
* block written inside a repository is the wrong file in the wrong place. A plain string keeps both in
|
|
10
|
+
* one dir. Directories are created lazily by the first commit (the ledger's write), so constructing a
|
|
11
|
+
* store over the user scope never touches the home directory.
|
|
12
|
+
*
|
|
13
|
+
* A block file is REPO DATA — `<cwd>/.rovecode/memory/MEMORY.md` is committable — so the boot read is
|
|
14
|
+
* bounded twice: at most READ_CAP_BYTES are read from disk (a longer file marks the block read-cut and
|
|
15
|
+
* every commit/rollback is refused — the store never rewrites a file it could not read whole), and the
|
|
16
|
+
* PROMPT view is cut at the block's char cap (line boundary) behind a visible `[truncated: …]` marker,
|
|
17
|
+
* so the prompt cannot balloon whatever the file holds; the live text (/memory, the tools) is what was
|
|
18
|
+
* read. A block may also be WITHHELD (`BlockStoreOptions.withhold`): read as usual, kept out of the
|
|
19
|
+
* prompt — memory/scope.ts withholds a project MEMORY block this machine has not approved. */
|
|
20
|
+
|
|
21
|
+
import { closeSync, existsSync, openSync, readSync } from "node:fs";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
import { TRUST_HINT } from "../core/trust.ts";
|
|
24
|
+
import { VersionLedger } from "../skills/versioned.ts";
|
|
25
|
+
|
|
26
|
+
export type BlockName = "memory" | "user";
|
|
27
|
+
|
|
28
|
+
/** the directory each block's file (+ its `.versions.jsonl` ledger sidecar) lives in */
|
|
29
|
+
export type BlockDirs = Record<BlockName, string>;
|
|
30
|
+
|
|
31
|
+
/** the block file names — the ONE place that spells them (memory/scope.ts copies them by these names) */
|
|
32
|
+
export const BLOCK_FILES: Readonly<Record<BlockName, string>> = { memory: "MEMORY.md", user: "USER.md" };
|
|
33
|
+
|
|
34
|
+
export interface BlockCaps { memory: number; user: number }
|
|
35
|
+
|
|
36
|
+
export const defaultCaps: BlockCaps = { memory: 2_200, user: 1_375 };
|
|
37
|
+
|
|
38
|
+
/** the most a block file is read at boot (bytes); past it the block is read-cut: shown cut, never rewritten */
|
|
39
|
+
export const READ_CAP_BYTES = 64 * 1024;
|
|
40
|
+
|
|
41
|
+
export interface BlockStoreOptions {
|
|
42
|
+
/** blocks read from disk but KEPT OUT of the prompt (a project MEMORY block this machine has not
|
|
43
|
+
* approved); live text and /memory see them as usual, and every WRITE to them is refused — appending
|
|
44
|
+
* our own note to a stranger's file would launder its content into the next prompt */
|
|
45
|
+
withhold?: readonly BlockName[];
|
|
46
|
+
/** called after a write THIS store performed landed on disk (commit or rollback), with the block and its
|
|
47
|
+
* file. memory/scope.ts uses it to keep the project block's trust digest current: the narrow self-trust —
|
|
48
|
+
* a file we just wrote, never "the file looks close enough". Never called for a refused write. */
|
|
49
|
+
onCommit?: (block: BlockName, file: string) => void;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface BlockEditResult {
|
|
53
|
+
ok: boolean;
|
|
54
|
+
reason?: string;
|
|
55
|
+
/** present on cap failures: current chars / cap */
|
|
56
|
+
current?: number;
|
|
57
|
+
limit?: number;
|
|
58
|
+
/** present on version conflicts: ledger numbers for logs/data surfaces only —
|
|
59
|
+
* the model-facing `reason` stays generic (ouroboros: no ledger internals). */
|
|
60
|
+
conflict?: { baseVersion: number; currentVersion: number };
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/** Model-facing conflict reason — deliberately free of ledger internals. */
|
|
64
|
+
export const CONFLICT_REASON = "memory changed since it was read — re-read and retry";
|
|
65
|
+
|
|
66
|
+
const THREAT = /(?:ignore previous|disregard above|system prompt)/i;
|
|
67
|
+
|
|
68
|
+
function scan(text: string): string {
|
|
69
|
+
return text.split("\n").map((l) => (THREAT.test(l) ? "[BLOCKED]" : l)).join("\n");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** `text` cut to at most `cap` chars at a line boundary (a hard cut when the last newline sits in the first half) */
|
|
73
|
+
export function cutAtCap(text: string, cap: number): string {
|
|
74
|
+
if (text.length <= cap) return text;
|
|
75
|
+
let cut = text.lastIndexOf("\n", cap);
|
|
76
|
+
if (cut < cap / 2) cut = cap;
|
|
77
|
+
return text.slice(0, cut);
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
/** at most `cap + 1` bytes of `p` (existence checked by the caller): the text and whether the file went on past the cap */
|
|
81
|
+
function readBounded(p: string, cap: number): { text: string; cut: boolean } {
|
|
82
|
+
const fd = openSync(p, "r");
|
|
83
|
+
try {
|
|
84
|
+
const buf = Buffer.alloc(cap + 1);
|
|
85
|
+
let n = 0;
|
|
86
|
+
while (n < buf.length) { const got = readSync(fd, buf, n, buf.length - n, n); if (got <= 0) break; n += got; }
|
|
87
|
+
return { text: buf.subarray(0, Math.min(n, cap)).toString("utf8"), cut: n > cap };
|
|
88
|
+
} finally { closeSync(fd); }
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function occurrences(haystack: string, needle: string): number {
|
|
92
|
+
if (!needle) return 0;
|
|
93
|
+
let n = 0;
|
|
94
|
+
let i = 0;
|
|
95
|
+
while ((i = haystack.indexOf(needle, i)) !== -1) { n++; i += needle.length; }
|
|
96
|
+
return n;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export class BlockStore {
|
|
100
|
+
private live: Record<BlockName, string> = { memory: "", user: "" };
|
|
101
|
+
private readonly frozen: Record<BlockName, string>;
|
|
102
|
+
/** port #16: versioned edits — every commit lands through a sidecar ledger
|
|
103
|
+
* (<file>.versions.jsonl), enabling one-call rollback + drift detection. */
|
|
104
|
+
private readonly ledgers: Record<BlockName, VersionLedger>;
|
|
105
|
+
/** Optimistic-concurrency baseline (prime baselineState, refinement.ts:735-749):
|
|
106
|
+
* the version THIS store last saw. Passing ledger.version() to commit would
|
|
107
|
+
* compare the guard to itself and silently drop a concurrent store's edit. */
|
|
108
|
+
private readonly baseVersions: Record<BlockName, number>;
|
|
109
|
+
|
|
110
|
+
private readonly dirs: BlockDirs;
|
|
111
|
+
/** the block's file went on past READ_CAP_BYTES at boot — shown cut, never rewritten */
|
|
112
|
+
private readonly readCut: Record<BlockName, boolean> = { memory: false, user: false };
|
|
113
|
+
private readonly withheld: ReadonlySet<BlockName>;
|
|
114
|
+
private readonly onCommit: ((block: BlockName, file: string) => void) | undefined;
|
|
115
|
+
|
|
116
|
+
/** `dir`: one directory for both blocks, or a per-block map (the scoped layout). Nothing is created here. */
|
|
117
|
+
constructor(
|
|
118
|
+
dir: string | BlockDirs,
|
|
119
|
+
private readonly caps: BlockCaps = defaultCaps,
|
|
120
|
+
opts: BlockStoreOptions = {},
|
|
121
|
+
) {
|
|
122
|
+
// no mkdir here: reads tolerate a missing directory and the ledger creates it on the first commit
|
|
123
|
+
// (skills/versioned.ts). Creating `<session>/memory` at construction was what made every session
|
|
124
|
+
// directory non-empty before anyone had said anything (core/session.ts materialize()).
|
|
125
|
+
this.dirs = typeof dir === "string" ? { memory: dir, user: dir } : { memory: dir.memory, user: dir.user };
|
|
126
|
+
this.withheld = new Set(opts.withhold ?? []);
|
|
127
|
+
this.onCommit = opts.onCommit;
|
|
128
|
+
this.live.memory = this.read("memory");
|
|
129
|
+
this.live.user = this.read("user");
|
|
130
|
+
// snapshot is the threat-scanned view of what boot loaded
|
|
131
|
+
this.frozen = { memory: scan(this.live.memory), user: scan(this.live.user) };
|
|
132
|
+
this.ledgers = {
|
|
133
|
+
memory: new VersionLedger(this.file("memory")),
|
|
134
|
+
user: new VersionLedger(this.file("user")),
|
|
135
|
+
};
|
|
136
|
+
this.baseVersions = { memory: this.ledgers.memory.version(), user: this.ledgers.user.version() };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** the bounded boot read (header): at most READ_CAP_BYTES; a longer file marks the block read-cut */
|
|
140
|
+
private read(block: BlockName): string {
|
|
141
|
+
const p = this.file(block);
|
|
142
|
+
if (!existsSync(p)) return "";
|
|
143
|
+
const r = readBounded(p, READ_CAP_BYTES);
|
|
144
|
+
this.readCut[block] = r.cut;
|
|
145
|
+
return r.text;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
private file(block: BlockName): string {
|
|
149
|
+
return join(this.dirs[block], BLOCK_FILES[block]);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/** the block's file path (where edits land) — for notes, /memory and tests. */
|
|
153
|
+
path(block: BlockName): string { return this.file(block); }
|
|
154
|
+
|
|
155
|
+
add(block: BlockName, text: string): BlockEditResult {
|
|
156
|
+
const t = text.trim();
|
|
157
|
+
if (!t) return { ok: false, reason: "empty text" };
|
|
158
|
+
const next = this.live[block] ? this.live[block] + "\n" + t : t;
|
|
159
|
+
return this.commit(block, next);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
replace(block: BlockName, oldText: string, newText: string): BlockEditResult {
|
|
163
|
+
const found = this.locate(block, oldText);
|
|
164
|
+
if (!found.ok) return found;
|
|
165
|
+
const t = newText.trim();
|
|
166
|
+
const next = t ? this.live[block].replace(oldText, t) : this.live[block].replace(oldText, "");
|
|
167
|
+
return this.commit(block, next.trim());
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
remove(block: BlockName, oldText: string): BlockEditResult {
|
|
171
|
+
const found = this.locate(block, oldText);
|
|
172
|
+
if (!found.ok) return found;
|
|
173
|
+
const next = this.live[block].replace(oldText, "").replace(/\n{3,}/g, "\n\n").trim();
|
|
174
|
+
return this.commit(block, next);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/** oldText must match exactly once: 0 → not found, >1 → ambiguous. */
|
|
178
|
+
private locate(block: BlockName, oldText: string): BlockEditResult {
|
|
179
|
+
if (!oldText) return { ok: false, reason: "oldText required" };
|
|
180
|
+
const n = occurrences(this.live[block], oldText);
|
|
181
|
+
if (n === 0) return { ok: false, reason: "oldText not found" };
|
|
182
|
+
if (n > 1) return { ok: false, reason: `oldText matches ${n} times; must match exactly once` };
|
|
183
|
+
return { ok: true };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
/** Char-cap guard shared by commit() and rollback(); runs BEFORE any ledger write. A read-cut block
|
|
187
|
+
* (header) refuses every write: the store never rewrites a file it could not read whole. */
|
|
188
|
+
private capCheck(block: BlockName, next: string): BlockEditResult | null {
|
|
189
|
+
const limit = this.caps[block];
|
|
190
|
+
if (this.withheld.has(block)) {
|
|
191
|
+
return { ok: false, reason: `${BLOCK_FILES[block]} came with this repository and is not approved on this machine — it is not in the prompt and cannot be written to (${TRUST_HINT})`, current: this.live[block].length, limit };
|
|
192
|
+
}
|
|
193
|
+
if (this.readCut[block]) {
|
|
194
|
+
return { ok: false, reason: `${BLOCK_FILES[block]} on disk is longer than the ${READ_CAP_BYTES}-byte read cap — trim it by hand before editing`, current: this.live[block].length, limit };
|
|
195
|
+
}
|
|
196
|
+
if (next.length <= limit) return null;
|
|
197
|
+
return {
|
|
198
|
+
ok: false, reason: `block would exceed cap (${next.length} > ${limit} chars)`,
|
|
199
|
+
current: this.live[block].length, limit,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
private commit(block: BlockName, next: string): BlockEditResult {
|
|
204
|
+
const capped = this.capCheck(block, next);
|
|
205
|
+
if (capped) return capped;
|
|
206
|
+
// ledger-first write (port #16): version record appended, then the target
|
|
207
|
+
// is written atomically by the ledger — never writeFileSync directly. The
|
|
208
|
+
// baseline is what THIS store last saw, so a concurrent store's commit is
|
|
209
|
+
// caught here instead of being silently overwritten.
|
|
210
|
+
const r = this.ledgers[block].commit(next, "memory_edit", this.baseVersions[block]);
|
|
211
|
+
if (!r.ok) {
|
|
212
|
+
// conflict: another store/process advanced the block. Re-read reality so
|
|
213
|
+
// the next edit builds on the merged state; the reason stays generic —
|
|
214
|
+
// ledger numbers ride the structured field only.
|
|
215
|
+
this.live[block] = this.ledgers[block].read();
|
|
216
|
+
this.baseVersions[block] = r.currentVersion;
|
|
217
|
+
return {
|
|
218
|
+
ok: false, reason: CONFLICT_REASON,
|
|
219
|
+
conflict: { baseVersion: r.baseVersion, currentVersion: r.currentVersion },
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
this.baseVersions[block] = r.edit.version;
|
|
223
|
+
this.live[block] = next;
|
|
224
|
+
this.onCommit?.(block, this.file(block)); // the write landed and it was ours (memory/scope.ts self-trust)
|
|
225
|
+
return { ok: true, current: next.length, limit: this.caps[block] };
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
/** port #16: one-call rollback of a block to any retained version.
|
|
229
|
+
* Restored content re-enters through the same cap guard as commits. */
|
|
230
|
+
rollback(block: BlockName, toVersion: number): BlockEditResult {
|
|
231
|
+
const restored = this.ledgers[block].contentAt(toVersion);
|
|
232
|
+
const capped = restored !== undefined ? this.capCheck(block, restored) : null;
|
|
233
|
+
if (capped) return capped;
|
|
234
|
+
const r = this.ledgers[block].rollback(toVersion, "memory_rollback");
|
|
235
|
+
if (!r.ok) return { ok: false, reason: r.message };
|
|
236
|
+
this.baseVersions[block] = r.edit.version;
|
|
237
|
+
this.live[block] = this.ledgers[block].read();
|
|
238
|
+
this.onCommit?.(block, this.file(block)); // a rollback rewrites the file too — same self-trust
|
|
239
|
+
return { ok: true, current: this.live[block].length, limit: this.caps[block] };
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** port #16: version history access for status/debug surfaces. */
|
|
243
|
+
ledger(block: BlockName): VersionLedger { return this.ledgers[block]; }
|
|
244
|
+
|
|
245
|
+
/** the prompt text of one block: the boot snapshot, cut at the block's cap (header) behind a visible marker */
|
|
246
|
+
private promptView(block: BlockName): string {
|
|
247
|
+
const text = this.frozen[block], cap = this.caps[block];
|
|
248
|
+
if (text.length <= cap && !this.readCut[block]) return text;
|
|
249
|
+
const size = this.readCut[block] ? `more than ${READ_CAP_BYTES} bytes` : `${this.live[block].length} chars`;
|
|
250
|
+
return `${cutAtCap(text, cap)}\n[truncated: ${BLOCK_FILES[block]} holds ${size}; the first ${cap} chars are shown — trim the file]`;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/** Boot snapshot (threat-scanned, cut at the cap; withheld blocks absent). Tool writes never change it. */
|
|
254
|
+
renderForPrompt(): string {
|
|
255
|
+
const parts: string[] = [];
|
|
256
|
+
if (this.frozen.memory && !this.withheld.has("memory")) parts.push(`# Memory\n${this.promptView("memory")}`);
|
|
257
|
+
if (this.frozen.user && !this.withheld.has("user")) parts.push(`# User\n${this.promptView("user")}`);
|
|
258
|
+
return parts.join("\n\n");
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/** Live (post-snapshot) raw text — for reads/debug, not for the prompt. */
|
|
262
|
+
liveText(block: BlockName): string { return this.live[block]; }
|
|
263
|
+
|
|
264
|
+
/** true when the block's live text no longer matches the boot snapshot the prompt shows
|
|
265
|
+
* (an edit this run — the prompt picks it up on the next run). */
|
|
266
|
+
edited(block: BlockName): boolean { return scan(this.live[block]) !== this.frozen[block]; }
|
|
267
|
+
|
|
268
|
+
cap(block: BlockName): number { return this.caps[block]; }
|
|
269
|
+
|
|
270
|
+
/** the block's boot read against its cap — null when it fits; `readCut` = the file went past READ_CAP_BYTES */
|
|
271
|
+
overCap(block: BlockName): { chars: number; cap: number; readCut: boolean } | null {
|
|
272
|
+
if (this.live[block].length <= this.caps[block] && !this.readCut[block]) return null;
|
|
273
|
+
return { chars: this.live[block].length, cap: this.caps[block], readCut: this.readCut[block] };
|
|
274
|
+
}
|
|
275
|
+
|
|
276
|
+
/** true when the block is read (live, /memory, memory_edit) but kept out of the prompt (an unapproved project file) */
|
|
277
|
+
isWithheld(block: BlockName): boolean { return this.withheld.has(block); }
|
|
278
|
+
}
|
|
@@ -0,0 +1,355 @@
|
|
|
1
|
+
/** Cross-session recall (port #17): tokenized full-text search over
|
|
2
|
+
* `.rovecode/sessions/<id>/entries.jsonl` message text, exposed as a `recall` tool.
|
|
3
|
+
*
|
|
4
|
+
* Ported from hermes-agent's session search (MIT), adapted from SQLite FTS5 to an
|
|
5
|
+
* in-memory inverted index over the JSONL session tree:
|
|
6
|
+
* - Index = tokenized message content, maintained incrementally; hermes keeps an
|
|
7
|
+
* external-content FTS5 table in sync via insert triggers + high-water/progress
|
|
8
|
+
* markers so only unindexed rows are (re)indexed (hermes_state_common.py:637-684,
|
|
9
|
+
* hermes_state_search.py:280-346 fts_rebuild_step). Here the incremental unit is
|
|
10
|
+
* the session FILE, keyed by mtime (+size guard): only changed files re-index.
|
|
11
|
+
* - Ranking tiers = hermes's routing: exact tokenized FTS5 match is the primary
|
|
12
|
+
* path, substring (trigram) matching is the fallback tier
|
|
13
|
+
* (hermes_state_search.py:1467-1489 _describe_search_path, 1846-1885 routing);
|
|
14
|
+
* within a tier hermes orders by BM25 `ORDER BY rank` with timestamp tiebreaks
|
|
15
|
+
* (hermes_state_search.py:1791-1798). Here: exact-term count desc, then weighted
|
|
16
|
+
* term frequency, then recency. Query terms are implicitly ANDed, matching FTS5
|
|
17
|
+
* (tools/session_search_tool.py:807-809).
|
|
18
|
+
* - Partial matches need terms >=3 chars, mirroring trigram eligibility
|
|
19
|
+
* (hermes_state_search.py:1322-1337 _trigram_eligible_tokens).
|
|
20
|
+
* - Hits are snippet + metadata only, never full content
|
|
21
|
+
* (hermes_state_search.py:1694-1701, 1827-1844); preview = 120-char window
|
|
22
|
+
* starting 40 before the first match, the LIKE-fallback snippet shape
|
|
23
|
+
* (hermes_state_search.py:1585-1595).
|
|
24
|
+
* - Result budget: limit clamped like hermes's max(1, min(limit, 10))
|
|
25
|
+
* (tools/session_search_tool.py:1040-1046).
|
|
26
|
+
* - NO LLM anywhere in the search path (tools/session_search_tool.py:25-33 — the
|
|
27
|
+
* historical "summary mode" was removed upstream); summarization is an optional
|
|
28
|
+
* injected fn here, off by default.
|
|
29
|
+
* - Trust boundary: previews get the same per-line injection neutralization as
|
|
30
|
+
* BlockStore (blocks.ts:24-27) — threat lines render as [BLOCKED]; disk is never
|
|
31
|
+
* rewritten. The query echo in tool output is bounded to MAX_QUERY_CHARS.
|
|
32
|
+
*/
|
|
33
|
+
|
|
34
|
+
import { readdirSync, readFileSync, statSync } from "node:fs";
|
|
35
|
+
import { join } from "node:path";
|
|
36
|
+
import type { Tool, ToolContext, ToolOutput } from "../core/types.ts";
|
|
37
|
+
|
|
38
|
+
/** Ranked hit — the bar's exact shape. Preview only; full text stays on disk. */
|
|
39
|
+
export interface RecallHit { sessionId: string; entryId: string; preview: string; timestamp: number }
|
|
40
|
+
|
|
41
|
+
/** Optional post-search summarizer (LLM or otherwise) — injected, never required. */
|
|
42
|
+
export type SummarizeFn = (query: string, hits: RecallHit[]) => Promise<string> | string;
|
|
43
|
+
|
|
44
|
+
export interface RefreshStats { scanned: number; indexed: number; removed: number }
|
|
45
|
+
|
|
46
|
+
const MAX_RESULTS = 10; // hermes limit ceiling (session_search_tool.py:1046)
|
|
47
|
+
const DEFAULT_RESULTS = 5;
|
|
48
|
+
const PREVIEW_WINDOW = 120; // hermes LIKE-fallback snippet width (hermes_state_search.py:1587)
|
|
49
|
+
const PREVIEW_LEAD = 40; // window starts 40 chars before the match (same line)
|
|
50
|
+
const MIN_PARTIAL_TERM = 3; // trigram eligibility (hermes_state_search.py:1322-1337)
|
|
51
|
+
const MAX_QUERY_CHARS = 512; // bounded adversarial input (hermes MAX_FTS5_QUERY_CHARS, hermes_state_search.py:1199-1201)
|
|
52
|
+
const EXACT_WEIGHT = 2;
|
|
53
|
+
const PARTIAL_WEIGHT = 1;
|
|
54
|
+
|
|
55
|
+
/** unicode61-style tokenization (hermes's base FTS5 index): case-fold, split on
|
|
56
|
+
* anything that is not a letter or digit. */
|
|
57
|
+
export function tokenize(text: string): string[] {
|
|
58
|
+
return text.toLowerCase().split(/[^\p{L}\p{N}]+/u).filter((t) => t.length > 0);
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
interface Doc {
|
|
62
|
+
sessionId: string;
|
|
63
|
+
entryId: string;
|
|
64
|
+
text: string;
|
|
65
|
+
timestamp: number;
|
|
66
|
+
tokens: Map<string, number>; // token -> tf (kept for O(tokens) removal on re-index)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
interface FileState { mtimeMs: number; size: number; docKeys: string[] }
|
|
70
|
+
|
|
71
|
+
/** One session-entry line, already filtered to message text. */
|
|
72
|
+
function parseLine(line: string): { entryId: string; text: string; timestamp: number } | null {
|
|
73
|
+
let raw: unknown;
|
|
74
|
+
try { raw = JSON.parse(line); } catch { return null; } // malformed lines: skip, never throw (session.ts reload pattern)
|
|
75
|
+
if (!raw || typeof raw !== "object") return null;
|
|
76
|
+
const w = raw as { id?: unknown; createdAt?: unknown; entry?: unknown };
|
|
77
|
+
if (typeof w.id !== "string") return null;
|
|
78
|
+
const e = w.entry;
|
|
79
|
+
// events (kind:"event") and non-message shapes are not "message text" — skip
|
|
80
|
+
if (!e || typeof e !== "object" || !("role" in e)) return null;
|
|
81
|
+
const parts = (e as { parts?: unknown }).parts;
|
|
82
|
+
if (!Array.isArray(parts)) return null;
|
|
83
|
+
const texts: string[] = [];
|
|
84
|
+
for (const p of parts) {
|
|
85
|
+
if (p && typeof p === "object"
|
|
86
|
+
&& (p as { kind?: unknown }).kind === "text"
|
|
87
|
+
&& typeof (p as { text?: unknown }).text === "string") {
|
|
88
|
+
texts.push((p as { text: string }).text);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const text = texts.join(" ").trim();
|
|
92
|
+
if (!text) return null; // tool_call/tool_result-only messages carry no text parts
|
|
93
|
+
return { entryId: w.id, text, timestamp: typeof w.createdAt === "number" ? w.createdAt : 0 };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/** Recalled transcript text sits at the SAME trust level as BlockStore markdown:
|
|
97
|
+
* text a past model/user wrote, re-entering a live context. Local copy of the
|
|
98
|
+
* blocks.ts:24-27 neutralization — any line matching the injection pattern
|
|
99
|
+
* renders as [BLOCKED]; the raw text on disk is never rewritten. */
|
|
100
|
+
const THREAT = /(?:ignore previous|disregard above|system prompt)/i;
|
|
101
|
+
function neutralize(text: string): string {
|
|
102
|
+
return text.split("\n").map((l) => (THREAT.test(l) ? "[BLOCKED]" : l)).join("\n");
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/** Slicing by UTF-16 unit can strand half of a surrogate pair at either cut —
|
|
106
|
+
* drop a leading low / trailing high orphan so emitted text stays well-formed. */
|
|
107
|
+
function trimOrphanSurrogates(s: string): string {
|
|
108
|
+
const head = s.charCodeAt(0); // NaN on empty: both range checks are false
|
|
109
|
+
if (head >= 0xdc00 && head <= 0xdfff) s = s.slice(1);
|
|
110
|
+
const tail = s.charCodeAt(s.length - 1);
|
|
111
|
+
if (tail >= 0xd800 && tail <= 0xdbff) s = s.slice(0, -1);
|
|
112
|
+
return s;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/** Single-line preview: 120-char window starting 40 before the first matched term
|
|
116
|
+
* (hermes_state_search.py:1585-1595). Ellipses mark clipping. */
|
|
117
|
+
function makePreview(text: string, terms: string[]): string {
|
|
118
|
+
const flat = text.replace(/\s+/g, " ").trim();
|
|
119
|
+
const lower = flat.toLowerCase();
|
|
120
|
+
let pos = -1;
|
|
121
|
+
for (const t of terms) {
|
|
122
|
+
const i = lower.indexOf(t);
|
|
123
|
+
if (i !== -1 && (pos === -1 || i < pos)) pos = i;
|
|
124
|
+
}
|
|
125
|
+
const start = pos === -1 ? 0 : Math.max(0, pos - PREVIEW_LEAD);
|
|
126
|
+
const clip = trimOrphanSurrogates(flat.slice(start, start + PREVIEW_WINDOW));
|
|
127
|
+
return (start > 0 ? "…" : "") + clip + (start + PREVIEW_WINDOW < flat.length ? "…" : "");
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/** Inverted index over every session's entries.jsonl. Incremental: a file is
|
|
131
|
+
* re-read only when its mtime (or size — appends always grow JSONL) changed;
|
|
132
|
+
* deleted session dirs drop out. Session identity = the DIRECTORY name, matching
|
|
133
|
+
* session.ts listSessions (a tampered meta.json must not redirect recall). */
|
|
134
|
+
export class RecallIndex {
|
|
135
|
+
private docs = new Map<string, Doc>(); // docKey -> doc
|
|
136
|
+
private postings = new Map<string, Map<string, number>>(); // token -> docKey -> tf
|
|
137
|
+
private files = new Map<string, FileState>(); // sessionId -> file state
|
|
138
|
+
|
|
139
|
+
constructor(private readonly root: string) {}
|
|
140
|
+
|
|
141
|
+
/** Stat every session file; (re)index changed/new ones, drop vanished ones. */
|
|
142
|
+
refresh(): RefreshStats {
|
|
143
|
+
const stats: RefreshStats = { scanned: 0, indexed: 0, removed: 0 };
|
|
144
|
+
let names: string[] = [];
|
|
145
|
+
try { names = readdirSync(this.root); } catch { /* missing root = empty index */ }
|
|
146
|
+
const live = new Set<string>();
|
|
147
|
+
for (const name of names) {
|
|
148
|
+
const file = join(this.root, name, "entries.jsonl");
|
|
149
|
+
let st: { mtimeMs: number; size: number };
|
|
150
|
+
try { const s = statSync(file); st = { mtimeMs: s.mtimeMs, size: s.size }; }
|
|
151
|
+
catch { continue; } // foreign dir / no entries yet: skip, never throw
|
|
152
|
+
live.add(name);
|
|
153
|
+
stats.scanned++;
|
|
154
|
+
const prev = this.files.get(name);
|
|
155
|
+
if (prev && prev.mtimeMs === st.mtimeMs && prev.size === st.size) continue; // unchanged: keyed by mtime
|
|
156
|
+
this.dropSession(name);
|
|
157
|
+
this.indexFile(name, file, st);
|
|
158
|
+
stats.indexed++;
|
|
159
|
+
}
|
|
160
|
+
for (const name of [...this.files.keys()]) {
|
|
161
|
+
if (!live.has(name)) { this.dropSession(name); stats.removed++; }
|
|
162
|
+
}
|
|
163
|
+
return stats;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
private dropSession(sessionId: string): void {
|
|
167
|
+
const prev = this.files.get(sessionId);
|
|
168
|
+
if (!prev) return;
|
|
169
|
+
for (const key of prev.docKeys) {
|
|
170
|
+
const doc = this.docs.get(key);
|
|
171
|
+
if (doc) {
|
|
172
|
+
for (const token of doc.tokens.keys()) {
|
|
173
|
+
const posting = this.postings.get(token);
|
|
174
|
+
if (posting) { posting.delete(key); if (posting.size === 0) this.postings.delete(token); }
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
this.docs.delete(key);
|
|
178
|
+
}
|
|
179
|
+
this.files.delete(sessionId);
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
private indexFile(sessionId: string, file: string, st: { mtimeMs: number; size: number }): void {
|
|
183
|
+
const docKeys: string[] = [];
|
|
184
|
+
let content = "";
|
|
185
|
+
try { content = readFileSync(file, "utf8"); } catch { /* raced deletion: index as empty */ }
|
|
186
|
+
for (const line of content.split("\n")) {
|
|
187
|
+
if (!line.trim()) continue;
|
|
188
|
+
const parsed = parseLine(line);
|
|
189
|
+
if (!parsed) continue;
|
|
190
|
+
// length-prefixed doc key: immune to separator-content collisions — session
|
|
191
|
+
// "s" + entry "0000e1" and session "s0000" + entry "e1" must stay distinct.
|
|
192
|
+
// The previous separator was the literal 4-char string "0000" (bytes 0x30
|
|
193
|
+
// 0x30 0x30 0x30 — an intended U+0000 NUL escape written without the
|
|
194
|
+
// backslash-u), so exactly that pair collided and the second doc was
|
|
195
|
+
// silently dropped by the duplicate-key guard below.
|
|
196
|
+
const key = `${sessionId.length}:${sessionId}:${parsed.entryId}`;
|
|
197
|
+
if (this.docs.has(key)) continue; // duplicate-id guard (session.ts corruption class)
|
|
198
|
+
const tokens = new Map<string, number>();
|
|
199
|
+
for (const t of tokenize(parsed.text)) tokens.set(t, (tokens.get(t) ?? 0) + 1);
|
|
200
|
+
if (tokens.size === 0) continue;
|
|
201
|
+
this.docs.set(key, { sessionId, entryId: parsed.entryId, text: parsed.text, timestamp: parsed.timestamp, tokens });
|
|
202
|
+
docKeys.push(key);
|
|
203
|
+
for (const [token, tf] of tokens) {
|
|
204
|
+
let posting = this.postings.get(token);
|
|
205
|
+
if (!posting) { posting = new Map(); this.postings.set(token, posting); }
|
|
206
|
+
posting.set(key, tf);
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
this.files.set(sessionId, { mtimeMs: st.mtimeMs, size: st.size, docKeys });
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
/** Ranked search. Terms are ANDed (FTS5 implicit AND). Tiering: docs with more
|
|
213
|
+
* exact-term matches ALWAYS outrank docs matching only partially (hermes's
|
|
214
|
+
* exact-FTS5-before-trigram routing as a rank, not a fallback); within a tier,
|
|
215
|
+
* weighted tf desc, then timestamp desc, then key for determinism. */
|
|
216
|
+
search(query: string, limit: number, excludeSession?: string): RecallHit[] {
|
|
217
|
+
this.refresh();
|
|
218
|
+
const terms = [...new Set(tokenize(query.slice(0, MAX_QUERY_CHARS)))];
|
|
219
|
+
if (terms.length === 0) return [];
|
|
220
|
+
|
|
221
|
+
let cands: Map<string, { exact: number; weighted: number }> | null = null;
|
|
222
|
+
for (const term of terms) {
|
|
223
|
+
const matched = new Map<string, { ex: number; part: number }>();
|
|
224
|
+
const exact = this.postings.get(term);
|
|
225
|
+
if (exact) for (const [key, tf] of exact) matched.set(key, { ex: tf, part: 0 });
|
|
226
|
+
if (term.length >= MIN_PARTIAL_TERM) { // substring tier, trigram-eligible terms only
|
|
227
|
+
for (const [token, posting] of this.postings) {
|
|
228
|
+
if (token === term || !token.includes(term)) continue;
|
|
229
|
+
for (const [key, tf] of posting) {
|
|
230
|
+
const m = matched.get(key) ?? { ex: 0, part: 0 };
|
|
231
|
+
m.part += tf;
|
|
232
|
+
matched.set(key, m);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
}
|
|
236
|
+
const next = new Map<string, { exact: number; weighted: number }>();
|
|
237
|
+
for (const [key, m] of matched) {
|
|
238
|
+
const prev = cands === null ? { exact: 0, weighted: 0 } : cands.get(key);
|
|
239
|
+
if (prev === undefined) continue; // AND: term missing from doc drops it
|
|
240
|
+
next.set(key, {
|
|
241
|
+
exact: prev.exact + (m.ex > 0 ? 1 : 0),
|
|
242
|
+
weighted: prev.weighted + EXACT_WEIGHT * m.ex + PARTIAL_WEIGHT * m.part,
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
cands = next;
|
|
246
|
+
if (cands.size === 0) return [];
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const ranked = [...(cands ?? new Map<string, { exact: number; weighted: number }>())]
|
|
250
|
+
.flatMap(([key, cand]) => {
|
|
251
|
+
const doc = this.docs.get(key);
|
|
252
|
+
// exclude the live session: recall is CROSS-session (hermes skips the
|
|
253
|
+
// current lineage, session_search_tool.py:852-860)
|
|
254
|
+
return doc && doc.sessionId !== excludeSession ? [{ key, cand, doc }] : [];
|
|
255
|
+
})
|
|
256
|
+
.sort((a, b) =>
|
|
257
|
+
b.cand.exact - a.cand.exact
|
|
258
|
+
|| b.cand.weighted - a.cand.weighted
|
|
259
|
+
|| b.doc.timestamp - a.doc.timestamp
|
|
260
|
+
|| (a.key < b.key ? -1 : 1));
|
|
261
|
+
|
|
262
|
+
return ranked.slice(0, Math.max(0, limit)).map(({ doc }) => ({
|
|
263
|
+
sessionId: doc.sessionId,
|
|
264
|
+
entryId: doc.entryId,
|
|
265
|
+
// neutralized HERE so every consumer — tool output, data.hits, the injected
|
|
266
|
+
// summarizer — sees the scanned view, never the verbatim transcript line
|
|
267
|
+
preview: neutralize(makePreview(doc.text, terms)),
|
|
268
|
+
timestamp: doc.timestamp,
|
|
269
|
+
}));
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
export interface RecallToolOptions {
|
|
274
|
+
/** injected summarizer; absent = raw hits only (upstream removed its LLM summary mode) */
|
|
275
|
+
summarize?: SummarizeFn;
|
|
276
|
+
/** result-budget ceiling; defaults to hermes's 10 */
|
|
277
|
+
maxResults?: number;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** Build the `recall` tool over a sessions root (one lazy index per tool instance).
|
|
281
|
+
*
|
|
282
|
+
* kind "read", NOT "memory": recall only READS session files from disk — it never
|
|
283
|
+
* mutates memory. core/tools.ts actionFor() maps "read" -> "file.read" but
|
|
284
|
+
* "memory" -> "memory.write"; gating a pure read behind a write action would let
|
|
285
|
+
* memory-write policies silently grant history reads AND lock recall out of
|
|
286
|
+
* read-only rule sets. With no `path` arg, describeResource() falls back to the
|
|
287
|
+
* tool name, so policy can target `file.read recall` precisely; deny-by-default
|
|
288
|
+
* still applies when no rule matches (core/tools.ts evaluatePermissions). */
|
|
289
|
+
export function recallTool(sessionsRoot: string, opts: RecallToolOptions = {}): Tool {
|
|
290
|
+
const index = new RecallIndex(sessionsRoot);
|
|
291
|
+
const ceiling = Math.max(1, opts.maxResults ?? MAX_RESULTS);
|
|
292
|
+
return {
|
|
293
|
+
schema: {
|
|
294
|
+
name: "recall",
|
|
295
|
+
description:
|
|
296
|
+
"Search past session transcripts (cross-session recall). Full-text over prior conversation " +
|
|
297
|
+
"message text — no LLM. Terms are ANDed; exact word matches rank above partial (substring) " +
|
|
298
|
+
`matches. Returns up to ${ceiling} hits: sessionId, entryId, timestamp, and a short preview. ` +
|
|
299
|
+
"Use for questions about past conversations: 'what did we decide about X', 'where did we leave Y'.",
|
|
300
|
+
args: {
|
|
301
|
+
type: "object",
|
|
302
|
+
properties: {
|
|
303
|
+
query: { type: "string", description: "words to find in past sessions" },
|
|
304
|
+
limit: { type: "integer", description: `max hits (default ${Math.min(DEFAULT_RESULTS, ceiling)}, max ${ceiling})` },
|
|
305
|
+
},
|
|
306
|
+
required: ["query"],
|
|
307
|
+
},
|
|
308
|
+
},
|
|
309
|
+
kind: "read",
|
|
310
|
+
sequential: false, // pure read: safe to run concurrently with sibling reads
|
|
311
|
+
async execute(args: unknown, ctx: ToolContext): Promise<ToolOutput> {
|
|
312
|
+
// Defense in depth: keep ONLY schema args (query, limit) — smuggled keys,
|
|
313
|
+
// notably `path`, must never influence behavior. core/tools.ts
|
|
314
|
+
// describeResource() prefers an args `path` over the tool-name fallback, so
|
|
315
|
+
// {query, path:"/x"} re-aims a `file.read recall` deny rule at "/x"; policy
|
|
316
|
+
// runs BEFORE execute, so this strip cannot repair that gate — the
|
|
317
|
+
// authoritative fix belongs in describeResource (validate against the tool
|
|
318
|
+
// schema). Residual gap documented in recall.test.ts. A fresh object (not
|
|
319
|
+
// deletes on `args`) because the registry reuses the caller's object for
|
|
320
|
+
// loop-guard identity and onToolResult after execute.
|
|
321
|
+
const raw = (args && typeof args === "object" ? args : {}) as Record<string, unknown>;
|
|
322
|
+
const a: { query?: unknown; limit?: unknown } = { query: raw.query, limit: raw.limit };
|
|
323
|
+
if (typeof a.query !== "string" || a.query.trim().length === 0) {
|
|
324
|
+
return { ok: false, output: "recall failed: query must be a non-empty string" };
|
|
325
|
+
}
|
|
326
|
+
// the echoed query is bounded like the searched one: a 200k-char query must
|
|
327
|
+
// not reflect 200k chars into tool output (truncation marked with an ellipsis)
|
|
328
|
+
const trimmed = a.query.trim();
|
|
329
|
+
const echo = trimmed.length > MAX_QUERY_CHARS
|
|
330
|
+
? trimOrphanSurrogates(trimmed.slice(0, MAX_QUERY_CHARS)) + "…" : trimmed;
|
|
331
|
+
// hermes limit clamp: max(1, min(limit, ceiling)) (session_search_tool.py:1040-1046)
|
|
332
|
+
let limit = Math.min(DEFAULT_RESULTS, ceiling);
|
|
333
|
+
if (typeof a.limit === "number" && Number.isFinite(a.limit)) limit = Math.trunc(a.limit);
|
|
334
|
+
limit = Math.max(1, Math.min(limit, ceiling));
|
|
335
|
+
|
|
336
|
+
const hits = index.search(a.query, limit, ctx.sessionId);
|
|
337
|
+
if (hits.length === 0) {
|
|
338
|
+
// actionable empty message, hermes session_search_tool.py:806-810
|
|
339
|
+
return { ok: true, output: `recall: no matches for "${echo}" — terms are ANDed; try fewer or broader terms`, data: { hits } };
|
|
340
|
+
}
|
|
341
|
+
const lines = hits.map((h) =>
|
|
342
|
+
`- [${h.sessionId}] entry ${h.entryId} @ ${h.timestamp > 0 ? new Date(h.timestamp).toISOString() : "unknown time"}\n ${h.preview}`);
|
|
343
|
+
let output = `recall: ${hits.length} hit(s) for "${echo}"\n` + lines.join("\n");
|
|
344
|
+
if (opts.summarize) {
|
|
345
|
+
try {
|
|
346
|
+
const summary = await opts.summarize(a.query, hits);
|
|
347
|
+
if (summary) output += `\n\nsummary: ${summary}`;
|
|
348
|
+
} catch (e) {
|
|
349
|
+
output += `\n\n(summarize step failed: ${e instanceof Error ? e.message : String(e)}; hits above are unaffected)`;
|
|
350
|
+
}
|
|
351
|
+
}
|
|
352
|
+
return { ok: true, output, data: { hits } };
|
|
353
|
+
},
|
|
354
|
+
};
|
|
355
|
+
}
|