typeclaw 0.45.1 → 0.46.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +3 -2
- package/src/agent/git-nudge.ts +2 -1
- package/src/agent/index.ts +44 -30
- package/src/agent/live-subagents.ts +18 -0
- package/src/agent/multimodal/index.ts +1 -1
- package/src/agent/multimodal/look-at.ts +78 -44
- package/src/agent/multimodal/looker.ts +161 -39
- package/src/agent/network/response-body.ts +26 -0
- package/src/agent/network/safe-http.ts +207 -0
- package/src/agent/plugin-tools.ts +377 -157
- package/src/agent/provider-error.ts +52 -1
- package/src/agent/restart/index.ts +61 -40
- package/src/agent/restart-handoff/index.ts +73 -4
- package/src/agent/reviewer-bash-policy.ts +42 -1
- package/src/agent/session-origin.ts +2 -0
- package/src/agent/subagent-drain.ts +117 -7
- package/src/agent/subagents.ts +49 -16
- package/src/agent/throttle-circuit.ts +25 -5
- package/src/agent/tool-error-remediation.ts +72 -0
- package/src/agent/tool-file-safety.ts +1310 -0
- package/src/agent/tools/channel-fetch-attachment.ts +10 -3
- package/src/agent/tools/ddg.ts +16 -2
- package/src/agent/tools/post-github-review.ts +199 -0
- package/src/agent/tools/spawn-subagent.ts +21 -18
- package/src/agent/tools/webfetch/fetch.ts +163 -183
- package/src/agent/tools/webfetch/tool.ts +2 -3
- package/src/agent/tools/websearch.ts +23 -2
- package/src/agent/tools-without-local-file-operands.ts +30 -0
- package/src/agent/turn-runner.ts +86 -6
- package/src/bundled-plugins/backup/README.md +23 -10
- package/src/bundled-plugins/backup/git-auth.ts +6 -4
- package/src/bundled-plugins/backup/index.ts +85 -44
- package/src/bundled-plugins/backup/runner.ts +85 -1
- package/src/bundled-plugins/doc-render/render.ts +93 -12
- package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
- package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
- package/src/bundled-plugins/github-cli-auth/git-askpass.ts +74 -26
- package/src/bundled-plugins/github-cli-auth/git-command.ts +224 -36
- package/src/bundled-plugins/github-cli-auth/graphql-auth-nudge.ts +9 -9
- package/src/bundled-plugins/github-cli-auth/index.ts +207 -132
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
- package/src/bundled-plugins/github-cli-auth/token-class.ts +4 -6
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
- package/src/bundled-plugins/memory/README.md +32 -11
- package/src/bundled-plugins/memory/append-tool.ts +15 -8
- package/src/bundled-plugins/memory/citations.ts +22 -9
- package/src/bundled-plugins/memory/dreaming.ts +13 -8
- package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
- package/src/bundled-plugins/memory/index.ts +0 -4
- package/src/bundled-plugins/memory/load-memory.ts +8 -3
- package/src/bundled-plugins/memory/provenance-index.ts +717 -0
- package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
- package/src/bundled-plugins/memory/search-tool.ts +163 -60
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/vector/embed-threads.ts +36 -0
- package/src/bundled-plugins/memory/vector/embedder.ts +19 -1
- package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
- package/src/bundled-plugins/researcher/write-report.ts +8 -5
- package/src/bundled-plugins/reviewer/index.ts +2 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
- package/src/bundled-plugins/reviewer/skills/code-review.ts +19 -9
- package/src/bundled-plugins/scout/scout.ts +8 -0
- package/src/bundled-plugins/security/index.ts +4 -4
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
- package/src/bundled-plugins/security/policies/private-surface-read.ts +787 -36
- package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
- package/src/bundled-plugins/security/policies/ssrf.ts +147 -42
- package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
- package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
- package/src/channels/adapters/discord-bot.ts +66 -6
- package/src/channels/adapters/discord-channel-resolver.ts +91 -23
- package/src/channels/adapters/discord-id.ts +4 -0
- package/src/channels/adapters/discord-resolver-cache.ts +41 -0
- package/src/channels/adapters/discord.ts +57 -7
- package/src/channels/adapters/github/auth-app.ts +41 -12
- package/src/channels/adapters/github/inbound.ts +11 -0
- package/src/channels/adapters/github/index.ts +71 -65
- package/src/channels/adapters/github/reconcile-cooldown-store.ts +178 -0
- package/src/channels/adapters/github/reconcile-open-prs.ts +31 -0
- package/src/channels/adapters/github/review-submitter.ts +302 -0
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +5 -3
- package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
- package/src/channels/adapters/kakaotalk-typing.ts +17 -15
- package/src/channels/adapters/slack-attachment-download.ts +71 -0
- package/src/channels/adapters/slack-bot-edit.ts +85 -20
- package/src/channels/adapters/slack-bot.ts +26 -10
- package/src/channels/adapters/slack-classify.ts +21 -2
- package/src/channels/adapters/slack-format.ts +245 -0
- package/src/channels/adapters/slack.ts +59 -7
- package/src/channels/adapters/telegram-bot.ts +18 -5
- package/src/channels/adapters/webex-bot.ts +8 -3
- package/src/channels/adapters/webex.ts +8 -3
- package/src/channels/fetch-attachment.ts +43 -0
- package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
- package/src/channels/github-token-bridge.ts +12 -6
- package/src/channels/manager.ts +9 -0
- package/src/channels/router.ts +248 -6
- package/src/channels/types.ts +28 -1
- package/src/cli/channel.ts +1 -1
- package/src/cli/init.ts +1 -1
- package/src/cli/model.ts +57 -7
- package/src/compose/stop.ts +17 -5
- package/src/config/config.ts +6 -6
- package/src/config/models-mutation.ts +13 -1
- package/src/config/providers.ts +176 -0
- package/src/container/docker-app.ts +42 -13
- package/src/container/start.ts +62 -10
- package/src/cron/bridge.ts +9 -2
- package/src/doctor/checks.ts +8 -3
- package/src/doctor/commit.ts +2 -1
- package/src/doctor/file-ownership.ts +52 -0
- package/src/dreams/git.ts +2 -1
- package/src/git/bound-gc-config.ts +82 -0
- package/src/git/hookless.ts +5 -0
- package/src/git/reconcile-ignored.ts +18 -5
- package/src/git/system-commit.ts +12 -6
- package/src/hostd/spawn.ts +133 -47
- package/src/init/dockerfile.ts +99 -5
- package/src/init/index.ts +17 -5
- package/src/init/models-dev.ts +4 -0
- package/src/init/validate-api-key.ts +2 -0
- package/src/inspect/replay.ts +21 -9
- package/src/inspect/session-list.ts +71 -33
- package/src/mcp/tools.ts +17 -3
- package/src/network/internal-destinations.ts +267 -0
- package/src/permissions/builtins.ts +3 -1
- package/src/plugin/context.ts +2 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +4 -0
- package/src/plugin/types.ts +22 -0
- package/src/run/index.ts +40 -0
- package/src/run/llm-fetch-observer.ts +41 -4
- package/src/sandbox/build.ts +37 -15
- package/src/sandbox/canonical-secrets.ts +18 -0
- package/src/sandbox/env-exposure.ts +83 -0
- package/src/sandbox/errors.ts +13 -6
- package/src/sandbox/hidden-paths.ts +102 -44
- package/src/sandbox/index.ts +22 -4
- package/src/sandbox/package-install.ts +3 -27
- package/src/sandbox/policy.ts +16 -28
- package/src/sandbox/privileged-runtime.ts +256 -0
- package/src/sandbox/session-tmp.ts +8 -2
- package/src/sandbox/writable-zones.ts +204 -140
- package/src/secrets/index.ts +8 -0
- package/src/secrets/refresh-provider-oauth.ts +128 -0
- package/src/server/command-runner.ts +48 -4
- package/src/server/index.ts +4 -0
- package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
- package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
- package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
- package/src/skills/typeclaw-config/SKILL.md +27 -11
- package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
- package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
- package/src/skills/typeclaw-memory/SKILL.md +9 -3
- package/src/skills/typeclaw-permissions/SKILL.md +36 -19
- package/src/skills/typeclaw-plugins/SKILL.md +37 -0
- package/src/usage/scan.ts +11 -2
- package/typeclaw.schema.json +25 -0
|
@@ -30,16 +30,28 @@ All fields are **restart-required** — the plugin reads them once at boot. The
|
|
|
30
30
|
|
|
31
31
|
## What it contributes
|
|
32
32
|
|
|
33
|
-
| Kind | Name | Notes
|
|
34
|
-
| -------- | -------------------------- |
|
|
35
|
-
| Subagent | `memory-logger` | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`.
|
|
36
|
-
| Subagent | `dreaming` | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run.
|
|
37
|
-
| Tool | `memory_search` | Main-agent tool. Pass `query`
|
|
38
|
-
| Tool | `delete_topic_shard` | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded.
|
|
39
|
-
| Cron | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`.
|
|
40
|
-
| Hook | `session.idle` | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip.
|
|
41
|
-
| Hook | `session.end` | Spawns `memory-logger` immediately and clears per-session retrieval de-dup state.
|
|
42
|
-
| Hook | `session.turn.start` | Renders the per-turn user-prompt memory block into `event.retrievalContext.results` (direct-mode shards under budget, de-duplicated across turns per session; top-K `hybridSearch` over budget; channel origins always index/headings-only); runs for every origin that supplies a `retrievalContext`, including user-delegated subagents (researcher, reviewer) — but NOT TypeClaw-owned infrastructure turns (`system` origins and `system`-spawned subagents memory-logger/backup, plus the dreaming cron job matched by its `__plugin_memory_dreaming` job id), whose `userPrompt` is a fixed instruction block, not a user message (see [Internal-infrastructure turns skip retrieval](#internal-infrastructure-turns-skip-retrieval)).
|
|
33
|
+
| Kind | Name | Notes |
|
|
34
|
+
| -------- | -------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
35
|
+
| Subagent | `memory-logger` | Reads a parent transcript past a watermark and appends fragments to `memory/streams/<today>.jsonl`. Coalesced per `agentDir`. |
|
|
36
|
+
| Subagent | `dreaming` | Reads shards under `memory/topics/` plus undreamed daily-stream events and rebalances the topic shards. Coalesced per `agentDir`. Citation-superset invariant enforced on every run. |
|
|
37
|
+
| Tool | `memory_search` | Main-agent tool. Pass `query` for phrase-first/token-fallback lexical search, or `topic` for exact slug lookup (not both). Searches topic shards, active cited-child provenance, undreamed streams, and references. Optional `workspace`, `chat`, and `thread` scopes apply to both query and exact-topic mode; legacy `where` aliases `chat`. Results include eligible active-child provenance. See [Provenance-aware recall](#provenance-aware-recall). |
|
|
38
|
+
| Tool | `delete_topic_shard` | Subagent-only (dreaming). Deletes a topic shard at `memory/topics/<slug>.md`. Path-guarded. |
|
|
39
|
+
| Cron | `__plugin_memory_dreaming` | `kind: 'prompt'`, `subagent: 'dreaming'`, scheduled per `memory.dreaming.schedule`. |
|
|
40
|
+
| Hook | `session.idle` | Per-session debouncer with size-based ceiling. Spawns `memory-logger` on idle or buffer-trip. |
|
|
41
|
+
| Hook | `session.end` | Spawns `memory-logger` immediately and clears per-session retrieval de-dup state. |
|
|
42
|
+
| Hook | `session.turn.start` | Renders the per-turn user-prompt memory block into `event.retrievalContext.results` (direct-mode shards under budget, de-duplicated across turns per session; top-K `hybridSearch` over budget; channel origins always index/headings-only); runs for every origin that supplies a `retrievalContext`, including user-delegated subagents (researcher, reviewer) — but NOT TypeClaw-owned infrastructure turns (`system` origins and `system`-spawned subagents memory-logger/backup, plus the dreaming cron job matched by its `__plugin_memory_dreaming` job id), whose `userPrompt` is a fixed instruction block, not a user message (see [Internal-infrastructure turns skip retrieval](#internal-infrastructure-turns-skip-retrieval)). |
|
|
43
|
+
|
|
44
|
+
## Provenance-aware recall
|
|
45
|
+
|
|
46
|
+
Fragments can carry runtime-stamped `where` coordinates (`adapter`, workspace/chat/thread IDs, captured names, and Discord parent-room metadata) plus an attributed `who`. A topic derives searchable provenance only from its **active** `fragments:` children. `superseded:` evidence is excluded, and unresolved citations remain explicit rather than receiving invented metadata. The same eligible children determine scope eligibility, lexical matching, and returned `provenance`, so a multi-workspace topic cannot return another workspace's child details through a scoped result.
|
|
47
|
+
|
|
48
|
+
`memory_search` accepts optional `workspace`, `chat`, and `thread` filters in query and exact-topic modes (`where` remains a `chat` alias). Matching is case-insensitive and accepts an optional leading `#` for names. Recall is globally visible by default across roles and origins. Provenance coordinates and captured names are additive lexical metadata rather than authorization inputs, and automatic hybrid retrieval or channel heading injection never derives a hidden scope from session ancestry.
|
|
49
|
+
|
|
50
|
+
When explicit filters are supplied, a topic remains eligible when any resolved active child matches every requested coordinate, and returned provenance contains only matching children. Undreamed streams use the same filters. References are excluded from scoped results because they have no workspace provenance; unscoped recall continues to include them.
|
|
51
|
+
|
|
52
|
+
Provenance participates only in lexical documents. It is never appended to `fragmentEmbeddableText` or topic vector passages, so origin changes cannot alter vector scores or consume the embedding budget. `hybridSearch` loads shards and streams once per call, derives the undreamed view from that corpus, and builds provenance from the same loaded data.
|
|
53
|
+
|
|
54
|
+
Historical stream JSONL stays append-only. Both Discord adapters run bounded, timeout-safe startup maintenance that resolves old ID-only guild/channel/thread coordinates into a process-local, bounded enrichment registry. The registry is rebuilt on every startup and never written to disk, eliminating sidecar path races entirely. Confirmed non-thread checks remain negatively cached for the process lifetime so they do not retry or starve the candidate batch. Names containing secrets, controls, bidi overrides, zero-width characters, Markdown prompt-shaping syntax, or excessive length are rejected. Resolver failures and timeouts degrade to raw-ID search. Search itself only reads local memory plus the in-memory registry and never performs network resolution.
|
|
43
55
|
|
|
44
56
|
## Memory injection (two-tier, topic shards only)
|
|
45
57
|
|
|
@@ -125,7 +137,7 @@ A `[dreaming] citation-superset violation: …` warning logs the dropped ids and
|
|
|
125
137
|
## Files on disk
|
|
126
138
|
|
|
127
139
|
- **`memory/topics/<slug>.md`** — per-topic shards with YAML frontmatter (`heading`, `cites`, `days`, `lastReinforced`, `tags?`) + body markdown. Runtime owns the frontmatter (recomputed after every dreaming run from the body's citations); dreaming subagent writes body only. The body is a **compact belief record**: one belief sentence stating the current truth (subject + predicate + essential scope qualifier; the strength wording — "mentioned" / "consistently" / "always" — is calibrated from `days`), followed by the `fragments:` list (active evidence behind the belief) and an optional `superseded:` list (evidence overturned by a later contradiction — kept cited for GC/history but excluded from vector retrieval). One sentence (not a paragraph) keeps bodies small so more topics stay under the direct-injection budget; the citation lists are the only load-bearing part of the body, so the prose shape is free to be terse. Existing verbose shards are rewritten into this form by the next dreaming run (no migration).
|
|
128
|
-
- **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards. A `fragment` event carries optional **situational provenance** for "who said what, where, and when": `who` (display name/handle of the attributed speaker, LLM-supplied per-fragment by `memory-logger` only when one transcript speaker clearly owns the evidence), `where` (`{ adapter, workspace, workspaceName?, chat, chatName?, thread }` — runtime-stamped from the session origin in the append tool, channel origins only), and `when` (reuses the existing `ts`). Both are optional, so legacy fragments and non-channel (TUI) origins parse with them absent. `who`/`where` are deliberately NOT in `fragmentContentHash` (dedup stays content-identity) and NOT in `fragmentEmbeddableText` (provenance is retrieval metadata, not embedded — it must not burn the 512-token budget or let same-room/same-person dominate recall). **Secret-scanning covers provenance too**: `who` (LLM-supplied) clears the same `detectSecrets` guard as `topic`/`body` and a credential-shaped value is refused with the retryable error; the origin-derived `where` names (`workspaceName`/`chatName`) are external channel data the agent can't rewrite, so a credential-shaped name is silently dropped (raw id kept) rather than wedging the logger. Raw ids are platform-issued identifiers, not free text, so they are not scanned. `where` is runtime-stamped rather than LLM-supplied because the channel is stable for a whole logger run; `who` is per-fragment because a single run spans many speakers and `origin.lastInboundAuthorId` is only the spawn-time one — auto-stamping it would misattribute fragments about earlier speakers. Surfaced to the agent (not embedded) via `memory_search` (`StreamMatch.who/when/where`), `hybridSearch` (`HybridSearchResult.who/when/where` on self-resolved undreamed fragments), and the per-turn retrieval render (`renderProvenanceLine` → `_Jisoo in #incidents on 2026-06-24_`).
|
|
140
|
+
- **`memory/streams/yyyy-MM-dd.jsonl`** — daily fragment streams. One event per line, discriminated union of `fragment | watermark | legacy_prose`. Force-committed alongside the shards. A `fragment` event carries optional **situational provenance** for "who said what, where, and when": `who` (display name/handle of the attributed speaker, LLM-supplied per-fragment by `memory-logger` only when one transcript speaker clearly owns the evidence), `where` (`{ adapter, workspace, workspaceName?, chat, chatName?, thread, parentChat?, parentChatName? }` — runtime-stamped from the session origin in the append tool, channel origins only), and `when` (reuses the existing `ts`). Both are optional, so legacy fragments and non-channel (TUI) origins parse with them absent. `who`/`where` are deliberately NOT in `fragmentContentHash` (dedup stays content-identity) and NOT in `fragmentEmbeddableText` (provenance is retrieval metadata, not embedded — it must not burn the 512-token budget or let same-room/same-person dominate recall). **Secret-scanning covers provenance too**: `who` (LLM-supplied) clears the same `detectSecrets` guard as `topic`/`body` and a credential-shaped value is refused with the retryable error; the origin-derived `where` names (`workspaceName`/`chatName`/`parentChatName`) are external channel data the agent can't rewrite, so a credential-shaped, overlong, or control-character-bearing name is silently dropped (raw id kept) rather than wedging the logger. Raw ids are platform-issued identifiers, not free text, so they are not scanned. `where` is runtime-stamped rather than LLM-supplied because the channel is stable for a whole logger run; `who` is per-fragment because a single run spans many speakers and `origin.lastInboundAuthorId` is only the spawn-time one — auto-stamping it would misattribute fragments about earlier speakers. Surfaced to the agent (not embedded) via `memory_search` (`StreamMatch.who/when/where`), `hybridSearch` (`HybridSearchResult.who/when/where` on self-resolved undreamed fragments), and the per-turn retrieval render (`renderProvenanceLine` → `_Jisoo in #incidents on 2026-06-24_`).
|
|
129
141
|
- **`memory/MEMORY.md.pre-shard.bak`** — legacy pre-shard backup left by older TypeClaw versions. Safe to delete after verifying.
|
|
130
142
|
- **`memory/skills/<name>/SKILL.md`** — muscle memory (Form A). Skills the dreaming subagent distills from repeated procedures. Auto-loaded as first-class skills. CLI/plugin suggestions (Forms B and C) are recorded as topic shards instead — see [Muscle memory](#muscle-memory-three-forms).
|
|
131
143
|
- **`memory/references/<slug>.md`** — verbatim reference artifacts. Written by `store_reference`, a `memory-logger` custom tool that is always registered (there is no `references.enabled` config gate — references are unconditionally available; they simply stay empty until the `memory-logger` chooses to store one). A fragment may cite a stored reference via `append`'s `references: ['<slug>']`, but the append tool validates each slug against the references that actually exist on disk and silently drops the rest (`resolveValidReferences`), so a hallucinated slug never lands a dangling citation in the stream. A freshly stored reference is embedded into the vector index immediately via the `store_reference` on-write hook (`vector/reference-index-on-write.ts`), mirroring the stream-fragment on-write hook — so it is vector-retrievable within the same container uptime instead of only after the next startup index build. Chunks are derived by the same `referencePassagesForOne` the startup build uses, and re-storing a shorter body prunes the stale higher-index chunk rows.
|
|
@@ -148,6 +160,15 @@ The embedding model (`Xenova/multilingual-e5-base`) caps inputs at **512 tokens*
|
|
|
148
160
|
1. **Deterministic bound at embed time (primary, always on).** `embed()` calls `boundEmbeddableText` on every input before the tokenizer sees it, trimming to a character budget derived from the same conservative per-script token estimate (CJK counted 1:1, Latin ~3.5 chars/token; the tokenizer is never loaded, preserving the embedder's lazy-load). The cut is now explicit and owned, not a hidden tokenizer side effect, and the leading heading/belief sentence — the load-bearing retrieval signal — always survives because it comes first. This guarantees no silent loss even for inputs nothing ever rewrites (raw legacy prose, long queries).
|
|
149
161
|
2. **Compaction routed to dreaming (remediation).** Bounding keeps retrieval correct but the bounded tail is still not embedded. The real fix is to make the shard fit. On each run the dreaming handler computes which topic shards are over budget (measuring `topicPassage(...).text` — the same citation-stripped string the embedder bounds — so the flag matches what is actually truncated) and renders an "Over the embedding budget" table into the subagent's prompt, alongside the strength-signal table. Dreaming rule 8 instructs it to compact those shards into the one-belief-sentence form **while preserving every `fragments:`/`superseded:` citation** — shrinking prose, never dropping a citation (a dropped citation would trip the citation-superset revert, leaving the shard both reverted and still over budget). The signal is gated on the vector index (`memory/.vectors/index.db`) existing; before the first index exists, the table is suppressed. The `over_budget=N` count rides the `[dreaming] done` line.
|
|
150
162
|
|
|
163
|
+
## Embedder resource tuning
|
|
164
|
+
|
|
165
|
+
The embedding model is a single onnxruntime-node session shared process-wide (`vector/embedder.ts`, `getEmbedder()` singleton). Two session options bound its memory footprint, since the container runs with no docker `--cpus` limit:
|
|
166
|
+
|
|
167
|
+
- **Adaptive intra-op threads** (`vector/embed-threads.ts`). ORT's default (`intraOpNumThreads: 0`) sizes the shared session's intra-op thread pool to one worker per visible host core, each holding its own activation workspace for the singleton's lifetime. `resolveEmbedThreadCount()` caps this at half the visible cores, clamped to `[1, 4]`. The pool is session-level and shared — the transformers.js Node path calls `session.run()` directly (concurrent `Run()` is allowed; the serialization only applies to the web backend), so concurrent embeds contend for it. Capping trades some potential intra-op parallelism (per embed and across overlapping embeds) for RSS bounded by the cap rather than by host core count; for this workload of short per-turn query embeds that trade is worth it.
|
|
168
|
+
- **CPU mem arena off** (`enableCpuMemArena: false`). ORT's arena never returns memory to the OS; for the intermittent per-turn query embed it would inflate steady-state RSS. Disabling frees per-embed at a negligible per-call malloc cost.
|
|
169
|
+
|
|
170
|
+
Neither is configurable — the adaptive default is intended to be correct on any host.
|
|
171
|
+
|
|
151
172
|
## How `session.idle` works
|
|
152
173
|
|
|
153
174
|
Core fires `session.idle` immediately after every `session.prompt()` completion. The plugin owns the debounce: a `Map<sessionId, Timeout>` reset on every event. When the timer fires, the plugin spawns `memory-logger` for that session — unless the min-delta gate suppresses the spawn (see below).
|
|
@@ -9,6 +9,7 @@ import { formatLocalDate } from '@/shared'
|
|
|
9
9
|
|
|
10
10
|
import { fragmentContentHash } from './fragment-parser'
|
|
11
11
|
import { streamFilePath } from './paths'
|
|
12
|
+
import { sanitizeProvenanceName } from './provenance-sanitize'
|
|
12
13
|
import { detectSecrets } from './secret-detector'
|
|
13
14
|
import { newEventId, timestampFromId } from './stream-events'
|
|
14
15
|
import type { FragmentEvent, FragmentProvenance, WatermarkEvent } from './stream-events'
|
|
@@ -52,18 +53,16 @@ export function provenanceFromOrigin(origin: SessionOrigin | undefined): Fragmen
|
|
|
52
53
|
// so a credential-shaped name is dropped rather than thrown — keep the raw id,
|
|
53
54
|
// never let the leaky display string reach git. Raw ids are platform-issued
|
|
54
55
|
// identifiers, not free text, so they are not scanned.
|
|
55
|
-
const workspaceName =
|
|
56
|
-
const chatName =
|
|
56
|
+
const workspaceName = sanitizeProvenanceName(origin.workspaceName)
|
|
57
|
+
const chatName = sanitizeProvenanceName(origin.chatName)
|
|
58
|
+
const parentChatName = sanitizeProvenanceName(origin.parentChatName)
|
|
57
59
|
if (workspaceName !== undefined) where.workspaceName = workspaceName
|
|
58
60
|
if (chatName !== undefined) where.chatName = chatName
|
|
61
|
+
if (origin.parentChat !== undefined) where.parentChat = origin.parentChat
|
|
62
|
+
if (parentChatName !== undefined) where.parentChatName = parentChatName
|
|
59
63
|
return where
|
|
60
64
|
}
|
|
61
65
|
|
|
62
|
-
function redactIfSecret(name: string | undefined): string | undefined {
|
|
63
|
-
if (name === undefined) return undefined
|
|
64
|
-
return detectSecrets(name).length === 0 ? name : undefined
|
|
65
|
-
}
|
|
66
|
-
|
|
67
66
|
export function createAppendTool(options: CreateAppendToolOptions = {}) {
|
|
68
67
|
const { onFragmentsAppended, originProvider, referenceSlugResolver } = options
|
|
69
68
|
return defineTool({
|
|
@@ -78,6 +77,13 @@ export function createAppendTool(options: CreateAppendToolOptions = {}) {
|
|
|
78
77
|
references: z.array(z.string()).optional(),
|
|
79
78
|
who: z.string().min(1).optional(),
|
|
80
79
|
}),
|
|
80
|
+
// None of these is a local path — topic/body are prose, the rest are session/
|
|
81
|
+
// entry/reference ids. The runtime chooses the stream filename; the tool never
|
|
82
|
+
// opens a caller path. Declared so a body carrying "/" or a slug/id that
|
|
83
|
+
// collides with an agent-root dir name is not misread as a file operand.
|
|
84
|
+
fileOperands: {
|
|
85
|
+
nonFile: ['topic', 'body', 'source', 'entry', 'latestEntryId', 'references', 'who'],
|
|
86
|
+
},
|
|
81
87
|
async execute({ topic, body, source, entry, latestEntryId, references, who }, ctx) {
|
|
82
88
|
const streamPath = dailyStreamPath(ctx.agentDir)
|
|
83
89
|
const where = provenanceFromOrigin(originProvider?.())
|
|
@@ -116,7 +122,8 @@ export function createAppendTool(options: CreateAppendToolOptions = {}) {
|
|
|
116
122
|
if (validReferences.length > 0) {
|
|
117
123
|
fragment.references = validReferences
|
|
118
124
|
}
|
|
119
|
-
|
|
125
|
+
const safeWho = sanitizeProvenanceName(who)
|
|
126
|
+
if (safeWho !== undefined) fragment.who = safeWho
|
|
120
127
|
if (where !== undefined) fragment.where = where
|
|
121
128
|
const watermark: WatermarkEvent = {
|
|
122
129
|
type: 'watermark',
|
|
@@ -87,9 +87,11 @@ const SECTION_HEADING = /^[\s-]*(fragments|superseded)\s*:\s*$/i
|
|
|
87
87
|
|
|
88
88
|
export type SectionedCitations = { active: Set<string>; superseded: Set<string> }
|
|
89
89
|
|
|
90
|
-
export
|
|
91
|
-
|
|
92
|
-
|
|
90
|
+
export type SectionedCitationRefs = { active: Citation[]; superseded: Citation[] }
|
|
91
|
+
|
|
92
|
+
export function splitCitationRefsBySection(body: string): SectionedCitationRefs {
|
|
93
|
+
const active = new Map<string, Citation>()
|
|
94
|
+
const superseded = new Map<string, Citation>()
|
|
93
95
|
let current: 'active' | 'superseded' = 'active'
|
|
94
96
|
|
|
95
97
|
for (const line of body.split('\n')) {
|
|
@@ -98,13 +100,24 @@ export function splitCitationsBySection(body: string): SectionedCitations {
|
|
|
98
100
|
current = heading[1]!.toLowerCase() === 'superseded' ? 'superseded' : 'active'
|
|
99
101
|
continue
|
|
100
102
|
}
|
|
101
|
-
const
|
|
102
|
-
if (
|
|
103
|
-
|
|
103
|
+
const match = CITATION_LINE.exec(line)
|
|
104
|
+
if (match === null) continue
|
|
105
|
+
const citation = { date: match[2]!, fragmentId: match[3]! }
|
|
106
|
+
const canonical = formatCitation(citation.date, citation.fragmentId)
|
|
107
|
+
;(current === 'superseded' ? superseded : active).set(canonical, citation)
|
|
104
108
|
}
|
|
105
109
|
|
|
106
|
-
|
|
107
|
-
for (const
|
|
110
|
+
const activeIds = new Set([...active.values()].map((citation) => citation.fragmentId))
|
|
111
|
+
for (const [canonical, citation] of superseded) {
|
|
112
|
+
if (activeIds.has(citation.fragmentId)) superseded.delete(canonical)
|
|
113
|
+
}
|
|
114
|
+
return { active: [...active.values()], superseded: [...superseded.values()] }
|
|
115
|
+
}
|
|
108
116
|
|
|
109
|
-
|
|
117
|
+
export function splitCitationsBySection(body: string): SectionedCitations {
|
|
118
|
+
const refs = splitCitationRefsBySection(body)
|
|
119
|
+
return {
|
|
120
|
+
active: new Set(refs.active.map((citation) => citation.fragmentId)),
|
|
121
|
+
superseded: new Set(refs.superseded.map((citation) => citation.fragmentId)),
|
|
122
|
+
}
|
|
110
123
|
}
|
|
@@ -5,6 +5,7 @@ import { basename, join } from 'node:path'
|
|
|
5
5
|
|
|
6
6
|
import { z } from 'zod'
|
|
7
7
|
|
|
8
|
+
import { hooklessGitArgs } from '@/git/hookless'
|
|
8
9
|
import { withGitLock } from '@/git/mutex'
|
|
9
10
|
import { type AgentGit, resolveAgentGit } from '@/git/resolve-agent-git'
|
|
10
11
|
import { defineTool, lsTool, readTool, type Subagent, writeTool } from '@/plugin'
|
|
@@ -773,7 +774,7 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
773
774
|
}
|
|
774
775
|
|
|
775
776
|
const add = bun.spawn({
|
|
776
|
-
cmd: ['git', ...repo.gitArgs, 'add', '-f', '--', ...presentPaths],
|
|
777
|
+
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'add', '-f', '--', ...presentPaths])],
|
|
777
778
|
cwd,
|
|
778
779
|
stdout: 'pipe',
|
|
779
780
|
stderr: 'pipe',
|
|
@@ -788,7 +789,10 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
788
789
|
// fails outright when `bar/` matches no tracked file, even if `foo` is
|
|
789
790
|
// staged.
|
|
790
791
|
const stagedNames = bun.spawn({
|
|
791
|
-
cmd: [
|
|
792
|
+
cmd: [
|
|
793
|
+
'git',
|
|
794
|
+
...hooklessGitArgs([...repo.gitArgs, 'diff', '--cached', '--name-only', '-z', '--', ...SNAPSHOT_PATHS]),
|
|
795
|
+
],
|
|
792
796
|
cwd,
|
|
793
797
|
stdout: 'pipe',
|
|
794
798
|
stderr: 'pipe',
|
|
@@ -807,7 +811,7 @@ async function commitMemorySnapshotUnlocked(cwd: string): Promise<void> {
|
|
|
807
811
|
const message = await buildCommitMessage(bun, cwd, staged, undefined, repo.gitArgs)
|
|
808
812
|
|
|
809
813
|
const commit = bun.spawn({
|
|
810
|
-
cmd: ['git', ...repo.gitArgs, 'commit', '-m', message, '--only', '--', ...staged],
|
|
814
|
+
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'commit', '-m', message, '--only', '--', ...staged])],
|
|
811
815
|
cwd,
|
|
812
816
|
stdout: 'pipe',
|
|
813
817
|
stderr: 'pipe',
|
|
@@ -864,7 +868,7 @@ async function buildDreamSummary(
|
|
|
864
868
|
// numstat: `<added>\t<deleted>\t<path>` per line. Use NUL-terminated so paths
|
|
865
869
|
// with whitespace round-trip; -z switches the record separator to NUL.
|
|
866
870
|
const numstat = bun.spawn({
|
|
867
|
-
cmd: ['git', ...gitArgs, 'diff', '--cached', '--numstat', '-z', '--', ...staged],
|
|
871
|
+
cmd: ['git', ...hooklessGitArgs([...gitArgs, 'diff', '--cached', '--numstat', '-z', '--', ...staged])],
|
|
868
872
|
cwd,
|
|
869
873
|
stdout: 'pipe',
|
|
870
874
|
stderr: 'pipe',
|
|
@@ -921,7 +925,7 @@ async function listNewlyAddedSkills(
|
|
|
921
925
|
gitArgs: readonly string[] = [],
|
|
922
926
|
): Promise<string[]> {
|
|
923
927
|
const proc = bun.spawn({
|
|
924
|
-
cmd: ['git', ...gitArgs, 'diff', '--cached', '--name-status', '-z', '--', ...staged],
|
|
928
|
+
cmd: ['git', ...hooklessGitArgs([...gitArgs, 'diff', '--cached', '--name-status', '-z', '--', ...staged])],
|
|
925
929
|
cwd,
|
|
926
930
|
stdout: 'pipe',
|
|
927
931
|
stderr: 'pipe',
|
|
@@ -949,7 +953,7 @@ async function listTrackedSnapshotFiles(
|
|
|
949
953
|
repo: AgentGit,
|
|
950
954
|
): Promise<string[]> {
|
|
951
955
|
const ls = bun.spawn({
|
|
952
|
-
cmd: ['git', ...repo.gitArgs, 'ls-files', '-z', '--', ...SNAPSHOT_PATHS],
|
|
956
|
+
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'ls-files', '-z', '--', ...SNAPSHOT_PATHS])],
|
|
953
957
|
cwd,
|
|
954
958
|
stdout: 'pipe',
|
|
955
959
|
stderr: 'pipe',
|
|
@@ -963,7 +967,7 @@ async function clearSkipWorktree(bun: { spawn: typeof Bun.spawn }, cwd: string,
|
|
|
963
967
|
const files = await listTrackedSnapshotFiles(bun, cwd, repo)
|
|
964
968
|
if (files.length === 0) return
|
|
965
969
|
const proc = bun.spawn({
|
|
966
|
-
cmd: ['git', ...repo.gitArgs, 'update-index', '--no-skip-worktree', '--', ...files],
|
|
970
|
+
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'update-index', '--no-skip-worktree', '--', ...files])],
|
|
967
971
|
cwd,
|
|
968
972
|
stdout: 'pipe',
|
|
969
973
|
stderr: 'pipe',
|
|
@@ -975,7 +979,7 @@ async function applySkipWorktree(bun: { spawn: typeof Bun.spawn }, cwd: string,
|
|
|
975
979
|
const files = await listTrackedSnapshotFiles(bun, cwd, repo)
|
|
976
980
|
if (files.length === 0) return
|
|
977
981
|
const proc = bun.spawn({
|
|
978
|
-
cmd: ['git', ...repo.gitArgs, 'update-index', '--skip-worktree', '--', ...files],
|
|
982
|
+
cmd: ['git', ...hooklessGitArgs([...repo.gitArgs, 'update-index', '--skip-worktree', '--', ...files])],
|
|
979
983
|
cwd,
|
|
980
984
|
stdout: 'pipe',
|
|
981
985
|
stderr: 'pipe',
|
|
@@ -1324,6 +1328,7 @@ function escapeTableCell(value: string): string {
|
|
|
1324
1328
|
const dreamingDeleteTopicShardTool = defineTool({
|
|
1325
1329
|
description: deleteTopicShardTool.description,
|
|
1326
1330
|
parameters: deleteTopicShardTool.inputSchema,
|
|
1331
|
+
fileOperands: { destructive: ['path'] },
|
|
1327
1332
|
async execute(args, ctx) {
|
|
1328
1333
|
const result = await deleteTopicShardTool.run(args, { agentDir: ctx.agentDir })
|
|
1329
1334
|
return { content: [{ type: 'text', text: JSON.stringify(result) }] }
|
|
@@ -19,6 +19,10 @@ export const findEntryTool = defineTool({
|
|
|
19
19
|
.min(1)
|
|
20
20
|
.describe('The entry id to locate (matches the JSONL row whose own `id` field equals this value).'),
|
|
21
21
|
}),
|
|
22
|
+
// `path` is a real local file this tool reads; declaring it pins an immutable
|
|
23
|
+
// snapshot before readFile (and restores the original path in the result)
|
|
24
|
+
// instead of the scanner rejecting the bare `sessions/...jsonl` path.
|
|
25
|
+
fileOperands: { input: ['path'] },
|
|
22
26
|
async execute({ path, entryId }) {
|
|
23
27
|
if (entryId.length === 0) {
|
|
24
28
|
throw new Error('find_entry requires a non-empty entryId; an empty needle would match every line.')
|
|
@@ -717,7 +717,3 @@ async function raceSpawn(work: Promise<void>, ms: number): Promise<void> {
|
|
|
717
717
|
if (timer !== null) clearTimeout(timer)
|
|
718
718
|
}
|
|
719
719
|
}
|
|
720
|
-
|
|
721
|
-
function isEnoent(err: unknown): boolean {
|
|
722
|
-
return typeof err === 'object' && err !== null && 'code' in err && err.code === 'ENOENT'
|
|
723
|
-
}
|
|
@@ -7,6 +7,7 @@ import { firstBeliefSentence, isTitleLikeHeading } from './belief-sentence'
|
|
|
7
7
|
import { buildInjectionPlan, type InjectionPlan } from './injection-plan'
|
|
8
8
|
import { loadAllShards, type TopicShard } from './load-shards'
|
|
9
9
|
import { topicsDir } from './paths'
|
|
10
|
+
import { sanitizeProvenanceName } from './provenance-sanitize'
|
|
10
11
|
import { slugIsHeadingEcho } from './slug'
|
|
11
12
|
import type { FragmentProvenance } from './stream-events'
|
|
12
13
|
import type { DedupedRetrievedItem } from './turn-dedup'
|
|
@@ -68,9 +69,13 @@ export type RetrievedMemoryItem = {
|
|
|
68
69
|
// parts that exist are shown; an undreamed fragment with none renders nothing.
|
|
69
70
|
export function renderProvenanceLine(item: Pick<RetrievedMemoryItem, 'who' | 'when' | 'where'>): string | null {
|
|
70
71
|
const parts: string[] = []
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
72
|
+
const who = sanitizeProvenanceName(item.who)
|
|
73
|
+
if (who !== undefined) parts.push(`speaker=<untrusted-name>${who}</untrusted-name>`)
|
|
74
|
+
const roomName = sanitizeProvenanceName(item.where?.chatName)
|
|
75
|
+
const room = roomName ?? item.where?.chat
|
|
76
|
+
if (room !== undefined) {
|
|
77
|
+
parts.push(roomName === undefined ? `in ${room}` : `in #<untrusted-name>${roomName}</untrusted-name>`)
|
|
78
|
+
}
|
|
74
79
|
if (item.when !== undefined) parts.push(`on ${item.when.slice(0, 10)}`)
|
|
75
80
|
return parts.length === 0 ? null : `_${parts.join(' ')}_`
|
|
76
81
|
}
|