typeclaw 0.45.1 → 0.46.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +2 -1
- package/src/agent/git-nudge.ts +2 -1
- package/src/agent/index.ts +44 -30
- package/src/agent/live-subagents.ts +18 -0
- package/src/agent/multimodal/index.ts +1 -1
- package/src/agent/multimodal/look-at.ts +78 -44
- package/src/agent/multimodal/looker.ts +151 -39
- package/src/agent/network/response-body.ts +26 -0
- package/src/agent/network/safe-http.ts +198 -0
- package/src/agent/plugin-tools.ts +379 -157
- package/src/agent/provider-error.ts +52 -1
- package/src/agent/restart/index.ts +61 -40
- package/src/agent/restart-handoff/index.ts +73 -4
- package/src/agent/reviewer-bash-policy.ts +42 -1
- package/src/agent/session-origin.ts +2 -0
- package/src/agent/subagent-drain.ts +117 -7
- package/src/agent/subagents.ts +49 -16
- package/src/agent/throttle-circuit.ts +25 -5
- package/src/agent/tool-error-remediation.ts +72 -0
- package/src/agent/tool-file-safety.ts +1050 -0
- package/src/agent/tools/channel-fetch-attachment.ts +10 -3
- package/src/agent/tools/ddg.ts +16 -2
- package/src/agent/tools/post-github-review.ts +199 -0
- package/src/agent/tools/spawn-subagent.ts +21 -18
- package/src/agent/tools/webfetch/fetch.ts +143 -183
- package/src/agent/tools/webfetch/tool.ts +2 -3
- package/src/agent/tools/websearch.ts +23 -2
- package/src/agent/tools-without-local-file-operands.ts +27 -0
- package/src/agent/turn-runner.ts +86 -6
- package/src/bundled-plugins/backup/index.ts +2 -1
- package/src/bundled-plugins/backup/runner.ts +2 -1
- package/src/bundled-plugins/doc-render/render.ts +93 -12
- package/src/bundled-plugins/doc-render/skills/typeclaw-render-pdf/SKILL.md +13 -13
- package/src/bundled-plugins/github-cli-auth/effective-approval.ts +16 -6
- package/src/bundled-plugins/github-cli-auth/gh-command.ts +555 -15
- package/src/bundled-plugins/github-cli-auth/index.ts +131 -135
- package/src/bundled-plugins/github-cli-auth/review-checkout.ts +138 -0
- package/src/bundled-plugins/guard/policies/uncommitted-changes.ts +2 -1
- package/src/bundled-plugins/memory/README.md +23 -11
- package/src/bundled-plugins/memory/append-tool.ts +15 -8
- package/src/bundled-plugins/memory/citations.ts +22 -9
- package/src/bundled-plugins/memory/dreaming.ts +13 -8
- package/src/bundled-plugins/memory/find-entry-tool.ts +4 -0
- package/src/bundled-plugins/memory/index.ts +0 -4
- package/src/bundled-plugins/memory/load-memory.ts +8 -3
- package/src/bundled-plugins/memory/provenance-index.ts +717 -0
- package/src/bundled-plugins/memory/provenance-sanitize.ts +40 -0
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +4 -0
- package/src/bundled-plugins/memory/search-tool.ts +163 -60
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/vector/hybrid.ts +126 -13
- package/src/bundled-plugins/researcher/write-report.ts +6 -3
- package/src/bundled-plugins/reviewer/index.ts +2 -2
- package/src/bundled-plugins/reviewer/reviewer.ts +10 -3
- package/src/bundled-plugins/reviewer/skills/code-review.ts +19 -9
- package/src/bundled-plugins/scout/scout.ts +8 -0
- package/src/bundled-plugins/security/index.ts +4 -4
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +1 -0
- package/src/bundled-plugins/security/policies/private-surface-read.ts +220 -29
- package/src/bundled-plugins/security/policies/secret-exfil-read.ts +1 -0
- package/src/bundled-plugins/security/policies/ssrf.ts +29 -9
- package/src/channels/adapters/discord-bot-channel-resolver.ts +26 -14
- package/src/channels/adapters/discord-bot-thread-room.ts +44 -12
- package/src/channels/adapters/discord-bot.ts +66 -6
- package/src/channels/adapters/discord-channel-resolver.ts +91 -23
- package/src/channels/adapters/discord-id.ts +4 -0
- package/src/channels/adapters/discord-resolver-cache.ts +41 -0
- package/src/channels/adapters/discord.ts +57 -7
- package/src/channels/adapters/github/auth-app.ts +41 -12
- package/src/channels/adapters/github/index.ts +23 -11
- package/src/channels/adapters/github/review-submitter.ts +302 -0
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +5 -3
- package/src/channels/adapters/kakaotalk-fetch-attachment.ts +8 -4
- package/src/channels/adapters/kakaotalk-typing.ts +17 -15
- package/src/channels/adapters/slack-attachment-download.ts +71 -0
- package/src/channels/adapters/slack-bot.ts +19 -8
- package/src/channels/adapters/slack-classify.ts +21 -2
- package/src/channels/adapters/slack.ts +59 -7
- package/src/channels/adapters/telegram-bot.ts +18 -5
- package/src/channels/adapters/webex-bot.ts +8 -3
- package/src/channels/adapters/webex.ts +8 -3
- package/src/channels/fetch-attachment.ts +43 -0
- package/src/{bundled-plugins/github-cli-auth/approve-idempotency.ts → channels/github-review-verdict-coordinator.ts} +25 -1
- package/src/channels/github-token-bridge.ts +10 -3
- package/src/channels/manager.ts +9 -0
- package/src/channels/router.ts +248 -6
- package/src/channels/types.ts +28 -1
- package/src/cli/channel.ts +1 -1
- package/src/cli/init.ts +1 -1
- package/src/cli/model.ts +57 -7
- package/src/compose/stop.ts +17 -5
- package/src/config/config.ts +6 -6
- package/src/config/models-mutation.ts +13 -1
- package/src/config/providers.ts +176 -0
- package/src/container/start.ts +49 -10
- package/src/doctor/checks.ts +3 -1
- package/src/doctor/commit.ts +2 -1
- package/src/doctor/file-ownership.ts +52 -0
- package/src/dreams/git.ts +2 -1
- package/src/git/hookless.ts +5 -0
- package/src/git/reconcile-ignored.ts +18 -5
- package/src/git/secret-history.ts +371 -0
- package/src/git/system-commit.ts +12 -6
- package/src/init/dockerfile.ts +86 -5
- package/src/init/index.ts +17 -5
- package/src/init/models-dev.ts +4 -0
- package/src/init/validate-api-key.ts +2 -0
- package/src/mcp/tools.ts +16 -3
- package/src/permissions/builtins.ts +3 -1
- package/src/plugin/context.ts +2 -0
- package/src/plugin/index.ts +1 -0
- package/src/plugin/manager.ts +4 -0
- package/src/plugin/types.ts +21 -0
- package/src/run/index.ts +40 -0
- package/src/run/llm-fetch-observer.ts +41 -4
- package/src/sandbox/build.ts +37 -15
- package/src/sandbox/canonical-secrets.ts +18 -0
- package/src/sandbox/env-exposure.ts +80 -0
- package/src/sandbox/errors.ts +13 -6
- package/src/sandbox/hidden-paths.ts +102 -44
- package/src/sandbox/index.ts +22 -4
- package/src/sandbox/package-install.ts +3 -27
- package/src/sandbox/policy.ts +11 -25
- package/src/sandbox/privileged-runtime.ts +255 -0
- package/src/sandbox/session-tmp.ts +8 -2
- package/src/sandbox/writable-zones.ts +204 -140
- package/src/secrets/index.ts +8 -0
- package/src/secrets/refresh-provider-oauth.ts +128 -0
- package/src/server/command-runner.ts +48 -4
- package/src/server/index.ts +4 -0
- package/src/skills/typeclaw-channel-github/SKILL.md +49 -56
- package/src/skills/typeclaw-claude-code/SKILL.md +24 -60
- package/src/skills/typeclaw-claude-code/references/auth-flow.md +17 -158
- package/src/skills/typeclaw-codex-cli/SKILL.md +19 -49
- package/src/skills/typeclaw-codex-cli/references/auth-flow.md +17 -154
- package/src/skills/typeclaw-config/SKILL.md +5 -5
- package/src/skills/typeclaw-config/references/recommended-mounts.md +24 -24
- package/src/skills/typeclaw-github-contributing/SKILL.md +23 -26
- package/src/skills/typeclaw-memory/SKILL.md +9 -3
- package/src/skills/typeclaw-permissions/SKILL.md +19 -15
- package/typeclaw.schema.json +20 -0
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { detectSecrets } from './secret-detector'
|
|
2
|
+
|
|
3
|
+
export const MAX_PROVENANCE_NAME_LENGTH = 256
|
|
4
|
+
|
|
5
|
+
export function sanitizeProvenanceName(value: string | undefined): string | undefined {
|
|
6
|
+
if (value === undefined || value.length === 0 || value.length > MAX_PROVENANCE_NAME_LENGTH) return undefined
|
|
7
|
+
if (hasUnsafeUnicode(value) || hasMarkdownPromptShaping(value) || detectSecrets(value).length > 0) return undefined
|
|
8
|
+
return value
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function isSafeProvenanceCoordinate(value: string): boolean {
|
|
12
|
+
return value.length > 0 && value.length <= MAX_PROVENANCE_NAME_LENGTH && !hasUnsafeUnicode(value)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function hasUnsafeUnicode(value: string): boolean {
|
|
16
|
+
for (const character of value) {
|
|
17
|
+
const codePoint = character.codePointAt(0)
|
|
18
|
+
if (codePoint === undefined) continue
|
|
19
|
+
if (codePoint <= 0x1f || (codePoint >= 0x7f && codePoint <= 0x9f)) return true
|
|
20
|
+
if (codePoint === 0x00ad || codePoint === 0x034f || codePoint === 0x180e) return true
|
|
21
|
+
if (codePoint === 0x061c || codePoint === 0x200e || codePoint === 0x200f) return true
|
|
22
|
+
if (codePoint >= 0x2028 && codePoint <= 0x202e) return true
|
|
23
|
+
if (codePoint >= 0x2061 && codePoint <= 0x2064) return true
|
|
24
|
+
if (codePoint >= 0x2066 && codePoint <= 0x2069) return true
|
|
25
|
+
if (codePoint === 0x200b || codePoint === 0x200c || codePoint === 0x200d) return true
|
|
26
|
+
if (codePoint === 0x2060 || codePoint === 0xfeff) return true
|
|
27
|
+
if (codePoint >= 0xe0000 && codePoint <= 0xe007f) return true
|
|
28
|
+
}
|
|
29
|
+
return false
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function hasMarkdownPromptShaping(value: string): boolean {
|
|
33
|
+
if (['`', '*', '_', '[', ']', '<', '>', '|', '\\'].some((character) => value.includes(character))) return true
|
|
34
|
+
if (value.includes('__') || value.includes('![')) return true
|
|
35
|
+
const trimmed = value.trimStart()
|
|
36
|
+
if (trimmed.startsWith('#') || trimmed.startsWith('>') || trimmed.startsWith('---')) return true
|
|
37
|
+
if (trimmed.startsWith('- ') || trimmed.startsWith('+ ')) return true
|
|
38
|
+
if (/^\d{1,9}[.)]\s/u.test(trimmed)) return true
|
|
39
|
+
return false
|
|
40
|
+
}
|
|
@@ -23,6 +23,10 @@ export function createStoreReferenceTool(onReferenceStored?: ReferenceStoredHook
|
|
|
23
23
|
origin: z.enum(['episode', 'curated', 'external']),
|
|
24
24
|
tags: z.array(z.string()).optional(),
|
|
25
25
|
}),
|
|
26
|
+
// title/body/tags are prose the tool writes under a runtime-computed slug
|
|
27
|
+
// path; none is a caller-supplied file path. Declared so a title like
|
|
28
|
+
// "README.md" or a body carrying "/" is not misread as a file operand.
|
|
29
|
+
fileOperands: { nonFile: ['title', 'body', 'tags'] },
|
|
26
30
|
async execute({ title, body, origin, tags }, ctx) {
|
|
27
31
|
const existingSlugs = new Set(await listReferenceSlugs(ctx.agentDir))
|
|
28
32
|
const slug = headingToSlug(title, existingSlugs)
|
|
@@ -5,10 +5,17 @@ import { z } from 'zod'
|
|
|
5
5
|
import { defineTool } from '@/plugin'
|
|
6
6
|
|
|
7
7
|
import { loadAllShards, loadShard, type TopicShard } from './load-shards'
|
|
8
|
+
import {
|
|
9
|
+
buildProvenanceIndex,
|
|
10
|
+
buildProvenanceIndexFrom,
|
|
11
|
+
type MemoryScope,
|
|
12
|
+
type ProvenanceChild,
|
|
13
|
+
} from './provenance-index'
|
|
14
|
+
import { sanitizeProvenanceName } from './provenance-sanitize'
|
|
8
15
|
import { renderReference } from './references/frontmatter'
|
|
9
16
|
import { loadAllReferences, loadReference, type Reference } from './references/load-references'
|
|
10
17
|
import type { FragmentEvent, FragmentProvenance, LegacyProseEvent, StreamEvent } from './stream-events'
|
|
11
|
-
import {
|
|
18
|
+
import { filterUndreamedEvents, readAllStreamDays, type UndreamedStreamDay } from './stream-io'
|
|
12
19
|
|
|
13
20
|
const DEFAULT_MAX_RESULTS = 10
|
|
14
21
|
const EXCERPT_CONTEXT_LINES = 3
|
|
@@ -21,6 +28,7 @@ export type TopicMatch = {
|
|
|
21
28
|
heading: string
|
|
22
29
|
excerpt: string
|
|
23
30
|
fullBody?: string
|
|
31
|
+
provenance?: ProvenanceChild[]
|
|
24
32
|
}
|
|
25
33
|
|
|
26
34
|
export type StreamMatch = {
|
|
@@ -54,7 +62,7 @@ export type Matcher = (haystack: string) => boolean
|
|
|
54
62
|
export function createMemorySearchTool() {
|
|
55
63
|
return defineTool({
|
|
56
64
|
description:
|
|
57
|
-
'Search
|
|
65
|
+
'Search long-term memory and recent undreamed fragments, or look up one exact topic/reference slug. Plain search is case-insensitive phrase-first with token-OR fallback; regex mode has no token fallback. Runtime-owned provenance from active topic citations is searchable but never embedded. `workspace`, `chat`, and `thread` scope both topics and recent fragments to matching captured origin IDs or names; legacy `where` remains an alias for `chat`. Scoped topic eligibility, lexical matching, and returned provenance use the same eligible active cited children. Unresolved citations remain explicit in unscoped results and never gain invented metadata.',
|
|
58
66
|
parameters: z.object({
|
|
59
67
|
query: z.string().optional(),
|
|
60
68
|
topic: z.string().optional(),
|
|
@@ -64,47 +72,95 @@ export function createMemorySearchTool() {
|
|
|
64
72
|
since: z.string().optional(),
|
|
65
73
|
before: z.string().optional(),
|
|
66
74
|
where: z.string().optional(),
|
|
75
|
+
workspace: z.string().optional(),
|
|
76
|
+
chat: z.string().optional(),
|
|
77
|
+
thread: z.string().optional(),
|
|
67
78
|
}),
|
|
68
|
-
|
|
79
|
+
// Every string field is a search term, an exact topic/reference slug, an ISO
|
|
80
|
+
// date, or an origin id/name — none is a local path (the tool reads memory/
|
|
81
|
+
// internally, never a caller path). Declared so a `topic` slug that collides
|
|
82
|
+
// with an agent-root dir, or a `since` date like "2026-01-01", is not
|
|
83
|
+
// misread as a file operand.
|
|
84
|
+
fileOperands: {
|
|
85
|
+
nonFile: ['query', 'topic', 'since', 'before', 'where', 'workspace', 'chat', 'thread'],
|
|
86
|
+
},
|
|
87
|
+
async execute({ query, topic, asRegex, full, maxResults, since, before, where, workspace, chat, thread }, ctx) {
|
|
69
88
|
if ((query === undefined) === (topic === undefined)) {
|
|
70
89
|
return resultToToolResult({ error: 'provide exactly one of `query` or `topic`' })
|
|
71
90
|
}
|
|
72
91
|
|
|
92
|
+
const chatScope = chat ?? where
|
|
93
|
+
const requestedScope: MemoryScope = {
|
|
94
|
+
...(workspace === undefined ? {} : { workspace }),
|
|
95
|
+
...(chatScope === undefined ? {} : { chat: chatScope }),
|
|
96
|
+
...(thread === undefined ? {} : { thread }),
|
|
97
|
+
}
|
|
98
|
+
const scope = requestedScope
|
|
99
|
+
|
|
73
100
|
if (topic !== undefined) {
|
|
74
|
-
return resultToToolResult(await lookupTopic(ctx.agentDir, topic, ctx.logger))
|
|
101
|
+
return resultToToolResult(await lookupTopic(ctx.agentDir, topic, scope, ctx.logger))
|
|
75
102
|
}
|
|
76
103
|
|
|
77
104
|
const matcherOrError = buildMatcher(query!, asRegex)
|
|
78
105
|
if (typeof matcherOrError === 'string') {
|
|
79
106
|
return resultToToolResult({ error: matcherOrError })
|
|
80
107
|
}
|
|
81
|
-
|
|
82
|
-
const [shards, streamDays, allReferences] = await Promise.all([
|
|
108
|
+
const [shards, allStreamDays, allReferences] = await Promise.all([
|
|
83
109
|
loadAllShards(ctx.agentDir, { logger: ctx.logger }),
|
|
84
|
-
|
|
110
|
+
readAllStreamDays(ctx.agentDir),
|
|
85
111
|
loadAllReferences(ctx.agentDir, { logger: ctx.logger }),
|
|
86
112
|
])
|
|
113
|
+
const provenanceIndex = await buildProvenanceIndexFrom(ctx.agentDir, shards, allStreamDays)
|
|
114
|
+
const scoped = hasMemoryScope(scope)
|
|
115
|
+
const streamDays: UndreamedStreamDay[] = allStreamDays.flatMap((day) => {
|
|
116
|
+
const events: StreamEvent[] = []
|
|
117
|
+
for (const event of filterUndreamedEvents(day.events, day.dreamedIds)) {
|
|
118
|
+
if (event.type !== 'fragment') {
|
|
119
|
+
if (!scoped) events.push(event)
|
|
120
|
+
continue
|
|
121
|
+
}
|
|
122
|
+
const citation = `streams/${day.date}#${event.id}`
|
|
123
|
+
if (provenanceIndex.isActivelyCited(citation) || provenanceIndex.isSuperseded(citation)) continue
|
|
124
|
+
const child = provenanceIndex.undreamedChild(citation, scope)
|
|
125
|
+
if (child !== undefined) events.push(safeStreamEvent(event, child))
|
|
126
|
+
}
|
|
127
|
+
return events.length === 0 ? [] : [{ date: day.date, path: day.path, name: day.name, events }]
|
|
128
|
+
})
|
|
87
129
|
const dateFilter = parseReferenceDateFilter(since, before)
|
|
88
130
|
if ('error' in dateFilter) return resultToToolResult(dateFilter)
|
|
89
131
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
const
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
132
|
+
const roomFilteredDays = streamDays
|
|
133
|
+
const scopedShards = shards.filter((shard) => provenanceIndex.topicEligible(shard.slug, scope))
|
|
134
|
+
const references = !scoped ? allReferences.filter((r) => referenceCandidateAllowed(r, dateFilter)) : []
|
|
135
|
+
const topicDocuments = new Map(
|
|
136
|
+
scopedShards.map((shard) => [
|
|
137
|
+
shard.slug,
|
|
138
|
+
`${shardSearchText(shard)}\n${provenanceIndex.lexicalTextForTopic(shard.slug, scope)}`,
|
|
139
|
+
]),
|
|
140
|
+
)
|
|
141
|
+
const topicProvenance = new Map(
|
|
142
|
+
scopedShards.map((shard) => [shard.slug, provenanceIndex.childrenForTopic(shard.slug, scope)]),
|
|
143
|
+
)
|
|
144
|
+
const streamDocuments = new Map<string, string>()
|
|
145
|
+
for (const day of roomFilteredDays) {
|
|
146
|
+
for (const event of day.events) {
|
|
147
|
+
if (event.type !== 'fragment') continue
|
|
148
|
+
const citation = `streams/${day.date}#${event.id}`
|
|
149
|
+
streamDocuments.set(
|
|
150
|
+
citation,
|
|
151
|
+
`${eventSearchText(event)}\n${provenanceIndex.lexicalTextForUndreamed(citation, scope)}`,
|
|
152
|
+
)
|
|
153
|
+
}
|
|
154
|
+
}
|
|
97
155
|
|
|
98
156
|
if (scopedShards.length === 0 && roomFilteredDays.length === 0 && references.length === 0) {
|
|
99
157
|
return resultToToolResult({ matches: [], truncatedAt: 0 })
|
|
100
158
|
}
|
|
101
159
|
|
|
102
|
-
|
|
160
|
+
const searchOptions = { full, maxResults, references, topicDocuments, topicProvenance, streamDocuments }
|
|
161
|
+
let result = searchAll(scopedShards, roomFilteredDays, matcherOrError, searchOptions)
|
|
103
162
|
if ('matches' in result && result.matches.length === 0) {
|
|
104
|
-
const fallback = tokenFallback(query!, asRegex, scopedShards, roomFilteredDays, references,
|
|
105
|
-
full,
|
|
106
|
-
maxResults,
|
|
107
|
-
})
|
|
163
|
+
const fallback = tokenFallback(query!, asRegex, scopedShards, roomFilteredDays, references, searchOptions)
|
|
108
164
|
if (fallback !== null) result = fallback
|
|
109
165
|
}
|
|
110
166
|
if ('matches' in result) await bumpReturnedReferences(allReferences, result.matches)
|
|
@@ -113,6 +169,15 @@ export function createMemorySearchTool() {
|
|
|
113
169
|
})
|
|
114
170
|
}
|
|
115
171
|
|
|
172
|
+
function safeStreamEvent(event: FragmentEvent, child: ProvenanceChild): FragmentEvent {
|
|
173
|
+
const safe = { ...event }
|
|
174
|
+
delete safe.who
|
|
175
|
+
delete safe.where
|
|
176
|
+
if (child.who !== undefined) safe.who = child.who
|
|
177
|
+
if (child.where !== undefined) safe.where = child.where
|
|
178
|
+
return safe
|
|
179
|
+
}
|
|
180
|
+
|
|
116
181
|
export const memorySearchTool = createMemorySearchTool()
|
|
117
182
|
|
|
118
183
|
// Exact slug lookup, so the agent can read a topic OR reference whose slug the
|
|
@@ -127,6 +192,7 @@ export const memorySearchTool = createMemorySearchTool()
|
|
|
127
192
|
async function lookupTopic(
|
|
128
193
|
agentDir: string,
|
|
129
194
|
slug: string,
|
|
195
|
+
scope: MemoryScope,
|
|
130
196
|
logger?: { warn(message: string): void },
|
|
131
197
|
): Promise<MemorySearchResult> {
|
|
132
198
|
const loaderOptions = logger === undefined ? {} : { logger }
|
|
@@ -136,8 +202,14 @@ async function lookupTopic(
|
|
|
136
202
|
} catch (err) {
|
|
137
203
|
return { error: `invalid topic slug: ${err instanceof Error ? err.message : String(err)}` }
|
|
138
204
|
}
|
|
139
|
-
if (shard !== null)
|
|
205
|
+
if (shard !== null) {
|
|
206
|
+
const index = await buildProvenanceIndex(agentDir)
|
|
207
|
+
if (!index.topicEligible(shard.slug, scope)) return { matches: [] }
|
|
208
|
+
const provenance = index.childrenForTopic(shard.slug, scope)
|
|
209
|
+
return { matches: [topicMatchWithFullBody(shard, provenance)] }
|
|
210
|
+
}
|
|
140
211
|
|
|
212
|
+
if (hasMemoryScope(scope)) return { matches: [] }
|
|
141
213
|
const reference = await loadReference(agentDir, slug, loaderOptions)
|
|
142
214
|
if (reference !== null) {
|
|
143
215
|
const match = referenceMatchWithFullBody(reference)
|
|
@@ -151,7 +223,11 @@ async function lookupTopic(
|
|
|
151
223
|
return { matches: [] }
|
|
152
224
|
}
|
|
153
225
|
|
|
154
|
-
function
|
|
226
|
+
function hasMemoryScope(scope: MemoryScope): boolean {
|
|
227
|
+
return scope.workspace !== undefined || scope.chat !== undefined || scope.thread !== undefined
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function topicMatchWithFullBody(shard: TopicShard, provenance: ProvenanceChild[] = []): TopicMatch {
|
|
155
231
|
return {
|
|
156
232
|
source: 'topic',
|
|
157
233
|
shardPath: shard.path,
|
|
@@ -159,6 +235,7 @@ function topicMatchWithFullBody(shard: TopicShard): TopicMatch {
|
|
|
159
235
|
heading: shard.frontmatter.heading,
|
|
160
236
|
excerpt: excerpt(shard.body),
|
|
161
237
|
fullBody: shard.body,
|
|
238
|
+
...(provenance.length > 0 ? { provenance } : {}),
|
|
162
239
|
}
|
|
163
240
|
}
|
|
164
241
|
|
|
@@ -193,7 +270,13 @@ function tokenFallback(
|
|
|
193
270
|
shards: TopicShard[],
|
|
194
271
|
streamDays: UndreamedStreamDay[],
|
|
195
272
|
references: Reference[],
|
|
196
|
-
options: {
|
|
273
|
+
options: {
|
|
274
|
+
full: boolean
|
|
275
|
+
maxResults: number
|
|
276
|
+
topicDocuments?: Map<string, string>
|
|
277
|
+
topicProvenance?: Map<string, ProvenanceChild[]>
|
|
278
|
+
streamDocuments?: Map<string, string>
|
|
279
|
+
},
|
|
197
280
|
): MemorySearchResult | null {
|
|
198
281
|
if (asRegex) return null
|
|
199
282
|
const tokens = distinctTokens(query)
|
|
@@ -238,7 +321,14 @@ export function searchAll(
|
|
|
238
321
|
shards: TopicShard[],
|
|
239
322
|
streamDays: UndreamedStreamDay[],
|
|
240
323
|
matcher: Matcher,
|
|
241
|
-
options: {
|
|
324
|
+
options: {
|
|
325
|
+
full: boolean
|
|
326
|
+
maxResults: number
|
|
327
|
+
references?: Reference[]
|
|
328
|
+
topicDocuments?: Map<string, string>
|
|
329
|
+
topicProvenance?: Map<string, ProvenanceChild[]>
|
|
330
|
+
streamDocuments?: Map<string, string>
|
|
331
|
+
},
|
|
242
332
|
): MemorySearchResult {
|
|
243
333
|
const matches: MemorySearchMatch[] = []
|
|
244
334
|
let truncatedAt: number | undefined
|
|
@@ -253,7 +343,13 @@ export function searchAll(
|
|
|
253
343
|
}
|
|
254
344
|
|
|
255
345
|
for (const shard of shards) {
|
|
256
|
-
const match = matchShard(
|
|
346
|
+
const match = matchShard(
|
|
347
|
+
shard,
|
|
348
|
+
matcher,
|
|
349
|
+
options.full,
|
|
350
|
+
options.topicDocuments?.get(shard.slug),
|
|
351
|
+
options.topicProvenance?.get(shard.slug),
|
|
352
|
+
)
|
|
257
353
|
if (match === null) continue
|
|
258
354
|
if (!push(match)) return { matches, truncatedAt: truncatedAt! }
|
|
259
355
|
}
|
|
@@ -267,7 +363,7 @@ export function searchAll(
|
|
|
267
363
|
for (let i = streamDays.length - 1; i >= 0; i--) {
|
|
268
364
|
const day = streamDays[i]!
|
|
269
365
|
for (const event of day.events) {
|
|
270
|
-
const match = matchStreamEvent(day, event, matcher, options.full)
|
|
366
|
+
const match = matchStreamEvent(day, event, matcher, options.full, options.streamDocuments)
|
|
271
367
|
if (match === null) continue
|
|
272
368
|
if (!push(match)) return { matches, truncatedAt: truncatedAt! }
|
|
273
369
|
}
|
|
@@ -299,6 +395,9 @@ export function searchAllRanked(
|
|
|
299
395
|
maxResults: number
|
|
300
396
|
references?: Reference[]
|
|
301
397
|
tokenMatchMode?: 'substring' | 'ascii-boundary'
|
|
398
|
+
topicDocuments?: Map<string, string>
|
|
399
|
+
topicProvenance?: Map<string, ProvenanceChild[]>
|
|
400
|
+
streamDocuments?: Map<string, string>
|
|
302
401
|
},
|
|
303
402
|
): MemorySearchResult {
|
|
304
403
|
const tokenMatchers = tokens.map((t) => buildTokenMatcher(t, options.tokenMatchMode ?? 'substring'))
|
|
@@ -315,9 +414,19 @@ export function searchAllRanked(
|
|
|
315
414
|
let order = 0
|
|
316
415
|
|
|
317
416
|
for (const shard of shards) {
|
|
318
|
-
const match = matchShard(
|
|
417
|
+
const match = matchShard(
|
|
418
|
+
shard,
|
|
419
|
+
anyToken,
|
|
420
|
+
options.full,
|
|
421
|
+
options.topicDocuments?.get(shard.slug),
|
|
422
|
+
options.topicProvenance?.get(shard.slug),
|
|
423
|
+
)
|
|
319
424
|
if (match === null) continue
|
|
320
|
-
scored.push({
|
|
425
|
+
scored.push({
|
|
426
|
+
match,
|
|
427
|
+
score: scoreOf(options.topicDocuments?.get(shard.slug) ?? shardSearchText(shard)),
|
|
428
|
+
order: order++,
|
|
429
|
+
})
|
|
321
430
|
}
|
|
322
431
|
|
|
323
432
|
for (const reference of options.references ?? []) {
|
|
@@ -329,9 +438,16 @@ export function searchAllRanked(
|
|
|
329
438
|
for (let i = streamDays.length - 1; i >= 0; i--) {
|
|
330
439
|
const day = streamDays[i]!
|
|
331
440
|
for (const event of day.events) {
|
|
332
|
-
const match = matchStreamEvent(day, event, anyToken, options.full)
|
|
441
|
+
const match = matchStreamEvent(day, event, anyToken, options.full, options.streamDocuments)
|
|
333
442
|
if (match === null) continue
|
|
334
|
-
|
|
443
|
+
const citation = event.type === 'fragment' ? `streams/${day.date}#${event.id}` : undefined
|
|
444
|
+
scored.push({
|
|
445
|
+
match,
|
|
446
|
+
score: scoreOf(
|
|
447
|
+
(citation === undefined ? undefined : options.streamDocuments?.get(citation)) ?? eventSearchText(event),
|
|
448
|
+
),
|
|
449
|
+
order: order++,
|
|
450
|
+
})
|
|
335
451
|
}
|
|
336
452
|
}
|
|
337
453
|
|
|
@@ -378,7 +494,13 @@ function referenceSearchText(reference: Reference): string {
|
|
|
378
494
|
return [reference.slug, reference.frontmatter.title, ...reference.frontmatter.tags, reference.body].join('\n')
|
|
379
495
|
}
|
|
380
496
|
|
|
381
|
-
function matchShard(
|
|
497
|
+
function matchShard(
|
|
498
|
+
shard: TopicShard,
|
|
499
|
+
matcher: Matcher,
|
|
500
|
+
full: boolean,
|
|
501
|
+
searchText = shardSearchText(shard),
|
|
502
|
+
provenance?: ProvenanceChild[],
|
|
503
|
+
): TopicMatch | null {
|
|
382
504
|
const bodyLines = splitBodyLines(shard.body)
|
|
383
505
|
const firstBodyLineIndex = bodyLines.findIndex((line) => matcher(line))
|
|
384
506
|
|
|
@@ -386,7 +508,8 @@ function matchShard(shard: TopicShard, matcher: Matcher, full: boolean): TopicMa
|
|
|
386
508
|
matcher(shard.slug) ||
|
|
387
509
|
matcher(shard.frontmatter.heading) ||
|
|
388
510
|
(shard.frontmatter.tags?.some((tag) => matcher(tag)) ?? false) ||
|
|
389
|
-
firstBodyLineIndex !== -1
|
|
511
|
+
firstBodyLineIndex !== -1 ||
|
|
512
|
+
matcher(searchText)
|
|
390
513
|
if (!matched) return null
|
|
391
514
|
|
|
392
515
|
const match: TopicMatch = {
|
|
@@ -398,6 +521,7 @@ function matchShard(shard: TopicShard, matcher: Matcher, full: boolean): TopicMa
|
|
|
398
521
|
firstBodyLineIndex === -1 ? fallbackExcerpt(shard, matcher) : excerptForLine(bodyLines, firstBodyLineIndex),
|
|
399
522
|
}
|
|
400
523
|
if (full) match.fullBody = shard.body
|
|
524
|
+
if (provenance !== undefined && provenance.length > 0) match.provenance = provenance
|
|
401
525
|
return match
|
|
402
526
|
}
|
|
403
527
|
|
|
@@ -439,9 +563,12 @@ function matchStreamEvent(
|
|
|
439
563
|
event: StreamEvent,
|
|
440
564
|
matcher: Matcher,
|
|
441
565
|
full: boolean,
|
|
566
|
+
streamDocuments?: Map<string, string>,
|
|
442
567
|
): StreamMatch | null {
|
|
443
568
|
if (event.type === 'watermark') return null
|
|
444
|
-
if (event.type === 'fragment')
|
|
569
|
+
if (event.type === 'fragment') {
|
|
570
|
+
return matchFragmentEvent(day, event, matcher, full, streamDocuments?.get(`streams/${day.date}#${event.id}`))
|
|
571
|
+
}
|
|
445
572
|
return matchLegacyProseEvent(day, event, matcher, full)
|
|
446
573
|
}
|
|
447
574
|
|
|
@@ -450,10 +577,11 @@ function matchFragmentEvent(
|
|
|
450
577
|
event: FragmentEvent,
|
|
451
578
|
matcher: Matcher,
|
|
452
579
|
full: boolean,
|
|
580
|
+
searchText = eventSearchText(event),
|
|
453
581
|
): StreamMatch | null {
|
|
454
582
|
const bodyLines = splitBodyLines(event.body)
|
|
455
583
|
const firstBodyLineIndex = bodyLines.findIndex((line) => matcher(line))
|
|
456
|
-
const matched = matcher(event.topic) || firstBodyLineIndex !== -1
|
|
584
|
+
const matched = matcher(event.topic) || firstBodyLineIndex !== -1 || matcher(searchText)
|
|
457
585
|
if (!matched) return null
|
|
458
586
|
|
|
459
587
|
const match: StreamMatch = {
|
|
@@ -465,7 +593,8 @@ function matchFragmentEvent(
|
|
|
465
593
|
excerpt: firstBodyLineIndex === -1 ? event.topic : excerptForLine(bodyLines, firstBodyLineIndex),
|
|
466
594
|
when: event.ts,
|
|
467
595
|
}
|
|
468
|
-
|
|
596
|
+
const who = sanitizeProvenanceName(event.who)
|
|
597
|
+
if (who !== undefined) match.who = who
|
|
469
598
|
if (event.where !== undefined) match.where = event.where
|
|
470
599
|
if (full) match.fullBody = event.body
|
|
471
600
|
return match
|
|
@@ -530,32 +659,6 @@ function parseDateParam(name: string, value: string): Date | string {
|
|
|
530
659
|
return date
|
|
531
660
|
}
|
|
532
661
|
|
|
533
|
-
// Keep only fragments whose `where` names this room, by raw `chat` id (canonical)
|
|
534
|
-
// or `chatName` (convenience, case-insensitive — the agent often knows only
|
|
535
|
-
// "#incidents"). The `#` prefix is stripped so "#incidents" and "incidents"
|
|
536
|
-
// both match. Non-fragment events (watermarks, legacy prose) have no `where` and
|
|
537
|
-
// are dropped under a room filter; days that end up empty are removed.
|
|
538
|
-
function filterStreamDaysByRoom(days: UndreamedStreamDay[], room: string): UndreamedStreamDay[] {
|
|
539
|
-
const normalizedKey = normalizeRoomKey(room)
|
|
540
|
-
return days.flatMap((day) => {
|
|
541
|
-
const events = day.events.filter(
|
|
542
|
-
(event) => event.type === 'fragment' && fragmentMatchesRoom(event, room, normalizedKey),
|
|
543
|
-
)
|
|
544
|
-
return events.length === 0 ? [] : [{ ...day, events }]
|
|
545
|
-
})
|
|
546
|
-
}
|
|
547
|
-
|
|
548
|
-
function fragmentMatchesRoom(event: FragmentEvent, rawRoom: string, normalizedKey: string): boolean {
|
|
549
|
-
const where = event.where
|
|
550
|
-
if (where === undefined) return false
|
|
551
|
-
if (where.chat === rawRoom) return true
|
|
552
|
-
return where.chatName !== undefined && normalizeRoomKey(where.chatName) === normalizedKey
|
|
553
|
-
}
|
|
554
|
-
|
|
555
|
-
function normalizeRoomKey(value: string): string {
|
|
556
|
-
return value.replace(/^#/, '').toLocaleLowerCase()
|
|
557
|
-
}
|
|
558
|
-
|
|
559
662
|
function referenceCandidateAllowed(reference: Reference, filter: ReferenceDateFilter): boolean {
|
|
560
663
|
if (reference.frontmatter.demoted) return false
|
|
561
664
|
const created = new Date(reference.frontmatter.created)
|