typeclaw 0.51.0 → 0.52.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/types/agent/bash-sandbox-env.d.ts +2 -0
- package/dist/types/agent/bash-sandbox.d.ts +4 -0
- package/dist/types/agent/index.d.ts +5 -0
- package/dist/types/agent/multimodal/look-at.d.ts +1 -1
- package/dist/types/agent/plugin-tools.d.ts +4 -0
- package/dist/types/agent/prompt-cache-key.d.ts +1 -0
- package/dist/types/agent/session-tail.d.ts +12 -0
- package/dist/types/agent/subagents.d.ts +5 -0
- package/dist/types/agent/system-prompt-composition.d.ts +1 -0
- package/dist/types/agent/system-prompt.d.ts +2 -0
- package/dist/types/agent/tools/channel-disengage.d.ts +1 -1
- package/dist/types/agent/tools/channel-edit.d.ts +1 -1
- package/dist/types/agent/tools/channel-fetch-attachment.d.ts +1 -1
- package/dist/types/agent/tools/channel-handoff.d.ts +1 -1
- package/dist/types/agent/tools/channel-history.d.ts +1 -1
- package/dist/types/agent/tools/channel-react.d.ts +1 -1
- package/dist/types/agent/tools/channel-read.d.ts +10 -2
- package/dist/types/agent/tools/channel-reply.d.ts +1 -1
- package/dist/types/agent/tools/channel-send.d.ts +1 -1
- package/dist/types/agent/tools/channel-session-search.d.ts +1 -1
- package/dist/types/agent/tools/grant-role.d.ts +1 -1
- package/dist/types/agent/tools/post-github-review.d.ts +1 -1
- package/dist/types/agent/tools/restart.d.ts +1 -1
- package/dist/types/agent/tools/skip-response.d.ts +1 -1
- package/dist/types/agent/tools/spawn-subagent.d.ts +2 -1
- package/dist/types/agent/tools/stream-snapshot.d.ts +1 -1
- package/dist/types/agent/tools/subagent-cancel.d.ts +1 -1
- package/dist/types/agent/tools/subagent-output.d.ts +1 -1
- package/dist/types/agent/tools/todo/index.d.ts +1 -1
- package/dist/types/agent/tools/webfetch/adaptive-html.d.ts +3 -3
- package/dist/types/agent/tools/webfetch/strategies/readability-process.d.ts +48 -0
- package/dist/types/agent/tools/webfetch/tool.d.ts +2 -2
- package/dist/types/agent/tools/websearch.d.ts +1 -1
- package/dist/types/channels/adapters/github/auth-app.d.ts +25 -0
- package/dist/types/channels/adapters/github/auth-pat.d.ts +17 -0
- package/dist/types/channels/adapters/github/auth.d.ts +24 -0
- package/dist/types/channels/github-token-bridge.d.ts +4 -0
- package/dist/types/channels/lost-background-context.d.ts +2 -0
- package/dist/types/channels/persistence.d.ts +1 -0
- package/dist/types/channels/router.d.ts +5 -1
- package/dist/types/channels/types.d.ts +1 -0
- package/dist/types/coding-agent/tool.d.ts +1 -1
- package/dist/types/config/config.d.ts +20 -11
- package/dist/types/config/providers.d.ts +47 -0
- package/dist/types/eval/sandbox.d.ts +2 -0
- package/dist/types/git/resolve-identity.d.ts +2 -0
- package/dist/types/init/dockerfile.d.ts +1 -0
- package/dist/types/init/gitignore.d.ts +1 -1
- package/dist/types/permissions/builtins.d.ts +1 -0
- package/dist/types/plugin/types.d.ts +3 -0
- package/dist/types/sandbox/canonical-secrets.d.ts +2 -2
- package/dist/types/sandbox/coding-agent.d.ts +1 -1
- package/dist/types/sandbox/dependency-bins.d.ts +1 -0
- package/dist/types/sandbox/dev-source.d.ts +2 -2
- package/dist/types/sandbox/hidden-paths.d.ts +1 -0
- package/dist/types/sandbox/index.d.ts +1 -1
- package/dist/types/sandbox/policy.d.ts +3 -0
- package/dist/types/typevault/cli-path.d.ts +2 -0
- package/dist/types/typevault/envelope-store.d.ts +11 -0
- package/dist/types/typevault/lifecycle.d.ts +13 -0
- package/dist/types/typevault/vault.d.ts +27 -0
- package/package.json +8 -7
- package/scripts/test.ts +252 -0
- package/src/agent/bash-sandbox-env.ts +33 -1
- package/src/agent/bash-sandbox.ts +20 -6
- package/src/agent/index.ts +120 -31
- package/src/agent/multimodal/look-at.ts +1 -1
- package/src/agent/plugin-tools.ts +35 -1
- package/src/agent/prompt-cache-key.ts +26 -3
- package/src/agent/session-origin.ts +39 -18
- package/src/agent/session-tail.ts +28 -0
- package/src/agent/subagents.ts +21 -2
- package/src/agent/system-prompt-composition.ts +18 -0
- package/src/agent/system-prompt.ts +39 -1
- package/src/agent/tool-error-remediation.ts +18 -3
- package/src/agent/tools/channel-disengage.ts +1 -1
- package/src/agent/tools/channel-edit.ts +1 -1
- package/src/agent/tools/channel-fetch-attachment.ts +1 -1
- package/src/agent/tools/channel-handoff.ts +3 -2
- package/src/agent/tools/channel-history.ts +2 -1
- package/src/agent/tools/channel-react.ts +1 -1
- package/src/agent/tools/channel-read.ts +36 -3
- package/src/agent/tools/channel-reply.ts +1 -1
- package/src/agent/tools/channel-send.ts +4 -3
- package/src/agent/tools/channel-session-search.ts +13 -4
- package/src/agent/tools/grant-role.ts +1 -1
- package/src/agent/tools/post-github-review.ts +1 -1
- package/src/agent/tools/restart.ts +1 -1
- package/src/agent/tools/skip-response.ts +1 -1
- package/src/agent/tools/spawn-subagent.ts +15 -1
- package/src/agent/tools/stream-snapshot.ts +1 -1
- package/src/agent/tools/subagent-cancel.ts +1 -1
- package/src/agent/tools/subagent-output.ts +1 -1
- package/src/agent/tools/todo/index.ts +1 -1
- package/src/agent/tools/webfetch/adaptive-html.ts +6 -4
- package/src/agent/tools/webfetch/strategies/readability-child.ts +25 -0
- package/src/agent/tools/webfetch/strategies/readability-process.ts +185 -0
- package/src/agent/tools/webfetch/tool.ts +6 -5
- package/src/agent/tools/websearch.ts +1 -1
- package/src/browser/dockerfile.ts +1 -1
- package/src/browser/fill-secret.ts +92 -35
- package/src/browser/registry.ts +6 -6
- package/src/bundled-plugins/agent-messenger-cli-auth/command.ts +211 -0
- package/src/bundled-plugins/agent-messenger-cli-auth/index.ts +186 -0
- package/src/bundled-plugins/backup/index.ts +17 -2
- package/src/bundled-plugins/backup/runner.ts +37 -2
- package/src/bundled-plugins/browser-agent/browser-agent.ts +51 -14
- package/src/bundled-plugins/browser-agent/index.ts +12 -5
- package/src/bundled-plugins/managed-browser/index.ts +25 -2
- package/src/bundled-plugins/managed-browser/skills/browserskill/browserskill/SKILL.md +13 -5
- package/src/bundled-plugins/managed-browser/skills/omowright/omowright/SKILL.md +56 -5
- package/src/bundled-plugins/memory/README.md +23 -2
- package/src/bundled-plugins/memory/append-fragment.ts +133 -0
- package/src/bundled-plugins/memory/append-tool.ts +80 -149
- package/src/bundled-plugins/memory/dreaming.ts +2 -0
- package/src/bundled-plugins/memory/index.ts +33 -6
- package/src/bundled-plugins/memory/memory-logger.ts +133 -11
- package/src/bundled-plugins/memory/references/store-reference-tool.ts +56 -32
- package/src/bundled-plugins/memory/remember-tool.ts +81 -0
- package/src/bundled-plugins/memory/search-tool-hybrid.ts +91 -0
- package/src/bundled-plugins/memory/search-tool.ts +58 -12
- package/src/bundled-plugins/memory/stream-events.ts +2 -0
- package/src/bundled-plugins/memory/transcript-cursor.ts +30 -0
- package/src/bundled-plugins/memory/transcript-preload.ts +131 -26
- package/src/bundled-plugins/memory/transcript-view.ts +36 -0
- package/src/bundled-plugins/memory/watermark.ts +19 -10
- package/src/bundled-plugins/reviewer/reviewer.ts +20 -0
- package/src/bundled-plugins/security/policies/outbound-secret-scan.ts +3 -6
- package/src/bundled-plugins/security/policies/outbound-text.ts +20 -0
- package/src/bundled-plugins/security/policies/system-prompt-leak.ts +2 -7
- package/src/bundled-plugins/typevault/grants.ts +119 -0
- package/src/bundled-plugins/typevault/index.ts +152 -0
- package/src/bundled-plugins/typevault/request.ts +87 -0
- package/src/bundled-plugins/typevault/runtime.ts +45 -0
- package/src/bundled-plugins/typevault/skills/typevault/SKILL.md +65 -0
- package/src/channels/adapter-halt.ts +17 -0
- package/src/channels/adapters/github/history.ts +23 -2
- package/src/channels/adapters/github/inbound.ts +60 -1
- package/src/channels/adapters/github/index.ts +27 -5
- package/src/channels/adapters/github/reconcile-open-prs.ts +7 -1
- package/src/channels/adapters/kakaotalk-author-resolver.ts +7 -5
- package/src/channels/adapters/kakaotalk-channel-resolver.ts +45 -15
- package/src/channels/adapters/kakaotalk-classify.ts +7 -9
- package/src/channels/adapters/kakaotalk-list.ts +62 -0
- package/src/channels/adapters/kakaotalk-member-cache.ts +67 -0
- package/src/channels/adapters/kakaotalk-membership.ts +27 -10
- package/src/channels/adapters/kakaotalk-mention.ts +61 -0
- package/src/channels/adapters/kakaotalk.ts +159 -125
- package/src/channels/adapters/teams-classify.ts +18 -8
- package/src/channels/github-token-bridge.ts +13 -0
- package/src/channels/lost-background-context.ts +46 -0
- package/src/channels/manager.ts +52 -3
- package/src/channels/persistence.ts +5 -0
- package/src/channels/router.ts +120 -21
- package/src/channels/schema.ts +1 -1
- package/src/channels/types.ts +3 -0
- package/src/coding-agent/tool.ts +1 -1
- package/src/config/config.ts +20 -1
- package/src/config/providers.ts +65 -0
- package/src/container/start.ts +15 -2
- package/src/doctor/channel-checks.ts +33 -0
- package/src/doctor/checks.ts +12 -2
- package/src/eval/sandbox.ts +3 -0
- package/src/eval/tool.ts +1 -0
- package/src/git/resolve-identity.ts +67 -2
- package/src/git/system-commit.ts +7 -0
- package/src/init/dockerfile.ts +5 -1
- package/src/init/gitignore.ts +3 -3
- package/src/init/index.ts +1 -0
- package/src/init/models-dev.ts +2 -0
- package/src/init/validate-api-key.ts +5 -0
- package/src/migrations/secrets-v1-to-v2.ts +2 -0
- package/src/permissions/builtins.ts +8 -0
- package/src/plugin/types.ts +6 -0
- package/src/run/bundled-plugins.ts +12 -0
- package/src/run/channel-session-factory.ts +3 -0
- package/src/run/index.ts +13 -2
- package/src/sandbox/build.ts +1 -0
- package/src/sandbox/canonical-secrets.ts +6 -0
- package/src/sandbox/dependency-bins.ts +13 -1
- package/src/sandbox/dev-source.ts +21 -12
- package/src/sandbox/hidden-paths.ts +1 -1
- package/src/sandbox/index.ts +1 -0
- package/src/sandbox/policy.ts +5 -0
- package/src/secrets/hydrate.ts +4 -1
- package/src/secrets/storage.ts +10 -1
- package/src/server/command-runner.ts +4 -0
- package/src/server/index.ts +8 -1
- package/src/skills/typeclaw-config/SKILL.md +16 -0
- package/src/skills/typeclaw-memory/SKILL.md +3 -1
- package/src/skills/typeclaw-permissions/SKILL.md +8 -8
- package/src/skills/typeclaw-troubleshooting/SKILL.md +18 -1
- package/src/tunnels/active.ts +14 -0
- package/src/typevault/bash-cli.ts +18 -0
- package/src/typevault/cli-path.ts +11 -0
- package/src/typevault/envelope-helper.ts +17 -0
- package/src/typevault/envelope-store.ts +98 -0
- package/src/typevault/lifecycle.ts +132 -0
- package/src/typevault/prompt-tunnel.ts +33 -0
- package/src/typevault/redaction.ts +37 -0
- package/src/typevault/vault.ts +283 -0
- package/typeclaw.schema.json +40 -10
|
@@ -6,7 +6,7 @@ import { defineTool } from '@/plugin'
|
|
|
6
6
|
|
|
7
7
|
import { referenceFilePath, referencesDir } from '../paths'
|
|
8
8
|
import { headingToSlug } from '../slug'
|
|
9
|
-
import { renderReference } from './frontmatter'
|
|
9
|
+
import { renderReference, type ReferenceOrigin } from './frontmatter'
|
|
10
10
|
import { listReferenceSlugs } from './load-references'
|
|
11
11
|
|
|
12
12
|
export type ReferenceStoredHook = (context: { slug: string; body: string }) => Promise<void>
|
|
@@ -27,37 +27,61 @@ export function createStoreReferenceTool(onReferenceStored?: ReferenceStoredHook
|
|
|
27
27
|
// path; none is a caller-supplied file path. Declared so a title like
|
|
28
28
|
// "README.md" or a body carrying "/" is not misread as a file operand.
|
|
29
29
|
fileOperands: { nonFile: ['title', 'body', 'tags'] },
|
|
30
|
-
async execute(
|
|
31
|
-
|
|
32
|
-
const slug = headingToSlug(title, existingSlugs)
|
|
33
|
-
const created = new Date().toISOString()
|
|
34
|
-
const path = referenceFilePath(ctx.agentDir, slug)
|
|
35
|
-
|
|
36
|
-
await mkdir(referencesDir(ctx.agentDir), { recursive: true })
|
|
37
|
-
await writeFile(
|
|
38
|
-
path,
|
|
39
|
-
renderReference(
|
|
40
|
-
{
|
|
41
|
-
title,
|
|
42
|
-
origin,
|
|
43
|
-
created,
|
|
44
|
-
lastAccessed: created,
|
|
45
|
-
accessCount: 0,
|
|
46
|
-
pinned: false,
|
|
47
|
-
demoted: false,
|
|
48
|
-
tags: tags ?? [],
|
|
49
|
-
},
|
|
50
|
-
body,
|
|
51
|
-
),
|
|
52
|
-
'utf8',
|
|
53
|
-
)
|
|
54
|
-
|
|
55
|
-
if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
|
|
56
|
-
|
|
57
|
-
return {
|
|
58
|
-
content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
|
|
59
|
-
details: { path, slug },
|
|
60
|
-
}
|
|
30
|
+
async execute(input, ctx) {
|
|
31
|
+
return storeReference(input, ctx.agentDir, onReferenceStored)
|
|
61
32
|
},
|
|
62
33
|
})
|
|
63
34
|
}
|
|
35
|
+
|
|
36
|
+
export async function storeReference(
|
|
37
|
+
input: {
|
|
38
|
+
readonly title: string
|
|
39
|
+
readonly body: string
|
|
40
|
+
readonly origin: ReferenceOrigin
|
|
41
|
+
readonly tags?: string[] | undefined
|
|
42
|
+
},
|
|
43
|
+
agentDir: string,
|
|
44
|
+
onReferenceStored?: ReferenceStoredHook,
|
|
45
|
+
) {
|
|
46
|
+
const { title, body, origin, tags } = input
|
|
47
|
+
const existingSlugs = new Set(await listReferenceSlugs(agentDir))
|
|
48
|
+
const created = new Date().toISOString()
|
|
49
|
+
const content = renderReference(
|
|
50
|
+
{
|
|
51
|
+
title,
|
|
52
|
+
origin,
|
|
53
|
+
created,
|
|
54
|
+
lastAccessed: created,
|
|
55
|
+
accessCount: 0,
|
|
56
|
+
pinned: false,
|
|
57
|
+
demoted: false,
|
|
58
|
+
tags: tags ?? [],
|
|
59
|
+
},
|
|
60
|
+
body,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
await mkdir(referencesDir(agentDir), { recursive: true })
|
|
64
|
+
// Concurrent writers (memory-logger and every main-agent session) can pick
|
|
65
|
+
// the same slug from the same listing, so the file is reserved with an
|
|
66
|
+
// exclusive create and a taken slug moves to the next deduplicated suffix.
|
|
67
|
+
let slug = headingToSlug(title, existingSlugs)
|
|
68
|
+
let path = referenceFilePath(agentDir, slug)
|
|
69
|
+
for (;;) {
|
|
70
|
+
try {
|
|
71
|
+
await writeFile(path, content, { encoding: 'utf8', flag: 'wx' })
|
|
72
|
+
break
|
|
73
|
+
} catch (err) {
|
|
74
|
+
if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST')) throw err
|
|
75
|
+
existingSlugs.add(slug)
|
|
76
|
+
slug = headingToSlug(title, existingSlugs)
|
|
77
|
+
path = referenceFilePath(agentDir, slug)
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
if (onReferenceStored !== undefined) await onReferenceStored({ slug, body })
|
|
82
|
+
|
|
83
|
+
return {
|
|
84
|
+
content: [{ type: 'text' as const, text: `Stored reference as ${slug}` }],
|
|
85
|
+
details: { path, slug },
|
|
86
|
+
}
|
|
87
|
+
}
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import { z } from 'zod'
|
|
2
|
+
|
|
3
|
+
import { defineTool } from '@/plugin'
|
|
4
|
+
import { formatLocalDate } from '@/shared'
|
|
5
|
+
|
|
6
|
+
import { appendFragment, type FragmentsAppendedHook } from './append-fragment'
|
|
7
|
+
import { provenanceFromOrigin } from './append-tool'
|
|
8
|
+
import { streamFilePath } from './paths'
|
|
9
|
+
import { listReferenceSlugs } from './references/load-references'
|
|
10
|
+
import { storeReference, type ReferenceStoredHook } from './references/store-reference-tool'
|
|
11
|
+
|
|
12
|
+
export function createMemoryRememberTool(
|
|
13
|
+
options: {
|
|
14
|
+
readonly onFragmentsAppended?: FragmentsAppendedHook
|
|
15
|
+
readonly onReferenceStored?: ReferenceStoredHook
|
|
16
|
+
} = {},
|
|
17
|
+
) {
|
|
18
|
+
return defineTool({
|
|
19
|
+
description:
|
|
20
|
+
'Remember a durable fact, decision, preference, or correction now. Use when asked to remember something or when durable knowledge is established, not for routine chatter. Supply a short topic and self-contained body in the evidence language; who optionally names the person it is attributable to. Set verbatim to also retain the exact body as a linked reference. The runtime stamps provenance and writes one daily-stream fragment; dreaming consolidates it later.',
|
|
21
|
+
parameters: z.object({
|
|
22
|
+
topic: z.string().min(1),
|
|
23
|
+
body: z.string().min(1),
|
|
24
|
+
who: z.string().min(1).optional(),
|
|
25
|
+
verbatim: z.boolean().optional(),
|
|
26
|
+
}),
|
|
27
|
+
// All arguments are content/control values, never caller-selected paths.
|
|
28
|
+
fileOperands: { nonFile: ['topic', 'body', 'who', 'verbatim'] },
|
|
29
|
+
async execute({ topic, body, who, verbatim }, ctx) {
|
|
30
|
+
const path = streamFilePath(ctx.agentDir, formatLocalDate())
|
|
31
|
+
const { fragment } = await appendFragment(
|
|
32
|
+
{
|
|
33
|
+
topic,
|
|
34
|
+
body,
|
|
35
|
+
who,
|
|
36
|
+
source: ctx.sessionId,
|
|
37
|
+
// An empty anchor is honest for direct calls without a transcript.
|
|
38
|
+
entry: ctx.entryId ?? '',
|
|
39
|
+
kind: 'explicit',
|
|
40
|
+
},
|
|
41
|
+
ctx,
|
|
42
|
+
{
|
|
43
|
+
path,
|
|
44
|
+
where: provenanceFromOrigin(ctx.origin),
|
|
45
|
+
onFragmentsAppended: options.onFragmentsAppended,
|
|
46
|
+
referenceSlugResolver: listReferenceSlugs,
|
|
47
|
+
...(verbatim
|
|
48
|
+
? {
|
|
49
|
+
storeReference: async () => {
|
|
50
|
+
// Indexing is best-effort once the reference is persisted, like
|
|
51
|
+
// the fragment hook: a failure must not orphan the stored file
|
|
52
|
+
// or drop the explicit fragment that cites it.
|
|
53
|
+
const { onReferenceStored } = options
|
|
54
|
+
const result = await storeReference(
|
|
55
|
+
{ title: topic, body, origin: 'episode' },
|
|
56
|
+
ctx.agentDir,
|
|
57
|
+
onReferenceStored === undefined
|
|
58
|
+
? undefined
|
|
59
|
+
: async (reference) => {
|
|
60
|
+
try {
|
|
61
|
+
await onReferenceStored(reference)
|
|
62
|
+
} catch (err) {
|
|
63
|
+
ctx.logger.warn(
|
|
64
|
+
`[memory] post-store reference index hook failed: ${err instanceof Error ? err.message : String(err)}`,
|
|
65
|
+
)
|
|
66
|
+
}
|
|
67
|
+
},
|
|
68
|
+
)
|
|
69
|
+
return result.details.slug
|
|
70
|
+
},
|
|
71
|
+
}
|
|
72
|
+
: {}),
|
|
73
|
+
},
|
|
74
|
+
)
|
|
75
|
+
return {
|
|
76
|
+
content: [{ type: 'text' as const, text: `Remembered "${topic}".` }],
|
|
77
|
+
details: { path, fragmentId: fragment.id, references: fragment.references ?? [] },
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
})
|
|
81
|
+
}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import { loadShard } from './load-shards'
|
|
4
|
+
import { loadReference } from './references/load-references'
|
|
5
|
+
import type { MemorySearchMatch, StreamMatch } from './search-tool'
|
|
6
|
+
import { readAllStreamDays } from './stream-io'
|
|
7
|
+
import type { HybridSearchResult } from './vector/hybrid'
|
|
8
|
+
|
|
9
|
+
// Hybrid results carry excerpts and provenance, while the tool also exposes
|
|
10
|
+
// canonical paths, citation IDs and optional complete bodies.
|
|
11
|
+
export async function hydrateHybridMatches(
|
|
12
|
+
agentDir: string,
|
|
13
|
+
results: HybridSearchResult[],
|
|
14
|
+
full: boolean,
|
|
15
|
+
): Promise<MemorySearchMatch[]> {
|
|
16
|
+
const streams = new Map<string, StreamMatch>()
|
|
17
|
+
if (results.some((hit) => hit.source === 'stream')) {
|
|
18
|
+
for (const day of await readAllStreamDays(agentDir)) {
|
|
19
|
+
for (const event of day.events) {
|
|
20
|
+
const base = { source: 'stream', streamPath: day.path, date: day.date, excerpt: '' } as const
|
|
21
|
+
switch (event.type) {
|
|
22
|
+
case 'fragment':
|
|
23
|
+
streams.set(`${day.date}#${event.id}`, {
|
|
24
|
+
...base,
|
|
25
|
+
topic: event.topic,
|
|
26
|
+
eventId: `streams/${day.date}#${event.id}`,
|
|
27
|
+
...(full ? { fullBody: event.body } : {}),
|
|
28
|
+
})
|
|
29
|
+
break
|
|
30
|
+
case 'legacy_prose':
|
|
31
|
+
streams.set(`${day.date}#legacy-${createHash('sha256').update(event.text).digest('hex').slice(0, 12)}`, {
|
|
32
|
+
...base,
|
|
33
|
+
topic: '[legacy prose from pre-shard migration]',
|
|
34
|
+
...(full ? { fullBody: event.text } : {}),
|
|
35
|
+
})
|
|
36
|
+
break
|
|
37
|
+
case 'watermark':
|
|
38
|
+
break
|
|
39
|
+
default:
|
|
40
|
+
event satisfies never
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
const matches = await Promise.all(
|
|
47
|
+
results.map(async (hit): Promise<MemorySearchMatch | null> => {
|
|
48
|
+
switch (hit.source) {
|
|
49
|
+
case 'topic': {
|
|
50
|
+
const shard = await loadShard(agentDir, hit.key)
|
|
51
|
+
if (shard === null) return null
|
|
52
|
+
return {
|
|
53
|
+
source: 'topic',
|
|
54
|
+
shardPath: shard.path,
|
|
55
|
+
slug: hit.key,
|
|
56
|
+
heading: hit.heading,
|
|
57
|
+
excerpt: hit.excerpt,
|
|
58
|
+
...(hit.provenance === undefined ? {} : { provenance: hit.provenance }),
|
|
59
|
+
...(full ? { fullBody: shard.body } : {}),
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
case 'reference': {
|
|
63
|
+
const reference = await loadReference(agentDir, hit.key)
|
|
64
|
+
if (reference === null) return null
|
|
65
|
+
return {
|
|
66
|
+
source: 'reference',
|
|
67
|
+
slug: hit.key,
|
|
68
|
+
title: hit.heading,
|
|
69
|
+
excerpt: hit.excerpt,
|
|
70
|
+
created: reference.frontmatter.created,
|
|
71
|
+
...(full ? { fullBody: reference.body } : {}),
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
case 'stream': {
|
|
75
|
+
const stream = streams.get(hit.key)
|
|
76
|
+
if (stream === undefined) return null
|
|
77
|
+
return {
|
|
78
|
+
...stream,
|
|
79
|
+
excerpt: hit.excerpt,
|
|
80
|
+
...(hit.who === undefined ? {} : { who: hit.who }),
|
|
81
|
+
...(hit.when === undefined ? {} : { when: hit.when }),
|
|
82
|
+
...(hit.where === undefined ? {} : { where: hit.where }),
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
default:
|
|
86
|
+
return hit.source satisfies never
|
|
87
|
+
}
|
|
88
|
+
}),
|
|
89
|
+
)
|
|
90
|
+
return matches.filter((match) => match !== null)
|
|
91
|
+
}
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { existsSync } from 'node:fs'
|
|
1
2
|
import { writeFile } from 'node:fs/promises'
|
|
2
3
|
|
|
3
4
|
import { z } from 'zod'
|
|
@@ -13,9 +14,13 @@ import {
|
|
|
13
14
|
} from './provenance-index'
|
|
14
15
|
import { sanitizeProvenanceName } from './provenance-sanitize'
|
|
15
16
|
import { renderReference } from './references/frontmatter'
|
|
16
|
-
import { loadAllReferences, loadReference, type Reference } from './references/load-references'
|
|
17
|
+
import { bumpReferenceAccess, loadAllReferences, loadReference, type Reference } from './references/load-references'
|
|
18
|
+
import { vectorDbPath } from './search-command'
|
|
19
|
+
import { hydrateHybridMatches } from './search-tool-hybrid'
|
|
17
20
|
import type { FragmentEvent, FragmentProvenance, LegacyProseEvent, StreamEvent } from './stream-events'
|
|
18
21
|
import { filterUndreamedEvents, readAllStreamDays, type UndreamedStreamDay } from './stream-io'
|
|
22
|
+
import { hybridSearch, type EmbedFn, type HybridSearchResult } from './vector/hybrid'
|
|
23
|
+
import { VectorStore } from './vector/store'
|
|
19
24
|
|
|
20
25
|
const DEFAULT_MAX_RESULTS = 10
|
|
21
26
|
const EXCERPT_CONTEXT_LINES = 3
|
|
@@ -59,12 +64,13 @@ export type MemorySearchResult = { matches: MemorySearchMatch[]; truncatedAt?: n
|
|
|
59
64
|
|
|
60
65
|
export type Matcher = (haystack: string) => boolean
|
|
61
66
|
|
|
62
|
-
export function createMemorySearchTool() {
|
|
67
|
+
export function createMemorySearchTool(embedFn?: EmbedFn) {
|
|
63
68
|
return defineTool({
|
|
64
69
|
description:
|
|
65
|
-
'Search long-term memory and recent undreamed fragments
|
|
70
|
+
'Search long-term memory and recent undreamed fragments by meaning and keywords (default mode: hybrid). Use mode: lexical for case-insensitive phrase-first search with token-OR fallback. Use asRegex: true for exact regex semantics without token fallback, or topic for one exact topic/reference slug; both bypass semantic search. If the semantic index is unavailable, the result reports lexical fallback. full: true returns complete bodies. Runtime-owned provenance from active topic citations is searchable but never embedded. `workspace`, `chat`, and `thread` scope both topics and recent fragments to matching captured origin IDs or names; legacy `where` remains an alias for `chat`. Scoped topic eligibility, lexical matching, and returned provenance use the same eligible active cited children. Unresolved citations remain explicit in unscoped results and never gain invented metadata.',
|
|
66
71
|
parameters: z.object({
|
|
67
72
|
query: z.string().optional(),
|
|
73
|
+
mode: z.enum(['hybrid', 'lexical']).default('hybrid'),
|
|
68
74
|
topic: z.string().optional(),
|
|
69
75
|
asRegex: z.boolean().default(false),
|
|
70
76
|
full: z.boolean().default(false),
|
|
@@ -82,9 +88,12 @@ export function createMemorySearchTool() {
|
|
|
82
88
|
// with an agent-root dir, or a `since` date like "2026-01-01", is not
|
|
83
89
|
// misread as a file operand.
|
|
84
90
|
fileOperands: {
|
|
85
|
-
nonFile: ['query', 'topic', 'since', 'before', 'where', 'workspace', 'chat', 'thread'],
|
|
91
|
+
nonFile: ['query', 'mode', 'topic', 'since', 'before', 'where', 'workspace', 'chat', 'thread'],
|
|
86
92
|
},
|
|
87
|
-
async execute(
|
|
93
|
+
async execute(
|
|
94
|
+
{ query, mode, topic, asRegex, full, maxResults, since, before, where, workspace, chat, thread },
|
|
95
|
+
ctx,
|
|
96
|
+
) {
|
|
88
97
|
if ((query === undefined) === (topic === undefined)) {
|
|
89
98
|
return resultToToolResult({ error: 'provide exactly one of `query` or `topic`' })
|
|
90
99
|
}
|
|
@@ -105,6 +114,37 @@ export function createMemorySearchTool() {
|
|
|
105
114
|
if (typeof matcherOrError === 'string') {
|
|
106
115
|
return resultToToolResult({ error: matcherOrError })
|
|
107
116
|
}
|
|
117
|
+
const dateFilter = parseReferenceDateFilter(since, before)
|
|
118
|
+
if ('error' in dateFilter) return resultToToolResult(dateFilter)
|
|
119
|
+
let fallbackReason: string | undefined
|
|
120
|
+
if (query !== undefined && mode === 'hybrid' && !asRegex) {
|
|
121
|
+
const dbPath = vectorDbPath(ctx.agentDir)
|
|
122
|
+
if (!existsSync(dbPath)) {
|
|
123
|
+
fallbackReason = 'memory/.vectors/index.db is absent'
|
|
124
|
+
} else {
|
|
125
|
+
try {
|
|
126
|
+
const store = VectorStore.open(dbPath)
|
|
127
|
+
let hits: HybridSearchResult[]
|
|
128
|
+
try {
|
|
129
|
+
hits = await hybridSearch(query, store, ctx.agentDir, maxResults, embedFn, { scope })
|
|
130
|
+
} finally {
|
|
131
|
+
store.close()
|
|
132
|
+
}
|
|
133
|
+
const matches = (await hydrateHybridMatches(ctx.agentDir, hits, full)).filter(
|
|
134
|
+
(match) => match.source !== 'reference' || referenceDateAllowed(match.created, dateFilter),
|
|
135
|
+
)
|
|
136
|
+
await bumpReferenceAccess(
|
|
137
|
+
ctx.agentDir,
|
|
138
|
+
matches.flatMap((match) => (match.source === 'reference' ? [match.slug] : [])),
|
|
139
|
+
{ logger: ctx.logger },
|
|
140
|
+
)
|
|
141
|
+
return resultToToolResult({ matches })
|
|
142
|
+
} catch (err) {
|
|
143
|
+
fallbackReason = err instanceof Error ? err.message : String(err)
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
ctx.logger.warn(`[memory-search] semantic index unavailable: ${fallbackReason}`)
|
|
147
|
+
}
|
|
108
148
|
const [shards, allStreamDays, allReferences] = await Promise.all([
|
|
109
149
|
loadAllShards(ctx.agentDir, { logger: ctx.logger }),
|
|
110
150
|
readAllStreamDays(ctx.agentDir),
|
|
@@ -126,8 +166,6 @@ export function createMemorySearchTool() {
|
|
|
126
166
|
}
|
|
127
167
|
return events.length === 0 ? [] : [{ date: day.date, path: day.path, name: day.name, events }]
|
|
128
168
|
})
|
|
129
|
-
const dateFilter = parseReferenceDateFilter(since, before)
|
|
130
|
-
if ('error' in dateFilter) return resultToToolResult(dateFilter)
|
|
131
169
|
|
|
132
170
|
const roomFilteredDays = streamDays
|
|
133
171
|
const scopedShards = shards.filter((shard) => provenanceIndex.topicEligible(shard.slug, scope))
|
|
@@ -154,7 +192,7 @@ export function createMemorySearchTool() {
|
|
|
154
192
|
}
|
|
155
193
|
|
|
156
194
|
if (scopedShards.length === 0 && roomFilteredDays.length === 0 && references.length === 0) {
|
|
157
|
-
return resultToToolResult({ matches: [], truncatedAt: 0 })
|
|
195
|
+
return resultToToolResult({ matches: [], truncatedAt: 0 }, fallbackReason)
|
|
158
196
|
}
|
|
159
197
|
|
|
160
198
|
const searchOptions = { full, maxResults, references, topicDocuments, topicProvenance, streamDocuments }
|
|
@@ -164,7 +202,7 @@ export function createMemorySearchTool() {
|
|
|
164
202
|
if (fallback !== null) result = fallback
|
|
165
203
|
}
|
|
166
204
|
if ('matches' in result) await bumpReturnedReferences(allReferences, result.matches)
|
|
167
|
-
return resultToToolResult(result)
|
|
205
|
+
return resultToToolResult(result, fallbackReason)
|
|
168
206
|
},
|
|
169
207
|
})
|
|
170
208
|
}
|
|
@@ -661,7 +699,11 @@ function parseDateParam(name: string, value: string): Date | string {
|
|
|
661
699
|
|
|
662
700
|
function referenceCandidateAllowed(reference: Reference, filter: ReferenceDateFilter): boolean {
|
|
663
701
|
if (reference.frontmatter.demoted) return false
|
|
664
|
-
|
|
702
|
+
return referenceDateAllowed(reference.frontmatter.created, filter)
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
function referenceDateAllowed(createdAt: string, filter: ReferenceDateFilter): boolean {
|
|
706
|
+
const created = new Date(createdAt)
|
|
665
707
|
if (filter.since !== undefined && created < filter.since) return false
|
|
666
708
|
if (filter.before !== undefined && created >= filter.before) return false
|
|
667
709
|
return true
|
|
@@ -702,9 +744,13 @@ const EMPTY_RESULT_GUIDANCE =
|
|
|
702
744
|
// The empty-set note rides in the LLM-facing `text` ONLY. `details` stays the
|
|
703
745
|
// pure struct: `keywordLane` reads `searchAll` directly (never this layer) and
|
|
704
746
|
// the structured tests assert on `details`, so both must see no `note`.
|
|
705
|
-
function resultToToolResult(result: MemorySearchResult) {
|
|
747
|
+
function resultToToolResult(result: MemorySearchResult, fallbackReason?: string) {
|
|
706
748
|
const isEmpty = 'matches' in result && result.matches.length === 0
|
|
707
|
-
const text =
|
|
749
|
+
const text = JSON.stringify({
|
|
750
|
+
...result,
|
|
751
|
+
...(isEmpty ? { note: EMPTY_RESULT_GUIDANCE } : {}),
|
|
752
|
+
...(fallbackReason === undefined ? {} : { notice: 'Semantic index unavailable; lexical results are shown.' }),
|
|
753
|
+
})
|
|
708
754
|
return {
|
|
709
755
|
content: [{ type: 'text' as const, text }],
|
|
710
756
|
details: result,
|
|
@@ -56,6 +56,8 @@ export const fragmentEventSchema = z
|
|
|
56
56
|
entry: z.string(),
|
|
57
57
|
topic: z.string(),
|
|
58
58
|
body: z.string(),
|
|
59
|
+
// Deliberate main-agent capture; absent on legacy/logger fragments.
|
|
60
|
+
kind: z.literal('explicit').optional(),
|
|
59
61
|
references: z.array(z.string()).optional(),
|
|
60
62
|
// WHO the evidence is attributable to (display name/handle of the speaker).
|
|
61
63
|
// LLM-supplied by the memory-logger, set ONLY when one transcript speaker
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto'
|
|
2
|
+
|
|
3
|
+
import { z } from 'zod'
|
|
4
|
+
|
|
5
|
+
const PREFIX = 'memory-page:'
|
|
6
|
+
const cursorSchema = z.object({
|
|
7
|
+
version: z.literal(1),
|
|
8
|
+
file: z.string(),
|
|
9
|
+
entry: z.string().min(1),
|
|
10
|
+
digest: z.string(),
|
|
11
|
+
offset: z.number().int().nonnegative(),
|
|
12
|
+
})
|
|
13
|
+
|
|
14
|
+
export function transcriptDigest(value: string): string {
|
|
15
|
+
return createHash('sha256').update(value).digest('hex')
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export function encodeTranscriptCursor(path: string, entry: string, text: string, offset: number): string {
|
|
19
|
+
const cursor = { version: 1, file: transcriptDigest(path), entry, digest: transcriptDigest(text), offset }
|
|
20
|
+
return `${PREFIX}${Buffer.from(JSON.stringify(cursor)).toString('base64url')}`
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export function decodeTranscriptCursor(value: string | null): z.infer<typeof cursorSchema> | null {
|
|
24
|
+
if (!value?.startsWith(PREFIX)) return null
|
|
25
|
+
return cursorSchema.parse(JSON.parse(Buffer.from(value.slice(PREFIX.length), 'base64url').toString('utf8')))
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export function transcriptEntryId(value: string): string {
|
|
29
|
+
return decodeTranscriptCursor(value)?.entry ?? value
|
|
30
|
+
}
|
|
@@ -1,15 +1,13 @@
|
|
|
1
1
|
import { readFile } from 'node:fs/promises'
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
//
|
|
7
|
-
//
|
|
8
|
-
//
|
|
3
|
+
import { decodeTranscriptCursor, encodeTranscriptCursor, transcriptDigest } from './transcript-cursor'
|
|
4
|
+
import { projectTranscriptLine } from './transcript-view'
|
|
5
|
+
|
|
6
|
+
// Pack complete projected records where possible. Large records are paged
|
|
7
|
+
// reversibly; the checkpoint includes the next character offset and digest,
|
|
8
|
+
// so an interrupted or changed entry is replayed rather than skipped.
|
|
9
9
|
export const PRELOAD_MAX_BYTES = 256 * 1024
|
|
10
|
-
export const PRELOAD_MAX_LINE_BYTES = 16 * 1024
|
|
11
10
|
const MIN_USEFUL_PRELOAD_BYTES = 8 * 1024
|
|
12
|
-
const TRUNCATION_MARKER_BYTES = 128
|
|
13
11
|
// Byte-level BPE tokenizers emit at most one token per UTF-8 byte, so byte
|
|
14
12
|
// length is a tokenizer-independent upper bound on token count. The overhead
|
|
15
13
|
// covers tool schemas and the preload header; each model's output allowance is
|
|
@@ -41,6 +39,9 @@ export type TranscriptPreload = {
|
|
|
41
39
|
totalLines: number
|
|
42
40
|
truncatedLines: number
|
|
43
41
|
watermarkFound: boolean
|
|
42
|
+
checkpoint: string | null
|
|
43
|
+
startCheckpoint: string | null
|
|
44
|
+
entryIds: string[]
|
|
44
45
|
// First unread line when the byte cap stopped the preload early; null when
|
|
45
46
|
// the preload reaches the end of the transcript.
|
|
46
47
|
nextOffset: number | null
|
|
@@ -49,10 +50,9 @@ export type TranscriptPreload = {
|
|
|
49
50
|
export async function buildTranscriptPreload(
|
|
50
51
|
path: string,
|
|
51
52
|
watermark: string | null,
|
|
52
|
-
options: { maxBytes?: number;
|
|
53
|
+
options: { maxBytes?: number; untilLine?: number } = {},
|
|
53
54
|
): Promise<TranscriptPreload | null> {
|
|
54
55
|
const maxBytes = options.maxBytes ?? PRELOAD_MAX_BYTES
|
|
55
|
-
const maxLineBytes = Math.min(options.maxLineBytes ?? PRELOAD_MAX_LINE_BYTES, maxBytes - TRUNCATION_MARKER_BYTES)
|
|
56
56
|
let raw: string
|
|
57
57
|
try {
|
|
58
58
|
raw = await readFile(path, 'utf8')
|
|
@@ -61,43 +61,148 @@ export async function buildTranscriptPreload(
|
|
|
61
61
|
}
|
|
62
62
|
const lines = raw.length === 0 ? [] : raw.split('\n')
|
|
63
63
|
if (lines.length > 0 && lines[lines.length - 1] === '') lines.pop()
|
|
64
|
-
const totalLines = lines.length
|
|
64
|
+
const totalLines = Math.min(lines.length, options.untilLine ?? lines.length)
|
|
65
65
|
|
|
66
66
|
let startIndex = 0
|
|
67
|
+
let startOffset = 0
|
|
67
68
|
let watermarkFound = false
|
|
69
|
+
const cursor = decodeTranscriptCursor(watermark)
|
|
68
70
|
if (watermark !== null) {
|
|
69
|
-
const
|
|
70
|
-
const found = lines.findIndex((line) => line
|
|
71
|
+
const entryId = cursor?.entry ?? watermark
|
|
72
|
+
const found = lines.findIndex((line) => recordId(line) === entryId)
|
|
71
73
|
if (found !== -1) {
|
|
72
|
-
startIndex = found + 1
|
|
74
|
+
startIndex = cursor === null ? found + 1 : found
|
|
73
75
|
watermarkFound = true
|
|
76
|
+
if (cursor !== null) {
|
|
77
|
+
const text = projectTranscriptLine(lines[found] ?? '')
|
|
78
|
+
if (
|
|
79
|
+
cursor.file === transcriptDigest(path) &&
|
|
80
|
+
cursor.digest === transcriptDigest(text) &&
|
|
81
|
+
cursor.offset <= text.length &&
|
|
82
|
+
!isInsideSurrogatePair(text, cursor.offset)
|
|
83
|
+
) {
|
|
84
|
+
startOffset = cursor.offset
|
|
85
|
+
}
|
|
86
|
+
}
|
|
74
87
|
}
|
|
75
88
|
}
|
|
76
89
|
|
|
77
90
|
const out: string[] = []
|
|
91
|
+
const entryIds: string[] = []
|
|
78
92
|
let bytes = 0
|
|
79
|
-
let
|
|
93
|
+
let checkpoint = watermark
|
|
94
|
+
const firstId = recordId(lines[startIndex] ?? '')
|
|
95
|
+
const startCheckpoint =
|
|
96
|
+
firstId === null
|
|
97
|
+
? null
|
|
98
|
+
: encodeTranscriptCursor(path, firstId, projectTranscriptLine(lines[startIndex] ?? ''), startOffset)
|
|
99
|
+
let partial = false
|
|
80
100
|
let index = startIndex
|
|
81
101
|
for (; index < totalLines; index++) {
|
|
82
|
-
|
|
83
|
-
const
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
102
|
+
const source = lines[index] ?? ''
|
|
103
|
+
const id = recordId(source)
|
|
104
|
+
// A live writer's incomplete tail is never acknowledged.
|
|
105
|
+
if (id === null) break
|
|
106
|
+
const text = projectTranscriptLine(source)
|
|
107
|
+
const offset = index === startIndex ? startOffset : 0
|
|
108
|
+
const separatorBytes = out.length === 0 ? 0 : 1
|
|
109
|
+
if (offset === 0 && bytes + separatorBytes + Buffer.byteLength(text, 'utf8') <= maxBytes) {
|
|
110
|
+
out.push(text)
|
|
111
|
+
entryIds.push(id)
|
|
112
|
+
bytes += separatorBytes + Buffer.byteLength(text, 'utf8')
|
|
113
|
+
checkpoint = id
|
|
114
|
+
continue
|
|
115
|
+
}
|
|
116
|
+
if (out.length > 0) break
|
|
117
|
+
|
|
118
|
+
const context = entryContext(text)
|
|
119
|
+
// Read-only overlap lets a fresh session interpret facts split at the
|
|
120
|
+
// boundary. Reserve most of even a small page for new progress.
|
|
121
|
+
let overlapStart = Math.max(0, offset - Math.min(2048, Math.floor(maxBytes / 16)))
|
|
122
|
+
if (isInsideSurrogatePair(text, overlapStart)) overlapStart++
|
|
123
|
+
const precedingText = offset > 0 ? text.slice(overlapStart, offset) : undefined
|
|
124
|
+
let low = offset
|
|
125
|
+
let high = text.length
|
|
126
|
+
while (low < high) {
|
|
127
|
+
const middle = Math.ceil((low + high) / 2)
|
|
128
|
+
const part = JSON.stringify({
|
|
129
|
+
type: 'entry-part',
|
|
130
|
+
id,
|
|
131
|
+
context,
|
|
132
|
+
offset,
|
|
133
|
+
precedingText,
|
|
134
|
+
text: text.slice(offset, middle),
|
|
135
|
+
})
|
|
136
|
+
if (Buffer.byteLength(part, 'utf8') <= maxBytes) low = middle
|
|
137
|
+
else high = middle - 1
|
|
87
138
|
}
|
|
88
|
-
const
|
|
89
|
-
if (
|
|
90
|
-
out.push(
|
|
91
|
-
|
|
139
|
+
const end = isInsideSurrogatePair(text, low) ? low - 1 : low
|
|
140
|
+
if (end <= offset) throw new Error('Memory transcript page budget cannot hold the next character.')
|
|
141
|
+
out.push(JSON.stringify({ type: 'entry-part', id, context, offset, precedingText, text: text.slice(offset, end) }))
|
|
142
|
+
entryIds.push(id)
|
|
143
|
+
checkpoint = end === text.length ? id : encodeTranscriptCursor(path, id, text, end)
|
|
144
|
+
partial = end < text.length
|
|
145
|
+
if (end === text.length) index++
|
|
146
|
+
break
|
|
92
147
|
}
|
|
93
148
|
|
|
94
149
|
return {
|
|
95
150
|
text: out.join('\n'),
|
|
96
151
|
startLine: startIndex + 1,
|
|
97
|
-
endLine: index,
|
|
152
|
+
endLine: index + (partial ? 1 : 0),
|
|
98
153
|
totalLines,
|
|
99
|
-
truncatedLines,
|
|
154
|
+
truncatedLines: 0,
|
|
100
155
|
watermarkFound,
|
|
156
|
+
checkpoint,
|
|
157
|
+
startCheckpoint,
|
|
158
|
+
entryIds,
|
|
101
159
|
nextOffset: index < totalLines ? index + 1 : null,
|
|
102
160
|
}
|
|
103
161
|
}
|
|
162
|
+
|
|
163
|
+
function entryContext(text: string): Record<string, unknown> {
|
|
164
|
+
const entry: unknown = JSON.parse(text)
|
|
165
|
+
if (!isRecord(entry)) return {}
|
|
166
|
+
const context: Record<string, unknown> = {}
|
|
167
|
+
for (const key of ['parentId', 'timestamp']) {
|
|
168
|
+
if (key in entry) context[key] = entry[key]
|
|
169
|
+
}
|
|
170
|
+
if (isRecord(entry.message)) {
|
|
171
|
+
for (const key of [
|
|
172
|
+
'role',
|
|
173
|
+
'timestamp',
|
|
174
|
+
'toolCallId',
|
|
175
|
+
'toolName',
|
|
176
|
+
'isError',
|
|
177
|
+
'stopReason',
|
|
178
|
+
'authorId',
|
|
179
|
+
'authorName',
|
|
180
|
+
'who',
|
|
181
|
+
'where',
|
|
182
|
+
]) {
|
|
183
|
+
if (key in entry.message) context[`message.${key}`] = entry.message[key]
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
return context
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function isRecord(value: unknown): value is Record<string, unknown> {
|
|
190
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
function recordId(line: string): string | null {
|
|
194
|
+
try {
|
|
195
|
+
const entry: unknown = JSON.parse(line)
|
|
196
|
+
if (typeof entry !== 'object' || entry === null || !('id' in entry)) return null
|
|
197
|
+
return typeof entry.id === 'string' ? entry.id : null
|
|
198
|
+
} catch (error) {
|
|
199
|
+
if (error instanceof SyntaxError) return null
|
|
200
|
+
throw error
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
function isInsideSurrogatePair(text: string, offset: number): boolean {
|
|
205
|
+
const before = text.charCodeAt(offset - 1)
|
|
206
|
+
const after = text.charCodeAt(offset)
|
|
207
|
+
return before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff
|
|
208
|
+
}
|