sakura-agent 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +132 -2
- package/README.zh-CN.md +133 -0
- package/THIRD_PARTY_NOTICES.md +63 -0
- package/dist/packages/agent/src/compose.d.ts +25 -0
- package/dist/packages/agent/src/compose.js +91 -0
- package/dist/packages/agent/src/index.d.ts +92 -0
- package/dist/packages/agent/src/index.js +166 -0
- package/dist/packages/agent/src/system-prompt.d.ts +10 -0
- package/dist/packages/agent/src/system-prompt.js +49 -0
- package/dist/packages/env/src/index.d.ts +3 -0
- package/dist/packages/env/src/index.js +2 -0
- package/dist/packages/env/src/local.d.ts +3 -0
- package/dist/packages/env/src/local.js +174 -0
- package/dist/packages/env/src/remote.d.ts +15 -0
- package/dist/packages/env/src/remote.js +120 -0
- package/dist/packages/kernel/src/channel.d.ts +7 -0
- package/dist/packages/kernel/src/channel.js +34 -0
- package/dist/packages/kernel/src/compaction.d.ts +25 -0
- package/dist/packages/kernel/src/compaction.js +79 -0
- package/dist/packages/kernel/src/constants.d.ts +2 -0
- package/dist/packages/kernel/src/constants.js +5 -0
- package/dist/packages/kernel/src/goal.d.ts +26 -0
- package/dist/packages/kernel/src/goal.js +37 -0
- package/dist/packages/kernel/src/hooks.d.ts +29 -0
- package/dist/packages/kernel/src/hooks.js +133 -0
- package/dist/packages/kernel/src/index.d.ts +6 -0
- package/dist/packages/kernel/src/index.js +3 -0
- package/dist/packages/kernel/src/loop.d.ts +51 -0
- package/dist/packages/kernel/src/loop.js +534 -0
- package/dist/packages/kernel/src/prompts.d.ts +18 -0
- package/dist/packages/kernel/src/prompts.js +99 -0
- package/dist/packages/kernel/src/system.d.ts +25 -0
- package/dist/packages/kernel/src/system.js +49 -0
- package/dist/packages/kernel/src/types.d.ts +377 -0
- package/dist/packages/kernel/src/types.js +23 -0
- package/dist/packages/llm/src/chat/compat.d.ts +19 -0
- package/dist/packages/llm/src/chat/compat.js +16 -0
- package/dist/packages/llm/src/chat/errors.d.ts +8 -0
- package/dist/packages/llm/src/chat/errors.js +44 -0
- package/dist/packages/llm/src/chat/request.d.ts +12 -0
- package/dist/packages/llm/src/chat/request.js +91 -0
- package/dist/packages/llm/src/chat/retry.d.ts +26 -0
- package/dist/packages/llm/src/chat/retry.js +57 -0
- package/dist/packages/llm/src/chat/stream.d.ts +17 -0
- package/dist/packages/llm/src/chat/stream.js +121 -0
- package/dist/packages/llm/src/index.d.ts +45 -0
- package/dist/packages/llm/src/index.js +115 -0
- package/dist/packages/plugins/agents/src/index.d.ts +33 -0
- package/dist/packages/plugins/agents/src/index.js +206 -0
- package/dist/packages/plugins/agents/src/view.d.ts +24 -0
- package/dist/packages/plugins/agents/src/view.js +60 -0
- package/dist/packages/plugins/context/src/index.d.ts +26 -0
- package/dist/packages/plugins/context/src/index.js +66 -0
- package/dist/packages/plugins/limits/src/cost.d.ts +9 -0
- package/dist/packages/plugins/limits/src/cost.js +3 -0
- package/dist/packages/plugins/limits/src/index.d.ts +28 -0
- package/dist/packages/plugins/limits/src/index.js +31 -0
- package/dist/packages/plugins/redaction/src/engine.d.ts +24 -0
- package/dist/packages/plugins/redaction/src/engine.js +110 -0
- package/dist/packages/plugins/redaction/src/index.d.ts +25 -0
- package/dist/packages/plugins/redaction/src/index.js +66 -0
- package/dist/packages/plugins/redaction/src/restore.d.ts +6 -0
- package/dist/packages/plugins/redaction/src/restore.js +44 -0
- package/dist/packages/plugins/redaction/src/rules.gen.d.ts +14 -0
- package/dist/packages/plugins/redaction/src/rules.gen.js +2059 -0
- package/dist/packages/plugins/resilience/src/index.d.ts +31 -0
- package/dist/packages/plugins/resilience/src/index.js +86 -0
- package/dist/packages/plugins/review/src/chain.d.ts +31 -0
- package/dist/packages/plugins/review/src/chain.js +24 -0
- package/dist/packages/plugins/review/src/hard-rules.d.ts +7 -0
- package/dist/packages/plugins/review/src/hard-rules.js +123 -0
- package/dist/packages/plugins/review/src/index.d.ts +45 -0
- package/dist/packages/plugins/review/src/index.js +58 -0
- package/dist/packages/plugins/review/src/policy.d.ts +15 -0
- package/dist/packages/plugins/review/src/policy.js +65 -0
- package/dist/packages/plugins/review/src/reviewer.d.ts +39 -0
- package/dist/packages/plugins/review/src/reviewer.js +119 -0
- package/dist/packages/plugins/review/src/shell.d.ts +10 -0
- package/dist/packages/plugins/review/src/shell.js +124 -0
- package/dist/packages/plugins/tools/src/artifacts.d.ts +27 -0
- package/dist/packages/plugins/tools/src/artifacts.js +61 -0
- package/dist/packages/plugins/tools/src/builtin/args.d.ts +11 -0
- package/dist/packages/plugins/tools/src/builtin/args.js +46 -0
- package/dist/packages/plugins/tools/src/builtin/bash.d.ts +11 -0
- package/dist/packages/plugins/tools/src/builtin/bash.js +96 -0
- package/dist/packages/plugins/tools/src/builtin/edit.d.ts +2 -0
- package/dist/packages/plugins/tools/src/builtin/edit.js +71 -0
- package/dist/packages/plugins/tools/src/builtin/read.d.ts +2 -0
- package/dist/packages/plugins/tools/src/builtin/read.js +75 -0
- package/dist/packages/plugins/tools/src/builtin/write.d.ts +2 -0
- package/dist/packages/plugins/tools/src/builtin/write.js +35 -0
- package/dist/packages/plugins/tools/src/edit-match.d.ts +21 -0
- package/dist/packages/plugins/tools/src/edit-match.js +159 -0
- package/dist/packages/plugins/tools/src/fold.d.ts +16 -0
- package/dist/packages/plugins/tools/src/fold.js +65 -0
- package/dist/packages/plugins/tools/src/index.d.ts +10 -0
- package/dist/packages/plugins/tools/src/index.js +10 -0
- package/dist/packages/plugins/toolset/src/bm25.d.ts +8 -0
- package/dist/packages/plugins/toolset/src/bm25.js +35 -0
- package/dist/packages/plugins/toolset/src/codemode.d.ts +36 -0
- package/dist/packages/plugins/toolset/src/codemode.js +168 -0
- package/dist/packages/plugins/toolset/src/index.d.ts +6 -0
- package/dist/packages/plugins/toolset/src/index.js +3 -0
- package/dist/packages/plugins/toolset/src/mcp.d.ts +27 -0
- package/dist/packages/plugins/toolset/src/mcp.js +173 -0
- package/dist/packages/plugins/toolset/src/search.d.ts +12 -0
- package/dist/packages/plugins/toolset/src/search.js +57 -0
- package/dist/packages/session/src/file-log.d.ts +20 -0
- package/dist/packages/session/src/file-log.js +124 -0
- package/dist/packages/session/src/index.d.ts +4 -0
- package/dist/packages/session/src/index.js +2 -0
- package/dist/packages/session/src/memory-log.d.ts +16 -0
- package/dist/packages/session/src/memory-log.js +27 -0
- package/dist/packages/session/src/projection.d.ts +2 -0
- package/dist/packages/session/src/projection.js +88 -0
- package/dist/packages/session/src/tree.d.ts +13 -0
- package/dist/packages/session/src/tree.js +35 -0
- package/dist/sdk/cli.d.ts +2 -0
- package/dist/sdk/cli.js +118 -0
- package/dist/sdk/index.d.ts +10 -0
- package/dist/sdk/index.js +6 -0
- package/licenses/codex-Apache-2.0.txt +201 -0
- package/licenses/gitleaks-MIT.txt +21 -0
- package/licenses/pi-mono-MIT.txt +21 -0
- package/package.json +74 -4
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
const textOf = (content) => content
|
|
2
|
+
.map((part) => (part.type === "text" ? part.text : part.type === "image" ? "[image]" : ""))
|
|
3
|
+
.join("")
|
|
4
|
+
.trim();
|
|
5
|
+
const cut = (text, max) => (text.length > max ? `${text.slice(0, max)}… [cut, ${text.length - max} more characters]` : text);
|
|
6
|
+
const oneLine = (text) => text.replace(/\s+/g, " ").trim();
|
|
7
|
+
export function summarize(id, name, status, entries) {
|
|
8
|
+
let turns = 0;
|
|
9
|
+
let tokens = 0;
|
|
10
|
+
let lastActivity;
|
|
11
|
+
for (const entry of entries) {
|
|
12
|
+
lastActivity = entry.ts;
|
|
13
|
+
if (entry.type === "message" && entry.message.role === "assistant")
|
|
14
|
+
turns++;
|
|
15
|
+
const usage = entry.type === "message" ? entry.message.usage : entry.type === "usage" ? entry.usage : undefined;
|
|
16
|
+
if (usage)
|
|
17
|
+
tokens += usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
|
|
18
|
+
}
|
|
19
|
+
return { id, name, status, turns, tokens, lastActivity };
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Turns another agent's log into something a model can read: one status line, then the latest items
|
|
23
|
+
* as short lines. Raw log lines are never shown. This is for the parent agent, not for people.
|
|
24
|
+
*/
|
|
25
|
+
export function renderChild(summary, entries, options) {
|
|
26
|
+
const max = options.maxChars ?? 600;
|
|
27
|
+
const lines = [];
|
|
28
|
+
const callNames = new Map();
|
|
29
|
+
for (const entry of entries) {
|
|
30
|
+
if (entry.type !== "message")
|
|
31
|
+
continue;
|
|
32
|
+
const { message } = entry;
|
|
33
|
+
if (message.role === "user")
|
|
34
|
+
lines.push(`user: ${cut(oneLine(textOf(message.content)), max)}`);
|
|
35
|
+
else if (message.role === "assistant") {
|
|
36
|
+
for (const part of message.content) {
|
|
37
|
+
if (part.type === "text" && part.text.trim())
|
|
38
|
+
lines.push(`assistant: ${cut(oneLine(part.text), max)}`);
|
|
39
|
+
if (part.type === "provider" && options.thinking) {
|
|
40
|
+
const reasoning = part.data?.reasoning;
|
|
41
|
+
if (reasoning)
|
|
42
|
+
lines.push(`thinking: ${cut(oneLine(reasoning), 300)}`);
|
|
43
|
+
}
|
|
44
|
+
if (part.type === "tool_call") {
|
|
45
|
+
callNames.set(part.id, { name: part.name, args: part.args });
|
|
46
|
+
lines.push(`tool ${part.name} ${cut(oneLine(JSON.stringify(part.args ?? {})), 200)}`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
else if (message.role === "tool") {
|
|
51
|
+
const call = callNames.get(message.toolCallId ?? "");
|
|
52
|
+
lines.push(`tool result${call ? ` (${call.name})` : ""}${message.isError ? " [error]" : ""}: ${cut(oneLine(textOf(message.content)), max)}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
const shown = lines.slice(-Math.max(1, options.last));
|
|
56
|
+
const when = summary.lastActivity ? new Date(summary.lastActivity).toISOString() : "never";
|
|
57
|
+
const header = `agent ${summary.id} (${summary.name}): ${summary.status}, ${summary.turns} turns, ${summary.tokens} tokens, last activity ${when}`;
|
|
58
|
+
const omitted = lines.length - shown.length;
|
|
59
|
+
return [header, ...(omitted > 0 ? [`(${omitted} earlier items not shown)`] : []), ...shown].join("\n");
|
|
60
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import type { Environment, Log, Plugin } from "../../../kernel/src/index.js";
|
|
2
|
+
export interface ContextOptions {
|
|
3
|
+
/** Load the AGENTS.md in the working directory. Default true. */
|
|
4
|
+
cwd?: boolean;
|
|
5
|
+
/** Path of a user-level AGENTS.md (for example ~/.config/agent/AGENTS.md). Off by default. */
|
|
6
|
+
userFile?: string | false;
|
|
7
|
+
fileName?: string;
|
|
8
|
+
/** Longer files are cut with a note. Default 32 KB. */
|
|
9
|
+
maxBytes?: number;
|
|
10
|
+
}
|
|
11
|
+
export interface ContextPlugin extends Plugin {
|
|
12
|
+
/**
|
|
13
|
+
* Read the files again and, if they changed, add a new pinned message that replaces the earlier
|
|
14
|
+
* instructions. Called by hand, and by the kernel after every compaction; nothing else re-reads files.
|
|
15
|
+
*/
|
|
16
|
+
reload(context: {
|
|
17
|
+
log: Log;
|
|
18
|
+
env: Environment;
|
|
19
|
+
}): Promise<boolean>;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Puts project instructions (AGENTS.md) at the start of the conversation as a pinned message, once.
|
|
23
|
+
* They survive compaction and are not summarised. Only the AGENTS.md name is understood; there is no
|
|
24
|
+
* parent-directory search, no per-directory loading and no @path includes.
|
|
25
|
+
*/
|
|
26
|
+
export declare function contextPlugin(options?: ContextOptions): ContextPlugin;
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
const CUSTOM_TYPE = "agents_md";
|
|
2
|
+
async function readSource(env, source, maxBytes) {
|
|
3
|
+
const info = await env.stat(source.path);
|
|
4
|
+
if (!info?.isFile)
|
|
5
|
+
return undefined;
|
|
6
|
+
const bytes = await env.readFile(source.path);
|
|
7
|
+
const text = new TextDecoder().decode(bytes.length > maxBytes ? bytes.subarray(0, maxBytes) : bytes).trim();
|
|
8
|
+
if (text === "")
|
|
9
|
+
return undefined;
|
|
10
|
+
const note = bytes.length > maxBytes ? `\n\n[The file was cut after ${maxBytes} bytes.]` : "";
|
|
11
|
+
return `Instructions from ${source.label} (${source.path}):\n\n${text}${note}`;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Puts project instructions (AGENTS.md) at the start of the conversation as a pinned message, once.
|
|
15
|
+
* They survive compaction and are not summarised. Only the AGENTS.md name is understood; there is no
|
|
16
|
+
* parent-directory search, no per-directory loading and no @path includes.
|
|
17
|
+
*/
|
|
18
|
+
export function contextPlugin(options = {}) {
|
|
19
|
+
const fileName = options.fileName ?? "AGENTS.md";
|
|
20
|
+
const maxBytes = options.maxBytes ?? 32 * 1024;
|
|
21
|
+
async function load(env) {
|
|
22
|
+
const sources = [];
|
|
23
|
+
if (options.userFile)
|
|
24
|
+
sources.push({ label: "the user's global file", path: options.userFile });
|
|
25
|
+
if (options.cwd !== false)
|
|
26
|
+
sources.push({ label: "this project", path: env.resolve(fileName) });
|
|
27
|
+
const parts = [];
|
|
28
|
+
for (const source of sources) {
|
|
29
|
+
const part = await readSource(env, source, maxBytes);
|
|
30
|
+
if (part)
|
|
31
|
+
parts.push(part);
|
|
32
|
+
}
|
|
33
|
+
return parts.length > 0 ? parts.join("\n\n---\n\n") : undefined;
|
|
34
|
+
}
|
|
35
|
+
const pinnedTexts = async (log) => (await log.branch()).flatMap((entry) => entry.type === "custom_message" && entry.customType === CUSTOM_TYPE ? [entry.content.map((part) => (part.type === "text" ? part.text : "")).join("")] : []);
|
|
36
|
+
/** Reads the files again; if they changed, adds a new pinned message that replaces the earlier instructions. */
|
|
37
|
+
async function reloadInstructions({ log, env }) {
|
|
38
|
+
const earlier = await pinnedTexts(log);
|
|
39
|
+
const text = await load(env);
|
|
40
|
+
if (text === undefined || text === earlier.at(-1)?.replace(/^These instructions replace all earlier project instructions\.\n\n/, ""))
|
|
41
|
+
return false;
|
|
42
|
+
await log.append({
|
|
43
|
+
type: "custom_message",
|
|
44
|
+
customType: CUSTOM_TYPE,
|
|
45
|
+
pinned: true,
|
|
46
|
+
content: [{ type: "text", text: `These instructions replace all earlier project instructions.\n\n${text}` }],
|
|
47
|
+
});
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return {
|
|
51
|
+
name: "context",
|
|
52
|
+
compensates: "models start every session knowing nothing about the project's rules",
|
|
53
|
+
setup: () => undefined,
|
|
54
|
+
async prepare({ log, env }) {
|
|
55
|
+
if ((await pinnedTexts(log)).length > 0)
|
|
56
|
+
return;
|
|
57
|
+
const text = await load(env);
|
|
58
|
+
if (text)
|
|
59
|
+
await log.append({ type: "custom_message", customType: CUSTOM_TYPE, pinned: true, content: [{ type: "text", text }] });
|
|
60
|
+
},
|
|
61
|
+
async afterCompaction(context) {
|
|
62
|
+
await reloadInstructions(context);
|
|
63
|
+
},
|
|
64
|
+
reload: (context) => reloadInstructions(context),
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { UsageTotals } from "../../../kernel/src/index.js";
|
|
2
|
+
/** Prices in US dollars per million tokens. */
|
|
3
|
+
export interface Prices {
|
|
4
|
+
input: number;
|
|
5
|
+
output: number;
|
|
6
|
+
cacheRead?: number;
|
|
7
|
+
cacheWrite?: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function costOf(totals: UsageTotals, prices: Prices): number;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import type { Plugin, ProviderErrorKind } from "../../../kernel/src/index.js";
|
|
2
|
+
import type { Prices } from "./cost.js";
|
|
3
|
+
export type { Prices } from "./cost.js";
|
|
4
|
+
export interface Budget {
|
|
5
|
+
/** Stop after this many model replies in the session. */
|
|
6
|
+
maxTurns?: number;
|
|
7
|
+
/** Stop after this many tokens in the session (input, output and cache together). */
|
|
8
|
+
maxTokens?: number;
|
|
9
|
+
/** Stop after this much spend in the session; needs `prices`. */
|
|
10
|
+
maxCostUsd?: number;
|
|
11
|
+
/** Stop a single run that has been going this long. */
|
|
12
|
+
maxRunSeconds?: number;
|
|
13
|
+
}
|
|
14
|
+
export interface LimitsOptions extends Budget {
|
|
15
|
+
prices?: Prices;
|
|
16
|
+
/** Provider errors that pause the run instead of ending it. Default: quota and rate_limit. */
|
|
17
|
+
pauseOn?: ProviderErrorKind[];
|
|
18
|
+
now?: () => number;
|
|
19
|
+
}
|
|
20
|
+
export interface LimitsPlugin extends Plugin {
|
|
21
|
+
/** Change the budget, for example before running a paused session again. */
|
|
22
|
+
update(budget: Budget): void;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Spending limits for a session. Unlimited unless configured. Usage is read from the log by the kernel,
|
|
26
|
+
* so limits keep counting across restarts.
|
|
27
|
+
*/
|
|
28
|
+
export declare function limitsPlugin(options?: LimitsOptions): LimitsPlugin;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { costOf } from "./cost.js";
|
|
2
|
+
const totalTokens = (totals) => totals.input + totals.output + totals.cacheRead + totals.cacheWrite;
|
|
3
|
+
/**
|
|
4
|
+
* Spending limits for a session. Unlimited unless configured. Usage is read from the log by the kernel,
|
|
5
|
+
* so limits keep counting across restarts.
|
|
6
|
+
*/
|
|
7
|
+
export function limitsPlugin(options = {}) {
|
|
8
|
+
const budget = { ...options };
|
|
9
|
+
const now = options.now ?? Date.now;
|
|
10
|
+
const pauseOn = new Set(options.pauseOn ?? ["quota", "rate_limit"]);
|
|
11
|
+
let runStartedAt = now();
|
|
12
|
+
return {
|
|
13
|
+
name: "limits",
|
|
14
|
+
compensates: "models do not know what they cost",
|
|
15
|
+
update: (next) => Object.assign(budget, next),
|
|
16
|
+
setup(api) {
|
|
17
|
+
api.on("event", (event) => {
|
|
18
|
+
if (event.type === "run_start")
|
|
19
|
+
runStartedAt = now();
|
|
20
|
+
});
|
|
21
|
+
api.on("turn.end", ({ totals }) => {
|
|
22
|
+
const exceeded = (budget.maxTurns !== undefined && totals.turns >= budget.maxTurns) ||
|
|
23
|
+
(budget.maxTokens !== undefined && totalTokens(totals) >= budget.maxTokens) ||
|
|
24
|
+
(budget.maxCostUsd !== undefined && options.prices !== undefined && costOf(totals, options.prices) >= budget.maxCostUsd) ||
|
|
25
|
+
(budget.maxRunSeconds !== undefined && now() - runStartedAt >= budget.maxRunSeconds * 1000);
|
|
26
|
+
return exceeded ? { pause: "budget" } : undefined;
|
|
27
|
+
});
|
|
28
|
+
api.on("error", (error) => (pauseOn.has(error.kind) ? { pause: "quota" } : undefined));
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
export interface SecretRule {
|
|
2
|
+
id: string;
|
|
3
|
+
regex: RegExp;
|
|
4
|
+
secretGroup: number;
|
|
5
|
+
entropy: number;
|
|
6
|
+
keywords: string[];
|
|
7
|
+
}
|
|
8
|
+
export declare function builtinRules(): SecretRule[];
|
|
9
|
+
export declare function shannonEntropy(text: string): number;
|
|
10
|
+
export declare const PLACEHOLDER: RegExp;
|
|
11
|
+
export interface Engine {
|
|
12
|
+
mask(text: string): string;
|
|
13
|
+
restore(text: string): string;
|
|
14
|
+
/** Number of secrets seen so far. */
|
|
15
|
+
size(): number;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* Finds secrets and swaps each for a placeholder like {{AWS_ACCESS_TOKEN_1a2b3c4d}}. The same secret always gets
|
|
19
|
+
* the same placeholder (an HMAC of the secret under a key), so the text sent to the model stays stable between requests.
|
|
20
|
+
*/
|
|
21
|
+
export declare function createEngine(options?: {
|
|
22
|
+
key?: string;
|
|
23
|
+
rules?: SecretRule[];
|
|
24
|
+
}): Engine;
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { createHmac, randomBytes } from "node:crypto";
|
|
2
|
+
import { gitleaksRules } from "./rules.gen.js";
|
|
3
|
+
/** A few rules of our own, for things the gitleaks table does not describe well. */
|
|
4
|
+
const ownRules = [
|
|
5
|
+
{ id: "private-key-block", regex: /-----BEGIN [A-Z ]*PRIVATE KEY(?: BLOCK)?-----[\s\S]+?-----END [A-Z ]*PRIVATE KEY(?: BLOCK)?-----/dg, secretGroup: 0, entropy: 0, keywords: ["private key"] },
|
|
6
|
+
{ id: "bearer-token", regex: /\bBearer\s+([A-Za-z0-9\-._~+/]{20,}=*)/dgi, secretGroup: 1, entropy: 3, keywords: ["bearer"] },
|
|
7
|
+
{ id: "url-credentials", regex: /[a-z][a-z0-9+.-]*:\/\/[^/\s:@]+:([^/\s@]{3,})@/dgi, secretGroup: 1, entropy: 0, keywords: ["://"] },
|
|
8
|
+
{ id: "sendgrid-style-key", regex: /\bSG\.[A-Za-z0-9_-]{16,32}\.[A-Za-z0-9_-]{16,64}\b/dg, secretGroup: 0, entropy: 3, keywords: ["sg."] },
|
|
9
|
+
{ id: "openai-style-key", regex: /\bsk-(?:proj-|svcacct-|ant-[a-z0-9]+-)?[A-Za-z0-9_-]{32,}\b/dg, secretGroup: 0, entropy: 3, keywords: ["sk-"] },
|
|
10
|
+
{ id: "mailgun-style-key", regex: /\bkey-[0-9a-f]{32}\b/dg, secretGroup: 0, entropy: 0, keywords: ["key-"] },
|
|
11
|
+
{ id: "jwt", regex: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b/dg, secretGroup: 0, entropy: 0, keywords: ["eyj"] },
|
|
12
|
+
{
|
|
13
|
+
id: "env-secret-assignment",
|
|
14
|
+
regex: /^[ \t]*(?:export[ \t]+)?[A-Z0-9_]*(?:SECRET|TOKEN|PASSWORD|PASSWD|API_?KEY|PRIVATE_KEY)[A-Z0-9_]*[ \t]*=[ \t]*['"]?([^\s'"]{8,})/dgm,
|
|
15
|
+
secretGroup: 1,
|
|
16
|
+
entropy: 0,
|
|
17
|
+
keywords: ["secret", "token", "password", "passwd", "api_key", "apikey", "private_key"],
|
|
18
|
+
},
|
|
19
|
+
];
|
|
20
|
+
export function builtinRules() {
|
|
21
|
+
const generated = gitleaksRules.map((rule) => ({
|
|
22
|
+
id: rule.id,
|
|
23
|
+
regex: new RegExp(rule.regex, `${rule.flags}gd`),
|
|
24
|
+
secretGroup: rule.secretGroup,
|
|
25
|
+
entropy: rule.entropy,
|
|
26
|
+
keywords: rule.keywords,
|
|
27
|
+
}));
|
|
28
|
+
return [...ownRules, ...generated];
|
|
29
|
+
}
|
|
30
|
+
export function shannonEntropy(text) {
|
|
31
|
+
if (text.length === 0)
|
|
32
|
+
return 0;
|
|
33
|
+
const counts = new Map();
|
|
34
|
+
for (const char of text)
|
|
35
|
+
counts.set(char, (counts.get(char) ?? 0) + 1);
|
|
36
|
+
let entropy = 0;
|
|
37
|
+
for (const count of counts.values()) {
|
|
38
|
+
const p = count / text.length;
|
|
39
|
+
entropy -= p * Math.log2(p);
|
|
40
|
+
}
|
|
41
|
+
return entropy;
|
|
42
|
+
}
|
|
43
|
+
export const PLACEHOLDER = /\{\{[A-Z0-9_]+_[0-9a-f]{8}\}\}/g;
|
|
44
|
+
const kindOf = (id) => id
|
|
45
|
+
.toUpperCase()
|
|
46
|
+
.replace(/[^A-Z0-9]+/g, "_")
|
|
47
|
+
.replace(/^_+|_+$/g, "");
|
|
48
|
+
/**
|
|
49
|
+
* Finds secrets and swaps each for a placeholder like {{AWS_ACCESS_TOKEN_1a2b3c4d}}. The same secret always gets
|
|
50
|
+
* the same placeholder (an HMAC of the secret under a key), so the text sent to the model stays stable between requests.
|
|
51
|
+
*/
|
|
52
|
+
export function createEngine(options = {}) {
|
|
53
|
+
const key = options.key ?? randomBytes(16).toString("hex");
|
|
54
|
+
// a rule you write may leave out the flags the engine relies on (d: where the secret is, g: every match) or write its keywords in capitals
|
|
55
|
+
const rules = (options.rules ?? builtinRules()).map((rule) => ({
|
|
56
|
+
...rule,
|
|
57
|
+
regex: rule.regex instanceof RegExp ? new RegExp(rule.regex.source, [...new Set(`${rule.regex.flags}dg`)].join("")) : rule.regex, // anything else is left to fail when it is used, and a failure closes the request
|
|
58
|
+
keywords: rule.keywords.map((keyword) => keyword.toLowerCase()),
|
|
59
|
+
}));
|
|
60
|
+
const secrets = new Map();
|
|
61
|
+
const placeholderFor = (kind, secret) => {
|
|
62
|
+
const digest = createHmac("sha256", key).update(secret).digest("hex").slice(0, 8);
|
|
63
|
+
const placeholder = `{{${kind}_${digest}}}`;
|
|
64
|
+
secrets.set(placeholder, secret);
|
|
65
|
+
return placeholder;
|
|
66
|
+
};
|
|
67
|
+
function mask(text) {
|
|
68
|
+
if (text.length < 8)
|
|
69
|
+
return text;
|
|
70
|
+
const lower = text.toLowerCase();
|
|
71
|
+
const protectedRanges = [...text.matchAll(PLACEHOLDER)].map((match) => [match.index ?? 0, (match.index ?? 0) + match[0].length]);
|
|
72
|
+
const spans = [];
|
|
73
|
+
for (const rule of rules) {
|
|
74
|
+
if (rule.keywords.length > 0 && !rule.keywords.some((keyword) => lower.includes(keyword)))
|
|
75
|
+
continue;
|
|
76
|
+
rule.regex.lastIndex = 0;
|
|
77
|
+
for (const match of text.matchAll(rule.regex)) {
|
|
78
|
+
const indices = match.indices;
|
|
79
|
+
const range = indices?.[rule.secretGroup] ?? indices?.[0];
|
|
80
|
+
if (!range)
|
|
81
|
+
continue;
|
|
82
|
+
const secret = text.slice(range[0], range[1]);
|
|
83
|
+
if (secret.length < 6 || /^(.)\1+$/.test(secret))
|
|
84
|
+
continue;
|
|
85
|
+
if (rule.entropy > 0 && shannonEntropy(secret) < rule.entropy)
|
|
86
|
+
continue;
|
|
87
|
+
if (protectedRanges.some(([start, end]) => range[0] < end && range[1] > start))
|
|
88
|
+
continue;
|
|
89
|
+
spans.push({ start: range[0], end: range[1], kind: kindOf(rule.id), secret });
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
if (spans.length === 0)
|
|
93
|
+
return text;
|
|
94
|
+
spans.sort((a, b) => a.start - b.start || b.end - b.start - (a.end - a.start));
|
|
95
|
+
let out = "";
|
|
96
|
+
let cursor = 0;
|
|
97
|
+
for (const span of spans) {
|
|
98
|
+
if (span.start < cursor)
|
|
99
|
+
continue; // overlaps an earlier, larger-or-equal span
|
|
100
|
+
out += text.slice(cursor, span.start) + placeholderFor(span.kind, span.secret);
|
|
101
|
+
cursor = span.end;
|
|
102
|
+
}
|
|
103
|
+
return out + text.slice(cursor);
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
mask,
|
|
107
|
+
restore: (text) => (text.includes("{{") ? text.replace(PLACEHOLDER, (placeholder) => secrets.get(placeholder) ?? placeholder) : text),
|
|
108
|
+
size: () => secrets.size,
|
|
109
|
+
};
|
|
110
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { LlmClient, Plugin } from "../../../kernel/src/index.js";
|
|
2
|
+
import type { SecretRule } from "./engine.js";
|
|
3
|
+
export interface RedactionOptions {
|
|
4
|
+
/** Key for the placeholders. A fixed key keeps placeholders the same across restarts (better prompt caching). */
|
|
5
|
+
key?: string;
|
|
6
|
+
/** Replaces the built-in rules (the gitleaks table plus a few of our own). */
|
|
7
|
+
rules?: SecretRule[];
|
|
8
|
+
}
|
|
9
|
+
export interface RedactionPlugin extends Plugin {
|
|
10
|
+
/** Swap secrets in a piece of text for placeholders, for logs and the like. */
|
|
11
|
+
scrub(text: string): string;
|
|
12
|
+
restore(text: string): string;
|
|
13
|
+
/**
|
|
14
|
+
* A client that masks every request before it is sent. For models that are not the agent's own (the reviewer, for example), whose
|
|
15
|
+
* requests do not pass the agent's `request` hook. Only the request is masked: nothing real ever comes back from such a model.
|
|
16
|
+
*/
|
|
17
|
+
wrap(llm: LlmClient): LlmClient;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Keeps secrets away from the model. Every request is scanned; secrets become stable placeholders, and the model's
|
|
21
|
+
* reply (text and tool arguments) gets the real values back before anything runs or is saved. If scanning fails the
|
|
22
|
+
* request is not sent. Off unless you add it.
|
|
23
|
+
*/
|
|
24
|
+
export declare function redactionPlugin(options?: RedactionOptions): RedactionPlugin;
|
|
25
|
+
export type { SecretRule } from "./engine.js";
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { FatalHookError } from "../../../kernel/src/index.js";
|
|
2
|
+
import { createEngine } from "./engine.js";
|
|
3
|
+
import { createRestorer } from "./restore.js";
|
|
4
|
+
const EXPLANATION = `<redaction>
|
|
5
|
+
Text such as {{KIND_xxxxxxxx}} stands for a secret that was redacted before it reached you. When you need such a value in a command, file or message, copy the placeholder exactly as written; the real value is restored when the command runs. Never guess, complete or alter a placeholder, and treat it as a real secret when judging risk.
|
|
6
|
+
</redaction>`;
|
|
7
|
+
const MAX_REMEMBERED = 5000;
|
|
8
|
+
/**
|
|
9
|
+
* Keeps secrets away from the model. Every request is scanned; secrets become stable placeholders, and the model's
|
|
10
|
+
* reply (text and tool arguments) gets the real values back before anything runs or is saved. If scanning fails the
|
|
11
|
+
* request is not sent. Off unless you add it.
|
|
12
|
+
*/
|
|
13
|
+
export function redactionPlugin(options = {}) {
|
|
14
|
+
const engine = createEngine({ ...(options.key ? { key: options.key } : {}), ...(options.rules ? { rules: options.rules } : {}) });
|
|
15
|
+
// Text that was sent before keeps the form it was sent in, so earlier parts of the request never change.
|
|
16
|
+
const frozen = new Map();
|
|
17
|
+
const maskText = (text) => {
|
|
18
|
+
const known = frozen.get(text);
|
|
19
|
+
if (known !== undefined)
|
|
20
|
+
return known;
|
|
21
|
+
const masked = engine.mask(text);
|
|
22
|
+
if (frozen.size >= MAX_REMEMBERED)
|
|
23
|
+
frozen.delete(frozen.keys().next().value);
|
|
24
|
+
frozen.set(text, masked);
|
|
25
|
+
return masked;
|
|
26
|
+
};
|
|
27
|
+
const maskDeep = (value) => {
|
|
28
|
+
if (typeof value === "string")
|
|
29
|
+
return maskText(value);
|
|
30
|
+
if (Array.isArray(value))
|
|
31
|
+
return value.map(maskDeep);
|
|
32
|
+
if (value && typeof value === "object")
|
|
33
|
+
return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, maskDeep(inner)]));
|
|
34
|
+
return value;
|
|
35
|
+
};
|
|
36
|
+
const maskPart = (part) => {
|
|
37
|
+
if (part.type === "text")
|
|
38
|
+
return { ...part, text: maskText(part.text) };
|
|
39
|
+
if (part.type === "tool_call")
|
|
40
|
+
return { ...part, args: maskDeep(part.args) };
|
|
41
|
+
if (part.type === "provider")
|
|
42
|
+
return { ...part, data: maskDeep(part.data) };
|
|
43
|
+
return part;
|
|
44
|
+
};
|
|
45
|
+
function maskRequest(request) {
|
|
46
|
+
const messages = request.messages.map((message) => ({ ...message, content: message.content.map(maskPart) }));
|
|
47
|
+
return { ...request, system: `${maskText(request.system)}\n\n${EXPLANATION}`, messages };
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
name: "redaction",
|
|
51
|
+
compensates: "secrets that reach a model provider cannot be taken back",
|
|
52
|
+
scrub: maskText,
|
|
53
|
+
restore: engine.restore,
|
|
54
|
+
wrap: (llm) => ({ model: llm.model, stream: (request, signal) => llm.stream(maskRequest(request), signal) }),
|
|
55
|
+
setup(api) {
|
|
56
|
+
api.on("request", (request) => {
|
|
57
|
+
try {
|
|
58
|
+
return { request: maskRequest(request), transform: createRestorer(engine.restore) };
|
|
59
|
+
}
|
|
60
|
+
catch (error) {
|
|
61
|
+
throw new FatalHookError(`redaction failed, so the request was not sent (${error instanceof Error ? error.message : String(error)})`);
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
},
|
|
65
|
+
};
|
|
66
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import type { EventTransform } from "../../../kernel/src/index.js";
|
|
2
|
+
/**
|
|
3
|
+
* Puts the real values back into the model's reply while it streams. A placeholder can arrive split across
|
|
4
|
+
* pieces, so text that might be the start of one is held back until it is complete or clearly is not one.
|
|
5
|
+
*/
|
|
6
|
+
export declare function createRestorer(restore: (text: string) => string): EventTransform;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Where to start holding text back: the end of `text` could be the beginning of a placeholder that is not complete yet. */
|
|
2
|
+
function holdBackFrom(text) {
|
|
3
|
+
const last = text.lastIndexOf("{");
|
|
4
|
+
if (last === -1)
|
|
5
|
+
return text.length;
|
|
6
|
+
const start = last > 0 && text[last - 1] === "{" ? last - 1 : last;
|
|
7
|
+
const tail = text.slice(start);
|
|
8
|
+
return tail.length <= 80 && /^\{(?:\{[A-Za-z0-9_]*\}?)?$/.test(tail) ? start : text.length;
|
|
9
|
+
}
|
|
10
|
+
function restoreDeep(value, restore) {
|
|
11
|
+
if (typeof value === "string")
|
|
12
|
+
return restore(value);
|
|
13
|
+
if (Array.isArray(value))
|
|
14
|
+
return value.map((item) => restoreDeep(item, restore));
|
|
15
|
+
if (value && typeof value === "object")
|
|
16
|
+
return Object.fromEntries(Object.entries(value).map(([key, inner]) => [key, restoreDeep(inner, restore)]));
|
|
17
|
+
return value;
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* Puts the real values back into the model's reply while it streams. A placeholder can arrive split across
|
|
21
|
+
* pieces, so text that might be the start of one is held back until it is complete or clearly is not one.
|
|
22
|
+
*/
|
|
23
|
+
export function createRestorer(restore) {
|
|
24
|
+
let pending = "";
|
|
25
|
+
return {
|
|
26
|
+
push(event) {
|
|
27
|
+
if (event.type === "text_delta") {
|
|
28
|
+
pending += event.delta;
|
|
29
|
+
const cut = holdBackFrom(pending);
|
|
30
|
+
const ready = pending.slice(0, cut);
|
|
31
|
+
pending = pending.slice(cut);
|
|
32
|
+
return ready ? [{ type: "text_delta", delta: restore(ready) }] : [];
|
|
33
|
+
}
|
|
34
|
+
if (event.type === "tool_call")
|
|
35
|
+
return [{ type: "tool_call", call: { ...event.call, args: restoreDeep(event.call.args, restore) } }];
|
|
36
|
+
return [event];
|
|
37
|
+
},
|
|
38
|
+
flush() {
|
|
39
|
+
const rest = pending;
|
|
40
|
+
pending = "";
|
|
41
|
+
return rest ? [{ type: "text_delta", delta: restore(rest) }] : [];
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface GeneratedRule {
|
|
2
|
+
id: string;
|
|
3
|
+
regex: string;
|
|
4
|
+
flags: string;
|
|
5
|
+
/** Which capture group holds the secret (0 = the whole match). */
|
|
6
|
+
secretGroup: number;
|
|
7
|
+
/** Minimum Shannon entropy of the secret (0 = no check). */
|
|
8
|
+
entropy: number;
|
|
9
|
+
/** At least one of these (lower case) must appear in the text before the rule is tried. */
|
|
10
|
+
keywords: string[];
|
|
11
|
+
}
|
|
12
|
+
export declare const gitleaksRules: GeneratedRule[];
|
|
13
|
+
/** Rules from the table that were left out because their regular expressions do not translate to JavaScript. */
|
|
14
|
+
export declare const skippedGitleaksRules: string[];
|