sandoichi 0.1.4 → 0.1.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/index.mjs +2 -0
- package/package.json +1 -1
- package/src/core.mjs +19 -6
- package/src/hook-cli.mjs +14 -7
- package/src/redaction-config.mjs +104 -0
- package/src/redaction-profile.mjs +163 -0
- package/src/secret-redaction.mjs +12 -16
- package/src/semantic-compactor.mjs +14 -5
package/README.md
CHANGED
|
@@ -10,6 +10,20 @@ npm install sandoichi
|
|
|
10
10
|
import { optimizeToolOutput, createProviderProxy } from 'sandoichi';
|
|
11
11
|
```
|
|
12
12
|
|
|
13
|
+
Project-specific detectors can be declared in `.sando/redaction.json`:
|
|
14
|
+
|
|
15
|
+
```json
|
|
16
|
+
{
|
|
17
|
+
"schema": "sando-redaction/v1",
|
|
18
|
+
"rules": [
|
|
19
|
+
{ "type": "assignment-key", "key": "DATABASE_URL" },
|
|
20
|
+
{ "type": "token-prefix", "prefix": "acme_", "minLength": 24, "maxLength": 128 }
|
|
21
|
+
]
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Built-ins stay enabled. Profiles are declarative and local to the current project; invalid profiles fail visibly.
|
|
26
|
+
|
|
13
27
|
The library requires Node.js `>=22.22.0 <23` and has no runtime dependencies. Installing it does not install or enable the plugin.
|
|
14
28
|
|
|
15
29
|
For plugin installation, see the [main project README](https://github.com/yuzushi-dev/Sando#readme).
|
package/index.mjs
CHANGED
|
@@ -5,6 +5,8 @@ export {
|
|
|
5
5
|
normalizePolicy,
|
|
6
6
|
optimizeToolOutput,
|
|
7
7
|
} from './src/core.mjs';
|
|
8
|
+
export { createRedactionProfile } from './src/redaction-profile.mjs';
|
|
9
|
+
export { loadProjectRedactionProfile } from './src/redaction-config.mjs';
|
|
8
10
|
export { detectProviderBody, listSemanticCandidates, transformProviderRequest } from './src/context-transform.mjs';
|
|
9
11
|
export {
|
|
10
12
|
buildSemanticPrompt,
|
package/package.json
CHANGED
package/src/core.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
|
|
3
3
|
import { planToolRoute, ROUTING_POLICY_VERSION } from './routing.mjs';
|
|
4
|
-
import {
|
|
4
|
+
import { loadProjectRedactionProfile } from './redaction-config.mjs';
|
|
5
5
|
|
|
6
6
|
const DEFAULT_POLICY = Object.freeze({
|
|
7
7
|
mode: 'apply', maxInlineBytes: 4096, maxArtifactBytes: 65536, headBytes: undefined, tailBytes: undefined,
|
|
@@ -31,6 +31,14 @@ function textOutput(output) {
|
|
|
31
31
|
return value;
|
|
32
32
|
}
|
|
33
33
|
|
|
34
|
+
function resolveRedactionProfile(cwd, candidate) {
|
|
35
|
+
const profile = candidate ?? loadProjectRedactionProfile(cwd).profile;
|
|
36
|
+
if (!profile || typeof profile.redact !== 'function' || typeof profile.digest !== 'string') {
|
|
37
|
+
throw new TypeError('redactionProfile is invalid');
|
|
38
|
+
}
|
|
39
|
+
return profile;
|
|
40
|
+
}
|
|
41
|
+
|
|
34
42
|
function truncateUtf8(text, maxBytes) {
|
|
35
43
|
if (Buffer.byteLength(text) <= maxBytes) return text;
|
|
36
44
|
let bytes = 0;
|
|
@@ -140,7 +148,7 @@ export function normalizePolicy(policy = {}) {
|
|
|
140
148
|
|
|
141
149
|
export function optimizeToolOutput({
|
|
142
150
|
toolName, output, cwd, policy, selector, raw, lineCount, fileBytes, prose, summarizeProse,
|
|
143
|
-
summarizeEnabled, grepScope, outputBytes, toolInput,
|
|
151
|
+
summarizeEnabled, grepScope, outputBytes, toolInput, redactionProfile,
|
|
144
152
|
} = {}) {
|
|
145
153
|
if (typeof toolName !== 'string' || !toolName.trim() || toolName.length > 128) throw new Error('toolName is invalid');
|
|
146
154
|
if (typeof cwd !== 'string' || !cwd) throw new Error('cwd is invalid');
|
|
@@ -154,7 +162,8 @@ export function optimizeToolOutput({
|
|
|
154
162
|
lineCount: derivedLineCount, fileBytes: derivedFileBytes, prose, summarizeProse, summarizeEnabled, grepScope,
|
|
155
163
|
outputBytes: outputBytes ?? Buffer.byteLength(input),
|
|
156
164
|
});
|
|
157
|
-
const
|
|
165
|
+
const profile = normalizedPolicy.redact ? resolveRedactionProfile(cwd, redactionProfile) : null;
|
|
166
|
+
const redacted = profile ? profile.redact(input) : { text: input, count: 0 };
|
|
158
167
|
let modelText = name === 'bash' && normalizedPolicy.maxColumns >= 32
|
|
159
168
|
? collapseRepeatedLines(redacted.text)
|
|
160
169
|
: redacted.text;
|
|
@@ -192,7 +201,7 @@ export function optimizeToolOutput({
|
|
|
192
201
|
sourceBytes,
|
|
193
202
|
truncated: false,
|
|
194
203
|
};
|
|
195
|
-
const header = `artifact ${artifact.ref} ${artifact.bytes}B\n`;
|
|
204
|
+
const header = `[sando] artifact ${artifact.ref} ${artifact.bytes}B\n`;
|
|
196
205
|
const viewBudget = Math.max(1, routePolicy.maxInlineBytes - Buffer.byteLength(header));
|
|
197
206
|
inline = `${truncateUtf8(header, routePolicy.maxInlineBytes)}${inlineView(
|
|
198
207
|
modelText,
|
|
@@ -214,7 +223,10 @@ export function optimizeToolOutput({
|
|
|
214
223
|
redactions: redacted.count,
|
|
215
224
|
artifactTruncated: artifact?.truncated ?? false,
|
|
216
225
|
};
|
|
217
|
-
const result = {
|
|
226
|
+
const result = {
|
|
227
|
+
inline, route: route.route, reason: route.source, policyVersion: ROUTING_POLICY_VERSION,
|
|
228
|
+
redactionProfileDigest: profile?.digest ?? null, stats,
|
|
229
|
+
};
|
|
218
230
|
if (artifact) result.artifact = artifact;
|
|
219
231
|
return result;
|
|
220
232
|
}
|
|
@@ -252,7 +264,8 @@ export function createReceipt({ host, event, optimization, replacement } = {}) {
|
|
|
252
264
|
sessionId: event.sessionId ?? null, inputDigest: sha256(textOutput(event.output)),
|
|
253
265
|
inlineDigest: sha256(textOutput(replacement === undefined ? optimization.inline : replacement)), artifactRef: optimization.artifact?.ref ?? null,
|
|
254
266
|
route: optimization.route ?? 'passthrough', reason: optimization.reason ?? 'spike-default',
|
|
255
|
-
policyVersion: optimization.policyVersion ?? ROUTING_POLICY_VERSION,
|
|
267
|
+
policyVersion: optimization.policyVersion ?? ROUTING_POLICY_VERSION,
|
|
268
|
+
redactionProfileDigest: optimization.redactionProfileDigest ?? null, stats: optimization.stats,
|
|
256
269
|
};
|
|
257
270
|
return { ...body, digest: sha256(stableJson(body)) };
|
|
258
271
|
}
|
package/src/hook-cli.mjs
CHANGED
|
@@ -3,6 +3,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
|
|
5
5
|
import { createReceipt, normalizeEvent, normalizePolicy, optimizeToolOutput } from './core.mjs';
|
|
6
|
+
import { loadProjectRedactionProfile } from './redaction-config.mjs';
|
|
6
7
|
import { defaultMetricsPath, recordMetrics } from './metrics.mjs';
|
|
7
8
|
import {
|
|
8
9
|
closeFinishedDays, defaultTelemetryConfigPath, defaultTelemetryStatePaths, incrementCounter, readTelemetryConfig,
|
|
@@ -87,7 +88,8 @@ export function runHookCli({ host, env = process.env } = {}) {
|
|
|
87
88
|
const eventName = input.hook_event_name ?? input.hookEventName ?? input.event_name ?? input.eventName;
|
|
88
89
|
if (eventName === 'PostToolUse') {
|
|
89
90
|
const event = normalizeEvent(input);
|
|
90
|
-
const
|
|
91
|
+
const redactionProfile = policy.redact ? loadProjectRedactionProfile(event.cwd).profile : undefined;
|
|
92
|
+
const optimization = optimizeToolOutput({ toolName: event.toolName, toolInput: event.toolInput, output: event.output, cwd: event.cwd, policy, redactionProfile });
|
|
91
93
|
let shaped;
|
|
92
94
|
if (host === 'claude' && policy.mode === 'apply') {
|
|
93
95
|
shaped = shapeForClaude({
|
|
@@ -95,7 +97,7 @@ export function runHookCli({ host, env = process.env } = {}) {
|
|
|
95
97
|
optimization,
|
|
96
98
|
toolName: event.toolName,
|
|
97
99
|
toolInput: event.toolInput,
|
|
98
|
-
cwd: event.cwd,
|
|
100
|
+
cwd: event.cwd, redactionProfile,
|
|
99
101
|
policy,
|
|
100
102
|
});
|
|
101
103
|
}
|
|
@@ -117,7 +119,12 @@ export function runHookCli({ host, env = process.env } = {}) {
|
|
|
117
119
|
}
|
|
118
120
|
}
|
|
119
121
|
}
|
|
120
|
-
} catch {
|
|
122
|
+
} catch (error) {
|
|
123
|
+
if (error?.code === 'SANDO_REDACTION_CONFIG') {
|
|
124
|
+
process.stderr.write(`sando invalid redaction config: ${error.message}\n`);
|
|
125
|
+
process.exitCode = 2;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
121
128
|
process.stdout.write('{}\n');
|
|
122
129
|
}
|
|
123
130
|
|
|
@@ -135,19 +142,19 @@ export function buildCodexFallback({ optimization, cwd }) {
|
|
|
135
142
|
};
|
|
136
143
|
}
|
|
137
144
|
|
|
138
|
-
function shapeForClaude({ original, optimization, toolName, toolInput, cwd, policy }) {
|
|
145
|
+
function shapeForClaude({ original, optimization, toolName, toolInput, cwd, policy, redactionProfile }) {
|
|
139
146
|
if (typeof original === 'string') return materialize(optimization, cwd);
|
|
140
147
|
if (!original || typeof original !== 'object' || Array.isArray(original)
|
|
141
148
|
|| !Object.hasOwn(original, 'stdout') || !Object.hasOwn(original, 'stderr')
|
|
142
149
|
|| typeof original.stdout !== 'string' || typeof original.stderr !== 'string'
|
|
143
150
|
|| (Object.hasOwn(original, 'interrupted') && typeof original.interrupted !== 'boolean')
|
|
144
151
|
|| (Object.hasOwn(original, 'isImage') && typeof original.isImage !== 'boolean')) return undefined;
|
|
145
|
-
const result = { ...original };
|
|
152
|
+
const result = policy.redact ? redactionProfile.redactStructured(original).value : { ...original };
|
|
146
153
|
if (typeof original.stdout === 'string') {
|
|
147
|
-
result.stdout = materialize(optimizeToolOutput({ toolName, toolInput, output: original.stdout, cwd, policy }), cwd);
|
|
154
|
+
result.stdout = materialize(optimizeToolOutput({ toolName, toolInput, output: original.stdout, cwd, policy, redactionProfile }), cwd);
|
|
148
155
|
}
|
|
149
156
|
if (typeof original.stderr === 'string') {
|
|
150
|
-
result.stderr = materialize(optimizeToolOutput({ toolName, toolInput, output: original.stderr, cwd, policy }), cwd);
|
|
157
|
+
result.stderr = materialize(optimizeToolOutput({ toolName, toolInput, output: original.stderr, cwd, policy, redactionProfile }), cwd);
|
|
151
158
|
}
|
|
152
159
|
return result;
|
|
153
160
|
}
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
|
|
4
|
+
import { createRedactionProfile } from './redaction-profile.mjs';
|
|
5
|
+
|
|
6
|
+
const MAX_CONFIG_BYTES = 64 * 1024;
|
|
7
|
+
const cache = new Map();
|
|
8
|
+
const builtInProfile = createRedactionProfile([]);
|
|
9
|
+
|
|
10
|
+
function lstatIfPresent(target) {
|
|
11
|
+
try {
|
|
12
|
+
return fs.lstatSync(target);
|
|
13
|
+
} catch (error) {
|
|
14
|
+
if (error?.code === 'ENOENT') return null;
|
|
15
|
+
throw error;
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function readBoundedFile(configPath, expectedStat) {
|
|
20
|
+
const flags = fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW ?? 0);
|
|
21
|
+
const descriptor = fs.openSync(configPath, flags);
|
|
22
|
+
try {
|
|
23
|
+
const stat = fs.fstatSync(descriptor);
|
|
24
|
+
if (!stat.isFile()) throw new Error(`redaction config is not a regular file: ${configPath}`);
|
|
25
|
+
if (stat.dev !== expectedStat.dev || stat.ino !== expectedStat.ino) {
|
|
26
|
+
throw new Error(`redaction config changed while opening: ${configPath}`);
|
|
27
|
+
}
|
|
28
|
+
if (stat.size > MAX_CONFIG_BYTES) {
|
|
29
|
+
throw new Error(`redaction config exceeds 64 KiB: ${configPath}`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
const content = Buffer.alloc(MAX_CONFIG_BYTES + 1);
|
|
33
|
+
let bytesRead = 0;
|
|
34
|
+
while (bytesRead < content.length) {
|
|
35
|
+
const count = fs.readSync(descriptor, content, bytesRead, content.length - bytesRead, null);
|
|
36
|
+
if (count === 0) break;
|
|
37
|
+
bytesRead += count;
|
|
38
|
+
}
|
|
39
|
+
if (bytesRead > MAX_CONFIG_BYTES) {
|
|
40
|
+
throw new Error(`redaction config exceeds 64 KiB: ${configPath}`);
|
|
41
|
+
}
|
|
42
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(content.subarray(0, bytesRead));
|
|
43
|
+
} finally {
|
|
44
|
+
fs.closeSync(descriptor);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function parseConfig(source, configPath) {
|
|
49
|
+
let config;
|
|
50
|
+
try {
|
|
51
|
+
config = JSON.parse(source);
|
|
52
|
+
} catch (error) {
|
|
53
|
+
throw new Error(`invalid JSON in redaction config ${configPath}: ${error.message}`, { cause: error });
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
if (!config || typeof config !== 'object' || Array.isArray(config)) {
|
|
57
|
+
throw new Error(`invalid redaction config schema: ${configPath}`);
|
|
58
|
+
}
|
|
59
|
+
const keys = Object.keys(config).sort();
|
|
60
|
+
if (keys.length !== 2 || keys[0] !== 'rules' || keys[1] !== 'schema'
|
|
61
|
+
|| config.schema !== 'sando-redaction/v1' || !Array.isArray(config.rules)) {
|
|
62
|
+
throw new Error(`invalid redaction config schema: ${configPath}`);
|
|
63
|
+
}
|
|
64
|
+
return config;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export function loadProjectRedactionProfile(cwd) {
|
|
68
|
+
try {
|
|
69
|
+
return loadProjectRedactionProfileUnsafe(cwd);
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error?.code === 'SANDO_REDACTION_CONFIG') throw error;
|
|
72
|
+
const wrapped = new Error(error instanceof Error ? error.message : String(error), { cause: error });
|
|
73
|
+
wrapped.code = 'SANDO_REDACTION_CONFIG';
|
|
74
|
+
throw wrapped;
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function loadProjectRedactionProfileUnsafe(cwd) {
|
|
79
|
+
const configDirectory = path.resolve(cwd, '.sando');
|
|
80
|
+
const directoryStat = lstatIfPresent(configDirectory);
|
|
81
|
+
if (!directoryStat) return { profile: builtInProfile, path: null };
|
|
82
|
+
if (directoryStat.isSymbolicLink()) {
|
|
83
|
+
throw new Error(`redaction config directory must not be a symlink: ${configDirectory}`);
|
|
84
|
+
}
|
|
85
|
+
if (!directoryStat.isDirectory()) {
|
|
86
|
+
throw new Error(`redaction config directory is not a directory: ${configDirectory}`);
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const configPath = path.join(configDirectory, 'redaction.json');
|
|
90
|
+
const stat = lstatIfPresent(configPath);
|
|
91
|
+
if (!stat) return { profile: builtInProfile, path: null };
|
|
92
|
+
if (stat.isSymbolicLink()) throw new Error(`redaction config must not be a symlink: ${configPath}`);
|
|
93
|
+
if (!stat.isFile()) throw new Error(`redaction config is not a regular file: ${configPath}`);
|
|
94
|
+
if (stat.size > MAX_CONFIG_BYTES) throw new Error(`redaction config exceeds 64 KiB: ${configPath}`);
|
|
95
|
+
|
|
96
|
+
const cached = cache.get(configPath);
|
|
97
|
+
if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) return cached.result;
|
|
98
|
+
|
|
99
|
+
const source = readBoundedFile(configPath, stat);
|
|
100
|
+
const config = parseConfig(source, configPath);
|
|
101
|
+
const result = { profile: createRedactionProfile(config.rules), path: configPath };
|
|
102
|
+
cache.set(configPath, { mtimeMs: stat.mtimeMs, size: stat.size, result });
|
|
103
|
+
return result;
|
|
104
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
|
|
3
|
+
const BUILT_IN_MATCHERS = Object.freeze([
|
|
4
|
+
[/-----BEGIN [A-Z ]+ KEY-----[\s\S]*?-----END [A-Z ]+ KEY-----/g, '[REDACTED PRIVATE KEY]'],
|
|
5
|
+
[/(authorization\s*[:=]\s*(?!(?:bearer\s+)?\[REDACTED(?: (?:PRIVATE KEY|TOKEN))?\](?=$|[\s,"'}]))(?:bearer\s+)?)[^\s,"'}]+/gi,
|
|
6
|
+
(_match, prefix) => `${prefix}[REDACTED]`],
|
|
7
|
+
[/(["']?(?:api[_-]?key|access[_-]?token|password|secret|private[_-]?key)["']?\s*[:=]\s*["']?)(?!\[REDACTED(?: (?:PRIVATE KEY|TOKEN))?\](?=$|[\s,"'}]))[^\s,"'}]+/gi,
|
|
8
|
+
(_match, prefix) => `${prefix}[REDACTED]`],
|
|
9
|
+
[/\b(?:sk|rk)-[A-Za-z0-9_-]{12,}\b/g, '[REDACTED TOKEN]'],
|
|
10
|
+
[/\bgh[pousr]_[A-Za-z0-9_-]{12,}\b/g, '[REDACTED TOKEN]'],
|
|
11
|
+
[/\bgithub_pat_[A-Za-z0-9_-]{20,}\b/g, '[REDACTED TOKEN]'],
|
|
12
|
+
[/\bAKIA[0-9A-Z]{16}\b/g, '[REDACTED TOKEN]'],
|
|
13
|
+
]);
|
|
14
|
+
const PROFILE_SCHEMA = 'sando-redaction-profile-v1';
|
|
15
|
+
const MAX_CUSTOM_RULES = 100;
|
|
16
|
+
const MAX_NAME_LENGTH = 64;
|
|
17
|
+
const MAX_TOKEN_LENGTH = 4096;
|
|
18
|
+
|
|
19
|
+
function escapeRegExp(value) {
|
|
20
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function compareCanonical(left, right) {
|
|
24
|
+
const a = JSON.stringify(left);
|
|
25
|
+
const b = JSON.stringify(right);
|
|
26
|
+
return a < b ? -1 : a > b ? 1 : 0;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function normalizeRules(customRules) {
|
|
30
|
+
if (!Array.isArray(customRules)) throw new TypeError('customRules must be an array');
|
|
31
|
+
if (customRules.length > MAX_CUSTOM_RULES) throw new TypeError('customRules must contain at most 100 rules');
|
|
32
|
+
const identities = new Set();
|
|
33
|
+
const normalized = customRules.map((rule) => {
|
|
34
|
+
if (!rule || typeof rule !== 'object' || Array.isArray(rule)
|
|
35
|
+
|| ![Object.prototype, null].includes(Object.getPrototypeOf(rule))) {
|
|
36
|
+
throw new TypeError('each custom rule must be a plain object');
|
|
37
|
+
}
|
|
38
|
+
let result;
|
|
39
|
+
let identity;
|
|
40
|
+
if (rule.type === 'assignment-key') {
|
|
41
|
+
const allowed = new Set(['type', 'key']);
|
|
42
|
+
if (Reflect.ownKeys(rule).some((field) => typeof field !== 'string' || !allowed.has(field))
|
|
43
|
+
|| typeof rule.key !== 'string'
|
|
44
|
+
|| !new RegExp(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,${MAX_NAME_LENGTH - 1}}$`).test(rule.key)) {
|
|
45
|
+
throw new TypeError('invalid assignment-key rule');
|
|
46
|
+
}
|
|
47
|
+
const key = rule.key.toLowerCase();
|
|
48
|
+
result = { type: rule.type, key };
|
|
49
|
+
identity = `${rule.type}:${key}`;
|
|
50
|
+
} else if (rule.type === 'token-prefix') {
|
|
51
|
+
const allowed = new Set(['type', 'prefix', 'minLength', 'maxLength']);
|
|
52
|
+
if (Reflect.ownKeys(rule).some((field) => typeof field !== 'string' || !allowed.has(field))
|
|
53
|
+
|| typeof rule.prefix !== 'string'
|
|
54
|
+
|| !new RegExp(`^[\\x21-\\x7e]{1,${MAX_NAME_LENGTH}}$`).test(rule.prefix)) {
|
|
55
|
+
throw new TypeError('invalid token-prefix rule');
|
|
56
|
+
}
|
|
57
|
+
const minLength = Object.hasOwn(rule, 'minLength') ? rule.minLength : 12;
|
|
58
|
+
const maxLength = Object.hasOwn(rule, 'maxLength') ? rule.maxLength : null;
|
|
59
|
+
if (!Number.isSafeInteger(minLength) || minLength < 1 || minLength > MAX_TOKEN_LENGTH
|
|
60
|
+
|| (maxLength !== null
|
|
61
|
+
&& (!Number.isSafeInteger(maxLength) || maxLength < minLength || maxLength > MAX_TOKEN_LENGTH))
|
|
62
|
+
|| (Object.hasOwn(rule, 'maxLength') && rule.maxLength === null)) {
|
|
63
|
+
throw new TypeError('invalid token-prefix lengths');
|
|
64
|
+
}
|
|
65
|
+
result = { type: rule.type, prefix: rule.prefix, minLength, maxLength };
|
|
66
|
+
identity = `${rule.type}:${rule.prefix}`;
|
|
67
|
+
} else {
|
|
68
|
+
throw new TypeError('unsupported custom rule type');
|
|
69
|
+
}
|
|
70
|
+
if (identities.has(identity)) throw new TypeError('duplicate custom rule');
|
|
71
|
+
identities.add(identity);
|
|
72
|
+
return result;
|
|
73
|
+
});
|
|
74
|
+
return normalized.sort(compareCanonical);
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function customMatchers(rules) {
|
|
78
|
+
const assignments = [];
|
|
79
|
+
const tokens = [];
|
|
80
|
+
for (const rule of rules) {
|
|
81
|
+
if (rule.type === 'assignment-key') {
|
|
82
|
+
const key = escapeRegExp(rule.key);
|
|
83
|
+
assignments.push([
|
|
84
|
+
new RegExp(`(?<![A-Za-z0-9_.-])(["']?${key}["']?\\s*[:=]\\s*["']?)(?!\\[REDACTED\\](?=$|[\\s,"'}]))[^\\s,"'}]+`, 'gi'),
|
|
85
|
+
(_match, prefix) => `${prefix}[REDACTED]`,
|
|
86
|
+
]);
|
|
87
|
+
} else {
|
|
88
|
+
const maximum = rule.maxLength === null ? '' : rule.maxLength;
|
|
89
|
+
tokens.push([
|
|
90
|
+
new RegExp(`(?<![A-Za-z0-9_-])${escapeRegExp(rule.prefix)}[A-Za-z0-9_-]{${rule.minLength},${maximum}}(?![A-Za-z0-9_-])`, 'g'),
|
|
91
|
+
'[REDACTED]',
|
|
92
|
+
]);
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
return { assignments, tokens };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function applyMatchers(text, matchers) {
|
|
99
|
+
let count = 0;
|
|
100
|
+
for (const [pattern, replacement] of matchers) {
|
|
101
|
+
text = text.replace(pattern, (...args) => {
|
|
102
|
+
count += 1;
|
|
103
|
+
return typeof replacement === 'function' ? replacement(...args) : replacement;
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
return { text, count };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export function createRedactionProfile(customRules = []) {
|
|
110
|
+
const rules = normalizeRules(customRules);
|
|
111
|
+
const custom = customMatchers(rules);
|
|
112
|
+
const matchers = [
|
|
113
|
+
...BUILT_IN_MATCHERS.slice(0, 3),
|
|
114
|
+
...custom.assignments,
|
|
115
|
+
...BUILT_IN_MATCHERS.slice(3),
|
|
116
|
+
...custom.tokens,
|
|
117
|
+
];
|
|
118
|
+
const digest = `sha256:${createHash('sha256')
|
|
119
|
+
.update(JSON.stringify({ schema: PROFILE_SCHEMA, customRules: rules }))
|
|
120
|
+
.digest('hex')}`;
|
|
121
|
+
const redact = (text) => {
|
|
122
|
+
if (typeof text !== 'string') throw new TypeError('text must be a string');
|
|
123
|
+
return applyMatchers(text, matchers);
|
|
124
|
+
};
|
|
125
|
+
const redactStructured = (value) => {
|
|
126
|
+
const ancestors = new Set();
|
|
127
|
+
const visit = (item) => {
|
|
128
|
+
if (typeof item === 'string') {
|
|
129
|
+
const result = redact(item);
|
|
130
|
+
return { value: result.text, count: result.count };
|
|
131
|
+
}
|
|
132
|
+
if (item === null || typeof item === 'boolean' || typeof item === 'number') {
|
|
133
|
+
return { value: item, count: 0 };
|
|
134
|
+
}
|
|
135
|
+
if (typeof item !== 'object'
|
|
136
|
+
|| (!Array.isArray(item) && ![Object.prototype, null].includes(Object.getPrototypeOf(item)))) {
|
|
137
|
+
throw new TypeError('value must contain only plain objects, arrays, and JSON primitives');
|
|
138
|
+
}
|
|
139
|
+
if (ancestors.has(item)) throw new TypeError('value must not be cyclic');
|
|
140
|
+
ancestors.add(item);
|
|
141
|
+
let count = 0;
|
|
142
|
+
const entries = Object.entries(item).map(([key, child]) => {
|
|
143
|
+
const result = visit(child);
|
|
144
|
+
count += result.count;
|
|
145
|
+
return [key, result.value];
|
|
146
|
+
});
|
|
147
|
+
ancestors.delete(item);
|
|
148
|
+
return {
|
|
149
|
+
value: Array.isArray(item) ? entries.map(([, child]) => child) : Object.fromEntries(entries),
|
|
150
|
+
count,
|
|
151
|
+
};
|
|
152
|
+
};
|
|
153
|
+
return visit(value);
|
|
154
|
+
};
|
|
155
|
+
return {
|
|
156
|
+
digest,
|
|
157
|
+
redact,
|
|
158
|
+
redactStructured,
|
|
159
|
+
hasSecret(text) {
|
|
160
|
+
return redact(text).count > 0;
|
|
161
|
+
},
|
|
162
|
+
};
|
|
163
|
+
}
|
package/src/secret-redaction.mjs
CHANGED
|
@@ -1,21 +1,17 @@
|
|
|
1
|
+
import { createRedactionProfile } from './redaction-profile.mjs';
|
|
2
|
+
|
|
3
|
+
const builtInProfile = createRedactionProfile();
|
|
4
|
+
|
|
5
|
+
export { createRedactionProfile };
|
|
6
|
+
|
|
1
7
|
export function redact(text) {
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
});
|
|
8
|
-
};
|
|
9
|
-
replace(/-----BEGIN [A-Z ]+ KEY-----[\s\S]*?-----END [A-Z ]+ KEY-----/g, '[REDACTED PRIVATE KEY]');
|
|
10
|
-
replace(/\b(?:sk|rk)-[A-Za-z0-9_-]{12,}\b/g, '[REDACTED TOKEN]');
|
|
11
|
-
replace(/\bgh[pousr]_[A-Za-z0-9_-]{12,}\b/g, '[REDACTED TOKEN]');
|
|
12
|
-
replace(/\bgithub_pat_[A-Za-z0-9_-]{20,}\b/g, '[REDACTED TOKEN]');
|
|
13
|
-
replace(/\bAKIA[0-9A-Z]{16}\b/g, '[REDACTED TOKEN]');
|
|
14
|
-
replace(/(authorization\s*[:=]\s*(?:bearer\s+)?)[^\s,"'}]+/gi, (_match, prefix) => `${prefix}[REDACTED]`);
|
|
15
|
-
replace(/(["']?(?:api[_-]?key|access[_-]?token|password|secret|private[_-]?key)["']?\s*[:=]\s*["']?)[^\s,"'}]+/gi, (_match, prefix) => `${prefix}[REDACTED]`);
|
|
16
|
-
return { text, count };
|
|
8
|
+
return builtInProfile.redact(text);
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function redactStructured(value, enabled = true) {
|
|
12
|
+
return enabled ? builtInProfile.redactStructured(value).value : value;
|
|
17
13
|
}
|
|
18
14
|
|
|
19
15
|
export function hasSecret(text) {
|
|
20
|
-
return
|
|
16
|
+
return builtInProfile.hasSecret(text);
|
|
21
17
|
}
|
|
@@ -60,7 +60,9 @@ export function buildSemanticPrompt({ provider, model, toolName, text, requiredF
|
|
|
60
60
|
].join('\n');
|
|
61
61
|
}
|
|
62
62
|
|
|
63
|
-
export function validateSemanticSummary({
|
|
63
|
+
export function validateSemanticSummary({
|
|
64
|
+
originalText, summary, requiredFacts = [], maxSummaryRatio = DEFAULT_POLICY.maxSummaryRatio, redactionProfile,
|
|
65
|
+
} = {}) {
|
|
64
66
|
if (typeof originalText !== 'string' || typeof summary !== 'string') {
|
|
65
67
|
return { valid: false, reason: 'invalid-text' };
|
|
66
68
|
}
|
|
@@ -77,7 +79,9 @@ export function validateSemanticSummary({ originalText, summary, requiredFacts =
|
|
|
77
79
|
}
|
|
78
80
|
const missing = required.find((fact) => !summary.includes(fact));
|
|
79
81
|
if (missing) return { valid: false, reason: 'missing-required-fact', missing, inputTokens, outputTokens };
|
|
80
|
-
if (hasSecret(summary)
|
|
82
|
+
if ((redactionProfile ? redactionProfile.hasSecret(summary) : hasSecret(summary))) {
|
|
83
|
+
return { valid: false, reason: 'secret-detected', inputTokens, outputTokens };
|
|
84
|
+
}
|
|
81
85
|
return { valid: true, inputTokens, outputTokens };
|
|
82
86
|
}
|
|
83
87
|
|
|
@@ -111,12 +115,15 @@ async function withTimeout(complete, request, timeoutMs) {
|
|
|
111
115
|
}
|
|
112
116
|
}
|
|
113
117
|
|
|
114
|
-
export function createSemanticCompactor({ complete, cache = new Map(), policy } = {}) {
|
|
118
|
+
export function createSemanticCompactor({ complete, cache = new Map(), policy, redactionProfile } = {}) {
|
|
115
119
|
if (typeof complete !== 'function') throw new TypeError('semantic compactor complete callback is required');
|
|
116
120
|
if (!cache || typeof cache.get !== 'function' || typeof cache.set !== 'function') {
|
|
117
121
|
throw new TypeError('semantic compactor cache must implement get and set');
|
|
118
122
|
}
|
|
119
123
|
const options = validatePolicy(policy);
|
|
124
|
+
if (redactionProfile && (typeof redactionProfile.redact !== 'function' || typeof redactionProfile.hasSecret !== 'function')) {
|
|
125
|
+
throw new TypeError('redactionProfile is invalid');
|
|
126
|
+
}
|
|
120
127
|
|
|
121
128
|
return async function compact({ provider, model, toolName, text, historical = true, isError = false, requiredFacts = [] } = {}) {
|
|
122
129
|
if (typeof text !== 'string') throw new TypeError('semantic compactor text must be a string');
|
|
@@ -136,8 +143,8 @@ export function createSemanticCompactor({ complete, cache = new Map(), policy }
|
|
|
136
143
|
if (isError) return { ...base, status: 'skipped', reason: 'error-result' };
|
|
137
144
|
if (inputTokens < options.minInputTokens) return { ...base, status: 'skipped', reason: 'below-threshold' };
|
|
138
145
|
|
|
139
|
-
const safe = redact(text);
|
|
140
|
-
const safeFacts = required.map((fact) => redact(fact).text);
|
|
146
|
+
const safe = redactionProfile ? redactionProfile.redact(text) : redact(text);
|
|
147
|
+
const safeFacts = required.map((fact) => (redactionProfile ? redactionProfile.redact(fact) : redact(fact)).text);
|
|
141
148
|
const prompt = buildSemanticPrompt({ provider, model, toolName, text: safe.text, requiredFacts: safeFacts });
|
|
142
149
|
const key = cacheKey({ provider, model, toolName, text: safe.text, requiredFacts: safeFacts });
|
|
143
150
|
const started = Date.now();
|
|
@@ -147,6 +154,7 @@ export function createSemanticCompactor({ complete, cache = new Map(), policy }
|
|
|
147
154
|
const validation = validateSemanticSummary({
|
|
148
155
|
originalText: text, summary: cached.summary, requiredFacts: required,
|
|
149
156
|
maxSummaryRatio: options.maxSummaryRatio,
|
|
157
|
+
redactionProfile,
|
|
150
158
|
});
|
|
151
159
|
if (validation.valid) {
|
|
152
160
|
const grossSavedTokens = inputTokens - validation.outputTokens;
|
|
@@ -191,6 +199,7 @@ export function createSemanticCompactor({ complete, cache = new Map(), policy }
|
|
|
191
199
|
const validation = validateSemanticSummary({
|
|
192
200
|
originalText: text, summary, requiredFacts: required,
|
|
193
201
|
maxSummaryRatio: options.maxSummaryRatio,
|
|
202
|
+
redactionProfile,
|
|
194
203
|
});
|
|
195
204
|
if (!validation.valid) return { ...base, reason: validation.reason, latencyMs: Date.now() - started };
|
|
196
205
|
|