switchroom 0.19.35 → 0.19.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/skill-validate-pretool.mjs +15 -2
- package/dist/cli/switchroom.js +672 -441
- package/dist/host-control/main.js +149 -3
- package/package.json +4 -2
- package/profiles/_base/start.sh.hbs +37 -12
- package/telegram-plugin/dist/gateway/gateway.js +311 -93
- package/telegram-plugin/format.ts +70 -15
- package/telegram-plugin/gateway/gateway.ts +22 -29
- package/telegram-plugin/gateway/ipc-server.ts +18 -15
- package/telegram-plugin/gateway/model-command.ts +34 -0
- package/telegram-plugin/gateway/outbound-send-path.ts +12 -1
- package/telegram-plugin/operator-events.ts +40 -16
- package/telegram-plugin/secret-detect/db-uri.ts +90 -0
- package/telegram-plugin/secret-detect/index.ts +24 -1
- package/telegram-plugin/secret-detect/inert-values.ts +147 -0
- package/telegram-plugin/secret-detect/kv-scanner.ts +108 -0
- package/telegram-plugin/secret-detect/patterns.ts +24 -4
- package/telegram-plugin/tests/format-consistency.test.ts +93 -0
- package/telegram-plugin/tests/gateway-session-model-relaunch.test.ts +40 -2
- package/telegram-plugin/tests/ipc-server-validate-operator.test.ts +20 -11
- package/telegram-plugin/tests/outbound-send-path.test.ts +1 -1
- package/telegram-plugin/tests/secret-detect-cross-engine.test.ts +263 -0
- package/telegram-plugin/tests/secret-detect-write-path.test.ts +403 -0
- package/telegram-plugin/tests/turn-flush-safety.test.ts +2 -2
- package/vendor/hindsight-memory/scripts/lib/client.py +58 -0
- package/vendor/hindsight-memory/scripts/lib/secret_patterns.json +431 -0
- package/vendor/hindsight-memory/scripts/lib/secret_redact.py +563 -0
- package/vendor/hindsight-memory/scripts/lib/secret_redaction_vectors.json +397 -0
- package/vendor/hindsight-memory/scripts/tests/test_secret_redact.py +522 -0
|
@@ -550,6 +550,9 @@ import {
|
|
|
550
550
|
handleModelCommand,
|
|
551
551
|
classifyModelSwitchConfirmation,
|
|
552
552
|
formatModelRelaunchDiagLog,
|
|
553
|
+
parseModelSwitchTarget,
|
|
554
|
+
resolveSessionModelResolutionTimeoutMs,
|
|
555
|
+
waitForSessionModelResolution,
|
|
553
556
|
servedModelMatchesRequested,
|
|
554
557
|
buildServedModelDivergenceHandler,
|
|
555
558
|
deliverModelSwitchBootNotice,
|
|
@@ -23665,23 +23668,26 @@ async function startGateway(): Promise<void> { // #2996 P0c: the boot IIFE, now
|
|
|
23665
23668
|
}
|
|
23666
23669
|
} catch {}
|
|
23667
23670
|
|
|
23668
|
-
//
|
|
23669
|
-
//
|
|
23670
|
-
// start.sh writes the EFFECTIVE launched model to `.active-session-model`
|
|
23671
|
-
// on every boot (the model actually passed to `claude --model`). Re-hydrate
|
|
23672
|
-
// the in-memory session-model override from it so `/status` and the welcome
|
|
23673
|
-
// card stay honest after a session-relaunch restart. Only treat it as an
|
|
23674
|
-
// override when it differs from the configured/default model — a plain boot
|
|
23675
|
-
// on the configured model leaves the override null.
|
|
23676
|
-
//
|
|
23677
|
-
// Also consume the `.session-model-alert` sentinel: start.sh drops it when
|
|
23678
|
-
// it had to DROP an sr-* override because LiteLLM was unreachable at boot
|
|
23679
|
-
// (booting on the configured default instead of 4xx-ing against Anthropic).
|
|
23680
|
-
// We turn it into a loud Telegram message to the operator, then delete it.
|
|
23671
|
+
// Rehydrate only after start.sh publishes this boot's resolved outputs.
|
|
23681
23672
|
try {
|
|
23682
23673
|
const smAgentDir = resolveAgentDirFromEnv()
|
|
23683
23674
|
if (smAgentDir) {
|
|
23684
|
-
const
|
|
23675
|
+
const resolutionTimeoutMs = resolveSessionModelResolutionTimeoutMs(
|
|
23676
|
+
process.env.SWITCHROOM_SESSION_MODEL_RESOLUTION_TIMEOUT_MS,
|
|
23677
|
+
)
|
|
23678
|
+
const resolved = await waitForSessionModelResolution({
|
|
23679
|
+
barrierExists: () => existsSync(join(smAgentDir, '.session-model-resolved')),
|
|
23680
|
+
timeoutMs: resolutionTimeoutMs,
|
|
23681
|
+
})
|
|
23682
|
+
if (!resolved) {
|
|
23683
|
+
const target = modelSwitchReason == null
|
|
23684
|
+
? '(none)'
|
|
23685
|
+
: (parseModelSwitchTarget(modelSwitchReason) ?? '(unknown)')
|
|
23686
|
+
process.stderr.write(
|
|
23687
|
+
`telegram gateway: gw /model relaunch UNRESOLVED agent=${getMyAgentName()} target=${target} (barrier timeout after ${resolutionTimeoutMs}ms)\n`,
|
|
23688
|
+
)
|
|
23689
|
+
} else {
|
|
23690
|
+
const activePath = join(smAgentDir, '.active-session-model')
|
|
23685
23691
|
if (existsSync(activePath)) {
|
|
23686
23692
|
try {
|
|
23687
23693
|
const launched = readFileSync(activePath, 'utf8').trim()
|
|
@@ -23697,21 +23703,9 @@ async function startGateway(): Promise<void> { // #2996 P0c: the boot IIFE, now
|
|
|
23697
23703
|
// a phantom session override.
|
|
23698
23704
|
return resolveMainModel(raw ?? undefined)
|
|
23699
23705
|
})()
|
|
23700
|
-
//
|
|
23701
|
-
// landed" signal (F1): the carrier is consume-once, so a launched
|
|
23702
|
-
// model that differs from the configured default only ever
|
|
23703
|
-
// happens on a genuine apply-boot. Seed the in-memory override
|
|
23704
|
-
// from it — this is the ONLY success reporting, sourced from the
|
|
23705
|
-
// real post-boot signal (`.active-session-model`), never from a
|
|
23706
|
-
// scraped pane or an optimistic record.
|
|
23706
|
+
// A launched model differing from configured is a genuine apply boot.
|
|
23707
23707
|
const isApplyBoot = launched.length > 0 && launched !== configured
|
|
23708
|
-
// { verify: true } (#3427 item 4 / H1): ONLY this site arms the
|
|
23709
|
-
// requested-vs-served tripwire — `launched` IS the token of the
|
|
23710
|
-
// session now serving. Command-time setOverride never arms.
|
|
23711
23708
|
sessionModelSource.setOverride(isApplyBoot ? launched : null, { verify: true })
|
|
23712
|
-
// Boot /model cards (#3427): the divergence tripwire warn and
|
|
23713
|
-
// the switch confirmation share one deps surface; the card
|
|
23714
|
-
// logic lives in model-command.ts (#2996 ratchet).
|
|
23715
23709
|
const modelBootCardDeps: ModelBootCardDeps = {
|
|
23716
23710
|
agent: getMyAgentName(),
|
|
23717
23711
|
chat: modelSwitchMarkerChat,
|
|
@@ -23722,8 +23716,6 @@ async function startGateway(): Promise<void> { // #2996 P0c: the boot IIFE, now
|
|
|
23722
23716
|
if (isApplyBoot) {
|
|
23723
23717
|
sessionModelSource.setDivergenceHandler(buildServedModelDivergenceHandler(modelBootCardDeps))
|
|
23724
23718
|
}
|
|
23725
|
-
// F1/N4: classify + log + one confirmation card. Formatters live
|
|
23726
|
-
// in model-command.ts so this file does not inflate (#2996 ratchet).
|
|
23727
23719
|
const confirmation = modelSwitchReason != null
|
|
23728
23720
|
? classifyModelSwitchConfirmation({
|
|
23729
23721
|
reason: modelSwitchReason,
|
|
@@ -23793,6 +23785,7 @@ async function startGateway(): Promise<void> { // #2996 P0c: the boot IIFE, now
|
|
|
23793
23785
|
process.stderr.write(`telegram gateway: session-model: LiteLLM-down override drop — ${alertText}\n`)
|
|
23794
23786
|
}
|
|
23795
23787
|
}
|
|
23788
|
+
}
|
|
23796
23789
|
}
|
|
23797
23790
|
} catch (err) {
|
|
23798
23791
|
process.stderr.write(`telegram gateway: session-model re-hydration failed: ${(err as Error)?.message ?? String(err)}\n`)
|
|
@@ -25,6 +25,7 @@ import type {
|
|
|
25
25
|
ToolCallResult,
|
|
26
26
|
} from "./ipc-protocol.js";
|
|
27
27
|
import { RICH_MESSAGE_MAX_CHARS } from "../format.js";
|
|
28
|
+
import { OPERATOR_EVENT_KINDS } from "../operator-events.js";
|
|
28
29
|
|
|
29
30
|
export interface IpcServerOptions {
|
|
30
31
|
socketPath: string;
|
|
@@ -196,21 +197,23 @@ type SocketData = { clientId: string; buffer: string };
|
|
|
196
197
|
* data without newline delimiters, which would cause unbounded memory growth. */
|
|
197
198
|
const MAX_BUFFER_SIZE = 1024 * 1024;
|
|
198
199
|
|
|
199
|
-
/** Allowlist of OperatorEventKind values that can arrive over IPC.
|
|
200
|
-
* the
|
|
201
|
-
*
|
|
202
|
-
* taxonomy
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
"
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
200
|
+
/** Allowlist of OperatorEventKind values that can arrive over IPC. DERIVED
|
|
201
|
+
* from the canonical `OPERATOR_EVENT_KINDS` array in
|
|
202
|
+
* `telegram-plugin/operator-events.ts` — the single source of truth for the
|
|
203
|
+
* taxonomy — so the validator can NEVER drift out of sync with it again.
|
|
204
|
+
*
|
|
205
|
+
* History: this used to be a hand-maintained literal Set that missed the
|
|
206
|
+
* `provider-credit-exhausted` / `mcp-dependency-blocked` / `proxy-misconfig`
|
|
207
|
+
* kinds after they were added to the union. The bridge forwarded a
|
|
208
|
+
* `provider-credit-exhausted` operator_event; this validator rejected it as an
|
|
209
|
+
* "invalid IPC message shape" and dropped it, so an OpenRouter/LiteLLM 402
|
|
210
|
+
* credit wall produced NO loud Telegram card (the 2026-07-30 incident).
|
|
211
|
+
* Deriving from the canonical array closes that drift class structurally.
|
|
212
|
+
*
|
|
213
|
+
* `operator-events.ts` is a pure module (its only imports are the pure
|
|
214
|
+
* `format` / `raw-error-scrub` / `model-unavailable` / `provider-credit`
|
|
215
|
+
* leaves), so importing it here introduces no cycle back into the gateway. */
|
|
216
|
+
const VALID_OPERATOR_KINDS = new Set<string>(OPERATOR_EVENT_KINDS);
|
|
214
217
|
|
|
215
218
|
/** Same regex as `assertSafeAgentName` and the op:* callback handler in
|
|
216
219
|
* gateway.ts — keeps every entry-point that touches an agent name on the
|
|
@@ -96,6 +96,40 @@ export function isValidModelArg(arg: string): boolean {
|
|
|
96
96
|
return MODEL_ARG_RE.test(arg)
|
|
97
97
|
}
|
|
98
98
|
|
|
99
|
+
export const DEFAULT_SESSION_MODEL_RESOLUTION_TIMEOUT_MS = 180_000
|
|
100
|
+
export const SESSION_MODEL_RESOLUTION_POLL_MS = 250
|
|
101
|
+
|
|
102
|
+
/** Parse the bounded boot-resolution wait without allowing NaN/negative hangs. */
|
|
103
|
+
export function resolveSessionModelResolutionTimeoutMs(raw: string | undefined): number {
|
|
104
|
+
if (raw == null || raw.trim() === '') return DEFAULT_SESSION_MODEL_RESOLUTION_TIMEOUT_MS
|
|
105
|
+
const value = Number(raw)
|
|
106
|
+
return Number.isFinite(value) && value >= 0
|
|
107
|
+
? value
|
|
108
|
+
: DEFAULT_SESSION_MODEL_RESOLUTION_TIMEOUT_MS
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Asynchronously wait until start.sh atomically publishes this boot's resolved
|
|
113
|
+
* session-model outputs. The deadline is checked on every iteration, including
|
|
114
|
+
* timeout=0, so a missing barrier can never hang gateway startup indefinitely.
|
|
115
|
+
*/
|
|
116
|
+
export async function waitForSessionModelResolution(input: {
|
|
117
|
+
barrierExists: () => boolean
|
|
118
|
+
timeoutMs: number
|
|
119
|
+
pollMs?: number
|
|
120
|
+
sleep?: (ms: number) => Promise<void>
|
|
121
|
+
}): Promise<boolean> {
|
|
122
|
+
const pollMs = input.pollMs ?? SESSION_MODEL_RESOLUTION_POLL_MS
|
|
123
|
+
const sleep = input.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)))
|
|
124
|
+
const startedAt = Date.now()
|
|
125
|
+
for (;;) {
|
|
126
|
+
if (input.barrierExists()) return true
|
|
127
|
+
const remaining = input.timeoutMs - (Date.now() - startedAt)
|
|
128
|
+
if (remaining <= 0) return false
|
|
129
|
+
await sleep(Math.min(pollMs, remaining))
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
99
133
|
/** True when `name` is an sr-* (LiteLLM/OpenRouter) model identifier. */
|
|
100
134
|
export function isSrModel(name: string): boolean {
|
|
101
135
|
return name.startsWith('sr-')
|
|
@@ -191,7 +191,18 @@ export function normalizeOutboundBody(
|
|
|
191
191
|
if (!literalText) text = normalizeParagraphBreaks(text)
|
|
192
192
|
text = redact(text, site)
|
|
193
193
|
if (!literalText) {
|
|
194
|
-
let formatted = stripExcessBold(normalizePunctuation(text))
|
|
194
|
+
let formatted = stripExcessBold(normalizePunctuation(text), (d) => {
|
|
195
|
+
// Observability: the over-bold tripwire silently flattened formatting.
|
|
196
|
+
// Emit one diagnostic line naming the rule + measured ratio so a lost
|
|
197
|
+
// reply is traceable (it previously vanished with no signal).
|
|
198
|
+
try {
|
|
199
|
+
process.stderr.write(
|
|
200
|
+
`telegram gateway: strip-excess-bold: fired site=${site} rule=${d.rule} ratio=${d.ratio.toFixed(3)}\n`,
|
|
201
|
+
)
|
|
202
|
+
} catch {
|
|
203
|
+
// stderr write must never break the send path. Swallow.
|
|
204
|
+
}
|
|
205
|
+
})
|
|
195
206
|
if (addSpacers) formatted = addParagraphSpacers(formatted)
|
|
196
207
|
text = formatted
|
|
197
208
|
// Temporal normalization (#3501): UTC/Zulu → local wall clock, then
|
|
@@ -23,11 +23,32 @@ import {
|
|
|
23
23
|
|
|
24
24
|
// ─── Taxonomy ────────────────────────────────────────────────────────────────
|
|
25
25
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
26
|
+
/**
|
|
27
|
+
* The CANONICAL runtime list of every operator-event kind — the SINGLE source
|
|
28
|
+
* of truth for the taxonomy. The `OperatorEventKind` type is derived from it
|
|
29
|
+
* (below), and every other place that needs the set at RUNTIME (notably the
|
|
30
|
+
* gateway IPC validator's `VALID_OPERATOR_KINDS` in `gateway/ipc-server.ts`)
|
|
31
|
+
* imports THIS array rather than re-declaring a hand-maintained literal.
|
|
32
|
+
*
|
|
33
|
+
* WHY THIS IS AN ARRAY, NOT JUST A TYPE (the drift this closes)
|
|
34
|
+
* ------------------------------------------------------------
|
|
35
|
+
* A TypeScript union erases at compile time, so a runtime allowlist could only
|
|
36
|
+
* mirror it by hand — and it drifted: `ipc-server.ts` shipped a 9-entry Set
|
|
37
|
+
* that never gained `provider-credit-exhausted` / `mcp-dependency-blocked` /
|
|
38
|
+
* `proxy-misconfig` when those kinds were added here. The bridge forwarded a
|
|
39
|
+
* `provider-credit-exhausted` operator_event; the gateway validator rejected it
|
|
40
|
+
* as an "invalid IPC message shape" and DROPPED it — so a real OpenRouter/
|
|
41
|
+
* LiteLLM 402 credit wall produced NO loud Telegram card at all (the 2026-07-30
|
|
42
|
+
* incident). Deriving both the type and the runtime allowlist from this one
|
|
43
|
+
* array makes that class of silent drop structurally impossible.
|
|
44
|
+
*
|
|
45
|
+
* Order is not significant. Adding a kind is a one-line edit HERE.
|
|
46
|
+
*/
|
|
47
|
+
export const OPERATOR_EVENT_KINDS = [
|
|
48
|
+
'credentials-expired',
|
|
49
|
+
'credentials-invalid',
|
|
50
|
+
'proxy-misconfig',
|
|
51
|
+
'credit-exhausted',
|
|
31
52
|
/**
|
|
32
53
|
* A THIRD-PARTY model provider (OpenRouter / OpenAI / Perplexity) reports no
|
|
33
54
|
* credit remaining — HTTP 402 `payment_required`, "insufficient credits",
|
|
@@ -37,7 +58,7 @@ export type OperatorEventKind =
|
|
|
37
58
|
* must not share a card. Operator-actionable (top up in the vendor console),
|
|
38
59
|
* never user-actionable — see {@link OPERATOR_ACTIONABLE_KINDS}.
|
|
39
60
|
*/
|
|
40
|
-
|
|
61
|
+
'provider-credit-exhausted',
|
|
41
62
|
/**
|
|
42
63
|
* A paid MCP dependency (Perplexity, Eraser, Brevo, Postiz, Meta/Google Ads,
|
|
43
64
|
* Cloudflare …) is refusing work because its KEY is the problem — out of
|
|
@@ -49,16 +70,19 @@ export type OperatorEventKind =
|
|
|
49
70
|
* `mcp-credential-failure.ts`. Operator-actionable: only the operator can top
|
|
50
71
|
* up or re-issue a key.
|
|
51
72
|
*/
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
73
|
+
'mcp-dependency-blocked',
|
|
74
|
+
'quota-exhausted',
|
|
75
|
+
'rate-limited',
|
|
76
|
+
'agent-crashed',
|
|
77
|
+
'agent-restarted-unexpectedly',
|
|
78
|
+
'unknown-4xx',
|
|
79
|
+
'unknown-5xx',
|
|
80
|
+
'config-warning',
|
|
81
|
+
'always-allow-persist-failed',
|
|
82
|
+
'mental-model-persist-failed',
|
|
83
|
+
] as const
|
|
84
|
+
|
|
85
|
+
export type OperatorEventKind = (typeof OPERATOR_EVENT_KINDS)[number]
|
|
62
86
|
|
|
63
87
|
export interface OperatorEvent {
|
|
64
88
|
kind: OperatorEventKind
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Database / service connection-URI credential scanner.
|
|
3
|
+
*
|
|
4
|
+
* Gap closed (2026-07 hindsight write-path audit): a connection string
|
|
5
|
+
* like `postgres://appuser:<password>@db.internal:5432/prod` passed the
|
|
6
|
+
* whole detection stack untouched and was stored verbatim in agent
|
|
7
|
+
* memory. `url-redact.ts` only rewrites `http(s)`/`ws(s)`/`ftp` (its
|
|
8
|
+
* `URL_RE` is scheme-limited), and no pattern in `patterns.ts` describes
|
|
9
|
+
* the `scheme://user:pass@host` shape, so a production database password
|
|
10
|
+
* was invisible to every caller of `detectSecrets`.
|
|
11
|
+
*
|
|
12
|
+
* This scanner is deliberately SCHEME-AGNOSTIC — it matches any
|
|
13
|
+
* `<scheme>://<user>:<password>@<host>` — so postgres, mysql, mongodb,
|
|
14
|
+
* redis, amqp, clickhouse, mssql and whatever ships next are all covered
|
|
15
|
+
* without a scheme allowlist to keep current. Precision comes from the
|
|
16
|
+
* shape, not from the scheme list: userinfo-with-password in a URI is a
|
|
17
|
+
* credential by construction.
|
|
18
|
+
*
|
|
19
|
+
* Only the PASSWORD bytes are reported. The username is left intact: it
|
|
20
|
+
* is rarely the secret, and keeping it preserves the diagnostic value of
|
|
21
|
+
* the stored line ("which account", not just "some database").
|
|
22
|
+
*
|
|
23
|
+
* Confidence is `ambiguous`, NOT `high`. `pipeline.ts` auto-writes every
|
|
24
|
+
* high-confidence inbound hit to the vault and the caller DELETES the
|
|
25
|
+
* user's Telegram message; promoting this rule to `high` would change
|
|
26
|
+
* inbound gating as a side effect of a redaction fix. `redact()` masks
|
|
27
|
+
* ambiguous hits, which is what this change is for.
|
|
28
|
+
*/
|
|
29
|
+
import type { RawHit } from './kv-scanner.js'
|
|
30
|
+
|
|
31
|
+
/**
|
|
32
|
+
* `<scheme>://<user>:<password>@<host>`.
|
|
33
|
+
*
|
|
34
|
+
* - scheme: RFC 3986 shape, at least two chars so a Windows drive letter
|
|
35
|
+
* (`c://`) can't anchor a match.
|
|
36
|
+
* - user: no `/`, `@`, `:` or whitespace.
|
|
37
|
+
* - password: anything up to an authority TERMINATOR (`/`, `?`, `#` or
|
|
38
|
+
* whitespace) — `@` deliberately INCLUDED. Greedy matching then backs
|
|
39
|
+
* off to the LAST `@` in the authority, which is where the host
|
|
40
|
+
* actually starts.
|
|
41
|
+
*
|
|
42
|
+
* #3982 review, MAJOR 3: excluding `@` (so the match stopped at the
|
|
43
|
+
* FIRST one) looked conservative and was the opposite. An unencoded
|
|
44
|
+
* `@` in a pasted DSN is common, and
|
|
45
|
+
* `postgres://appuser:p@ssW0rd123@db.internal:5432/prod` masked one
|
|
46
|
+
* byte and stored the other ten in clear —
|
|
47
|
+
* `postgres://appuser:[REDACTED:db_uri_password]@ssW0rd123@db.internal…`
|
|
48
|
+
* — while also telling a reader exactly how long the masked prefix
|
|
49
|
+
* was. `url-redact.ts` already did this right for http(s) by taking
|
|
50
|
+
* `lastIndexOf('@')` over the authority; this is the same rule for the
|
|
51
|
+
* non-HTTP schemes.
|
|
52
|
+
* - host: at least one non-delimiter byte, so `user:pass@` with nothing
|
|
53
|
+
* after it is not a connection string.
|
|
54
|
+
*/
|
|
55
|
+
const DB_URI_RE =
|
|
56
|
+
/\b([a-zA-Z][a-zA-Z0-9+.-]+):\/\/([^\s:/@]+):([^\s/?#]+)@([^\s/@?#]+)/g
|
|
57
|
+
|
|
58
|
+
export const DB_URI_RULE_ID = 'db_uri_password'
|
|
59
|
+
|
|
60
|
+
/** Values that are already masked or are obviously not a live credential. */
|
|
61
|
+
function isInertPassword(value: string): boolean {
|
|
62
|
+
if (value.startsWith('[REDACTED')) return true
|
|
63
|
+
// `redactUrls()` rewrites `user:pass@` to `***@` (empty password), but a
|
|
64
|
+
// hand-written `***`/`xxx` mask should not round-trip into a detection
|
|
65
|
+
// either.
|
|
66
|
+
return /^[*x]+$/i.test(value)
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function scanDbUris(text: string): RawHit[] {
|
|
70
|
+
const hits: RawHit[] = []
|
|
71
|
+
DB_URI_RE.lastIndex = 0
|
|
72
|
+
let m: RegExpExecArray | null
|
|
73
|
+
while ((m = DB_URI_RE.exec(text)) !== null) {
|
|
74
|
+
const scheme = m[1]!
|
|
75
|
+
const user = m[2]!
|
|
76
|
+
const password = m[3]!
|
|
77
|
+
if (isInertPassword(password)) continue
|
|
78
|
+
// Offset of the password = match start + scheme + "://" + user + ":".
|
|
79
|
+
const start = m.index + scheme.length + 3 + user.length + 1
|
|
80
|
+
hits.push({
|
|
81
|
+
rule_id: DB_URI_RULE_ID,
|
|
82
|
+
start,
|
|
83
|
+
end: start + password.length,
|
|
84
|
+
matched_text: password,
|
|
85
|
+
key_name: `${scheme}_password`,
|
|
86
|
+
confidence: 'ambiguous',
|
|
87
|
+
})
|
|
88
|
+
}
|
|
89
|
+
return hits
|
|
90
|
+
}
|
|
@@ -30,7 +30,9 @@
|
|
|
30
30
|
* scrub-coverage PR. Gitleaks TOML is loaded via `gitleaks-loader.ts`.
|
|
31
31
|
*/
|
|
32
32
|
import { ALL_PATTERNS } from './patterns.js'
|
|
33
|
-
import { scanKeyValue, type RawHit } from './kv-scanner.js'
|
|
33
|
+
import { scanKeyValue, scanMemorablePasswords, type RawHit } from './kv-scanner.js'
|
|
34
|
+
import { scanDbUris } from './db-uri.js'
|
|
35
|
+
import { INERT_GATED_RULES, isInertValue } from './inert-values.js'
|
|
34
36
|
import { scanGenericSecrets } from './generic-entropy.js'
|
|
35
37
|
import { shannonEntropy } from './entropy.js'
|
|
36
38
|
import { chunk } from './chunker.js'
|
|
@@ -87,6 +89,17 @@ export function detectSecrets(text: string): Detection[] {
|
|
|
87
89
|
const globalEnd = globalStart + cap.length
|
|
88
90
|
// For env_key_value (captureIndex=3), the LHS is group 1.
|
|
89
91
|
const keyName = p.rule_id === 'env_key_value' ? m[1] : undefined
|
|
92
|
+
// Inert VALUE gate. `PASSWORD: ${DB_PASSWORD}`,
|
|
93
|
+
// `POSTGRES_PASSWORD: vault:pg/password`,
|
|
94
|
+
// `JWT_SECRET=<generate-with-openssl-rand>` and `--token <value>`
|
|
95
|
+
// are documentation, not credentials — and a vault key NAME is
|
|
96
|
+
// exactly what an agent is meant to remember, so masking it
|
|
97
|
+
// deletes information and keeps none. Until #3982's review this
|
|
98
|
+
// list applied only inside the memorable-password rule, so letter
|
|
99
|
+
// case decided the outcome: `password: ${DB_PASSWORD}` survived
|
|
100
|
+
// and `PASSWORD: ${DB_PASSWORD}` was destroyed. See
|
|
101
|
+
// inert-values.ts.
|
|
102
|
+
if (INERT_GATED_RULES.has(p.rule_id) && isInertValue(cap)) continue
|
|
90
103
|
// 2026-05-12: shape gate on env_key_value — the pattern matches
|
|
91
104
|
// any value after an ALLCAPS *_KEY/_TOKEN/_SECRET/_PASSWORD
|
|
92
105
|
// identifier, which previously fired on casual chat like
|
|
@@ -125,6 +138,16 @@ export function detectSecrets(text: string): Detection[] {
|
|
|
125
138
|
for (const h of kvHits) {
|
|
126
139
|
raw.push({ ...h, start: h.start + win.offset, end: h.end + win.offset })
|
|
127
140
|
}
|
|
141
|
+
// Connection-URI credentials (`postgres://user:pass@host`) — the
|
|
142
|
+
// scheme-agnostic shape url-redact.ts cannot see.
|
|
143
|
+
for (const h of scanDbUris(win.text)) {
|
|
144
|
+
raw.push({ ...h, start: h.start + win.offset, end: h.end + win.offset })
|
|
145
|
+
}
|
|
146
|
+
// Human-memorable passwords behind an explicit `password` label — the
|
|
147
|
+
// shape the Shannon-entropy gate above is structurally blind to.
|
|
148
|
+
for (const h of scanMemorablePasswords(win.text)) {
|
|
149
|
+
raw.push({ ...h, start: h.start + win.offset, end: h.end + win.offset })
|
|
150
|
+
}
|
|
128
151
|
// Generic bare-high-entropy fallback (ambiguous). Catches standalone
|
|
129
152
|
// tokens no prefix/KV rule matched. dropOverlaps/dedupeRaw below prefer
|
|
130
153
|
// a high-confidence pattern hit over a generic one on the same range,
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Inert VALUES — placeholders, variable references and already-masked
|
|
3
|
+
* text that occupy a credential-shaped slot without being a credential.
|
|
4
|
+
*
|
|
5
|
+
* Masking one of these buys nothing and costs real information:
|
|
6
|
+
*
|
|
7
|
+
* - `POSTGRES_PASSWORD: vault:pg/password` — the vault KEY NAME is
|
|
8
|
+
* precisely what an agent is supposed to remember. Redacting it
|
|
9
|
+
* deletes the pointer and keeps nothing.
|
|
10
|
+
* - `JWT_SECRET=<generate-with-openssl-rand>` — an instruction.
|
|
11
|
+
* - `const API_KEY = process.env.ANTHROPIC_API_KEY` — a reference.
|
|
12
|
+
*
|
|
13
|
+
* Before #3982's review this list existed only inside the new
|
|
14
|
+
* `memorable_password` rule, so letter case decided the outcome:
|
|
15
|
+
* `password: ${DB_PASSWORD}` survived while `PASSWORD: ${DB_PASSWORD}`
|
|
16
|
+
* (matched by the ALL-CAPS `env_key_value` rule) was destroyed. Every
|
|
17
|
+
* rule whose capture is a LABELLED slot now shares the list —
|
|
18
|
+
* `env_key_value`, `json_secret_field`, `cli_flag` and
|
|
19
|
+
* `memorable_password`.
|
|
20
|
+
*
|
|
21
|
+
* Mirrored in `vendor/hindsight-memory/scripts/lib/secret_redact.py`
|
|
22
|
+
* (`_INERT_VALUE_RES` / `_INERT_GATED_RULES`) and pinned across both
|
|
23
|
+
* engines by `secret_redaction_vectors.json`.
|
|
24
|
+
*/
|
|
25
|
+
|
|
26
|
+
export const INERT_VALUE_RE = [
|
|
27
|
+
// Our own marker (idempotence). `maskToken` only ever emits
|
|
28
|
+
// `[REDACTED]` or `[REDACTED:<rule_id>]`, so the closing bracket is
|
|
29
|
+
// part of the shape.
|
|
30
|
+
/^\[REDACTED(?::[A-Za-z0-9_]+)?\]$/i,
|
|
31
|
+
/^\$\{?[A-Za-z_][A-Za-z0-9_]*\}?$/, // $PASSWORD, ${DB_PASSWORD}
|
|
32
|
+
// `.` is spelled out: JS `.` and Python `.` exclude different line
|
|
33
|
+
// terminators, and the two engines must agree byte for byte.
|
|
34
|
+
/^\{\{[^\n\r\u2028\u2029]*\}\}$/, // {{ handlebars }}
|
|
35
|
+
/^<[^<>\n\r\u2028\u2029]*>$/, // <your-password-here>
|
|
36
|
+
/^%[A-Za-z_][A-Za-z0-9_]*%$/, // %PASSWORD% (Windows)
|
|
37
|
+
/^vault:[A-Za-z0-9_./-]*$/i, // switchroom vault reference
|
|
38
|
+
/^[*x•.]+$/i, // ***, xxxx, ••••
|
|
39
|
+
// Code reference — `process.env.FOO`, `import.meta.env.VITE_X`,
|
|
40
|
+
// `os.environ["FOO"]`. The member path is PART OF THE SHAPE: a bare
|
|
41
|
+
// prefix followed by anything else is not a reference.
|
|
42
|
+
/^(?:process\.env|import\.meta\.env|os\.environ)(?:\.[A-Za-z_$][A-Za-z0-9_$]*|\[["']?[A-Za-z_$][A-Za-z0-9_$]*["']?\])*$/,
|
|
43
|
+
// Well-known "fill this in" values. A live credential never begins
|
|
44
|
+
// with the word telling you to replace it.
|
|
45
|
+
/^(?:changeme|change-me|replaceme|replace-me|placeholder|todo|tbd|yourkey|your-key|yourpassword|your-password|yoursecret|your-secret|yourtoken|your-token)(?:[-_][A-Za-z0-9-]+)?$/i,
|
|
46
|
+
// A lowercase English phrase — a help string or a JSON doc value such
|
|
47
|
+
// as `{"token": "the bearer token to use"}`, never a credential.
|
|
48
|
+
/^[a-z]+(?: [a-z]+){2,}$/,
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* An unbroken run of `[A-Za-z0-9]` long enough to BE a credential.
|
|
53
|
+
*
|
|
54
|
+
* Placeholder text is words joined by separators (`your-password-here`,
|
|
55
|
+
* `generate-with-openssl-rand`, `pg/password`, `ANTHROPIC_API_KEY`) — the
|
|
56
|
+
* longest alphanumeric run across every value this list protects is 10
|
|
57
|
+
* (`production`). A credential is the opposite shape: one dense run.
|
|
58
|
+
*/
|
|
59
|
+
const CREDENTIAL_RUN_RE = /[A-Za-z0-9]{12,}/g
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* Mixed-class floor — `ENV_KV_MIN_LEN`, the length below which the env
|
|
63
|
+
* scanner already declines to call something a secret. Must not exceed
|
|
64
|
+
* `CREDENTIAL_RUN_RE`'s own floor.
|
|
65
|
+
*/
|
|
66
|
+
const MIXED_CLASS_RUN_MIN = 12
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* Single-class floor. A run of only letters or only digits is far more
|
|
70
|
+
* likely to be an English word than a credential (`authentication`,
|
|
71
|
+
* `configuration`), so it gets more rope — but not unlimited rope:
|
|
72
|
+
* `abcdefghijklmnop` and a 40-`x` filler are values `main` masked, and no
|
|
73
|
+
* word in the protected corpus reaches 16.
|
|
74
|
+
*/
|
|
75
|
+
const SINGLE_CLASS_RUN_MIN = 16
|
|
76
|
+
|
|
77
|
+
function runIsCredentialShaped(run: string): boolean {
|
|
78
|
+
// Classes present among {lower, upper, digit}. Hex, base62 and
|
|
79
|
+
// CamelCase-with-digits tokens are two or three; `yourtokenhere` and
|
|
80
|
+
// the segments of `YOUR_TOKEN_HERE` are one.
|
|
81
|
+
let classes = 0
|
|
82
|
+
if (/[a-z]/.test(run)) classes++
|
|
83
|
+
if (/[A-Z]/.test(run)) classes++
|
|
84
|
+
if (/[0-9]/.test(run)) classes++
|
|
85
|
+
return (
|
|
86
|
+
run.length >= (classes >= 2 ? MIXED_CLASS_RUN_MIN : SINGLE_CLASS_RUN_MIN)
|
|
87
|
+
)
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* True when `value` carries a credential-shaped run anywhere inside it.
|
|
92
|
+
*
|
|
93
|
+
* #3982 review, MAJOR 7. The inert SHAPES above are necessary but not
|
|
94
|
+
* sufficient: `<a8f3…40 hex…>` and `{{tok_…}}` are legal instances of the
|
|
95
|
+
* `<…>` / `{{…}}` shapes, and a wrapper left behind after someone filled
|
|
96
|
+
* the value in (`vault:pg/password<secret>`, `[REDACTED]<secret>`,
|
|
97
|
+
* `changeme-<secret>`) turned a false-positive fix into a bypass of
|
|
98
|
+
* already-shipped coverage. "Inert shape AND carries no credential" is
|
|
99
|
+
* the property actually wanted; end-anchoring alone is not enough,
|
|
100
|
+
* because `<secret>` is a well-formed instance of the shape.
|
|
101
|
+
*/
|
|
102
|
+
export function hasCredentialShapedRun(value: string): boolean {
|
|
103
|
+
for (const run of value.match(CREDENTIAL_RUN_RE) ?? []) {
|
|
104
|
+
if (runIsCredentialShaped(run)) return true
|
|
105
|
+
}
|
|
106
|
+
return false
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/** True when `value` is a placeholder / reference rather than a secret. */
|
|
110
|
+
export function isInertValue(value: string): boolean {
|
|
111
|
+
if (!INERT_VALUE_RE.some((re) => re.test(value))) return false
|
|
112
|
+
return !hasCredentialShapedRun(value)
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
/**
|
|
116
|
+
* Rule ids whose captured value is a labelled slot, and which therefore
|
|
117
|
+
* honour `isInertValue`. Keep in sync with `_INERT_GATED_RULES` in
|
|
118
|
+
* `secret_redact.py`.
|
|
119
|
+
*/
|
|
120
|
+
export const INERT_GATED_RULES = new Set([
|
|
121
|
+
'env_key_value',
|
|
122
|
+
'json_secret_field',
|
|
123
|
+
'cli_flag',
|
|
124
|
+
])
|
|
125
|
+
|
|
126
|
+
/**
|
|
127
|
+
* The heuristic KEY=VALUE scanner honours the same list — it is the rule
|
|
128
|
+
* that ate `ANTHROPIC_API_KEY: vault:anthropic/api_key` and
|
|
129
|
+
* `const API_KEY = process.env.ANTHROPIC_API_KEY`. Kept as its own
|
|
130
|
+
* export because `scanKeyValue` applies it directly rather than through
|
|
131
|
+
* the `ALL_PATTERNS` loop.
|
|
132
|
+
*/
|
|
133
|
+
export const KV_ENTROPY_RULE_ID = 'kv_entropy'
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Trailing punctuation an English sentence puts AFTER a value. The value
|
|
137
|
+
* character classes are `[^\s"']`-shaped, so a sentence-final `.` or a
|
|
138
|
+
* list comma is swallowed INTO the value — and a `.` alone supplies a
|
|
139
|
+
* second character class, which is what made
|
|
140
|
+
* "The password is required." redact as a credential (#3982 review,
|
|
141
|
+
* BLOCKER 2).
|
|
142
|
+
*/
|
|
143
|
+
const TRAILING_PUNCT_RE = /[.,;:!?)\]}]+$/
|
|
144
|
+
|
|
145
|
+
export function stripTrailingPunctuation(value: string): string {
|
|
146
|
+
return value.replace(TRAILING_PUNCT_RE, '')
|
|
147
|
+
}
|