redac---n 0.0.3 → 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +11 -10
  2. package/dist/calls.d.ts +8 -0
  3. package/dist/calls.js +60 -3
  4. package/dist/checksums.js +4 -3
  5. package/dist/commands.d.ts +5 -0
  6. package/dist/commands.js +93 -14
  7. package/dist/context.d.ts +5 -2
  8. package/dist/context.js +36 -9
  9. package/dist/encoded.js +13 -6
  10. package/dist/environment.d.ts +5 -3
  11. package/dist/environment.js +38 -14
  12. package/dist/formats.js +15 -0
  13. package/dist/index.js +10 -4
  14. package/dist/json.js +29 -5
  15. package/dist/known.d.ts +7 -7
  16. package/dist/known.js +33 -25
  17. package/dist/named.d.ts +6 -0
  18. package/dist/named.js +421 -125
  19. package/dist/names.d.ts +5 -1
  20. package/dist/names.js +143 -18
  21. package/dist/normalize.d.ts +4 -1
  22. package/dist/normalize.js +37 -11
  23. package/dist/pem.d.ts +2 -0
  24. package/dist/pem.js +12 -0
  25. package/dist/policy.js +12 -5
  26. package/dist/public.js +35 -13
  27. package/dist/secretfile.d.ts +5 -0
  28. package/dist/secretfile.js +49 -20
  29. package/dist/shell.d.ts +12 -3
  30. package/dist/shell.js +51 -7
  31. package/dist/spans.d.ts +4 -3
  32. package/dist/spans.js +79 -8
  33. package/dist/sql.d.ts +12 -8
  34. package/dist/sql.js +94 -16
  35. package/dist/stream.d.ts +2 -1
  36. package/dist/stream.js +15 -3
  37. package/dist/structure.js +210 -61
  38. package/dist/syntax.d.ts +5 -2
  39. package/dist/syntax.js +7 -4
  40. package/dist/transcript.js +116 -28
  41. package/dist/unnamed.js +128 -20
  42. package/dist/urls.d.ts +8 -3
  43. package/dist/urls.js +16 -7
  44. package/dist/values.d.ts +2 -1
  45. package/dist/values.js +70 -25
  46. package/docs/holdout.svg +8 -8
  47. package/docs/layers.md +1 -1
  48. package/package.json +1 -1
@@ -1,10 +1,10 @@
1
1
  /**
2
2
  * Redacting a whole transcript or request body: any JSON value, typically a list of chat messages in
3
- * the Anthropic or OpenAI format. Two passes:
3
+ * the Anthropic, OpenAI or Gemini format. Two passes:
4
4
  *
5
5
  * 1. Every string is scanned, with the command or path of the tool call that produced it as a hint
6
- * (a `tool_result` is matched to its `tool_use` by id), and every confident finding is learned as a
7
- * known value.
6
+ * (a `tool_result` is matched to its `tool_use` by id, a Gemini `functionResponse` to its `functionCall` too), and
7
+ * every confident finding is learned as a known value.
8
8
  * 2. Every string is redacted, so a secret found once (`API_KEY=…` in a `cat .env`) is also hidden where
9
9
  * it appears with no context at all (the model echoing it, a later `curl` using it).
10
10
  *
@@ -24,18 +24,24 @@ import { isLearnable, isNeverSecretShape, loginPassword, merge, secretValue } fr
24
24
  import { occurrences } from "./repeats.js";
25
25
  import { isCredentialName } from "./names.js";
26
26
  import { isNamedSecret } from "./named.js";
27
+ import { KnownValues } from "./known.js";
27
28
  /** Keys whose values are IDs the provider minted (`toolu_01Ab…`, `chatcmpl-B9…`): one token, no spaces. */
28
- const ID_KEYS = new Set(["id", "tool_use_id", "tool_call_id", "call_id"]);
29
+ const ID_KEYS = new Set(["id", "tool_use_id", "tool_call_id", "call_id", "file_id"]);
29
30
  /** Keys whose values are opaque blobs the provider signed or encrypted, which no one reads. */
30
31
  const OPAQUE_KEYS = new Set(["signature", "thoughtSignature", "thought_signature", "encrypted_content"]);
31
- /** Keys whose values are a short enum or a model name (`assistant`, `end_turn`, `image/png`, `claude-sonnet-4-5-20250929`). */
32
+ /**
33
+ * Keys whose values are a short enum, a model's or a tool's name (`assistant`, `end_turn`, `image/png`,
34
+ * `claude-sonnet-4-5-20250929`, `run_shell_command`).
35
+ */
32
36
  const ENUM_KEYS = new Set([
33
37
  "type",
34
38
  "role",
35
39
  "model",
40
+ "name",
36
41
  "stop_reason",
37
42
  "finish_reason",
38
43
  "media_type",
44
+ "mimeType",
39
45
  "object",
40
46
  "status",
41
47
  "mime_type",
@@ -111,8 +117,17 @@ function isEnvelope(o) {
111
117
  (typeof o.type === "string" && BLOCK_TYPES.has(o.type)) ||
112
118
  typeof o.tool_call_id === "string" ||
113
119
  typeof o.tool_use_id === "string" ||
120
+ // (OpenAI Responses' items of a tool call and its output, whatever the tool: `shell_call`, `custom_tool_call_output`.)
121
+ typeof o.call_id === "string" ||
114
122
  (typeof o.object === "string" && typeof o.model === "string"));
115
123
  }
124
+ /**
125
+ * Keys of an envelope's object that hold its blocks: a tool result's `content` (MCP's `resource` and `resource_link`
126
+ * blocks, a server tool's `bash_code_execution_result`), a document's `source`, an MCP resource block's `resource`, a
127
+ * Responses output's `output`. An object there is a block too when it has a `type` of an enum's shape, whatever the type
128
+ * (an allow-list of them falls behind every new tool), and so is an MCP `resource`, which has none.
129
+ */
130
+ const BLOCK_KEYS = new Set(["content", "source", "resource", "output"]);
116
131
  /** Keys of tool inputs that hold what a tool writes to the file at its path (`Write`, `create_file`, `str_replace`). */
117
132
  const CONTENT_KEYS = new Set(TOOL_CONTENT_KEYS);
118
133
  /** Files a patch adds or changes: `*** Add File: x` (apply_patch), `+++ b/x` (unified diff). */
@@ -171,7 +186,8 @@ function toolHints(value) {
171
186
  const o = v;
172
187
  // Anthropic: {type: "tool_use", id, input: {...}}. OpenAI: {id, type: "function", function: {arguments: "<json>"}}.
173
188
  // OpenAI Responses: {type: "function_call", call_id, arguments}; {type: "local_shell_call", call_id, action: {command: [...]}}.
174
- let input = o.input ?? o.action ?? o.arguments;
189
+ // Gemini: {functionCall: {id, name, args: {...}}} (the id is the client's: Gemini CLI and pydantic-ai send one).
190
+ let input = o.input ?? o.action ?? o.arguments ?? o.args;
175
191
  const fn = o.function && typeof o.function === "object" ? o.function.arguments : undefined;
176
192
  if (fn !== undefined && fn !== null)
177
193
  input = fn;
@@ -235,11 +251,13 @@ function encodeBase64Text(text, like) {
235
251
  */
236
252
  function base64Text(o, key, x) {
237
253
  const uri = BASE64_URI.exec(x);
238
- const textual = uri
239
- ? TEXT_MEDIA_TYPE.test(uri[1])
240
- : key === "data" && typeof o.media_type === "string" && TEXT_MEDIA_TYPE.test(o.media_type.split(";")[0].trim());
254
+ const type = uri
255
+ ? uri[1]
256
+ : key === "data" && typeof o.media_type === "string"
257
+ ? o.media_type.split(";")[0].trim()
258
+ : "";
241
259
  const payload = uri ? uri[3] : x;
242
- const text = textual ? decodeBase64Text(payload) : undefined;
260
+ const text = TEXT_MEDIA_TYPE.test(type) ? decodeBase64Text(payload) : undefined;
243
261
  return text === undefined
244
262
  ? undefined
245
263
  : { text, encode: (redacted) => (uri ? `data:${uri[1]}${uri[2]}` : "") + encodeBase64Text(redacted, payload) };
@@ -303,6 +321,7 @@ function collect(value, hints, keepThinking) {
303
321
  hint,
304
322
  key,
305
323
  parent,
324
+ lone: scope.lone,
306
325
  });
307
326
  return;
308
327
  }
@@ -311,11 +330,17 @@ function collect(value, hints, keepThinking) {
311
330
  const o = v;
312
331
  if (keepThinking && (o.type === "thinking" || o.type === "redacted_thinking"))
313
332
  return "skip";
314
- // A tool result inherits the hint of its call; OpenAI tool messages carry `tool_call_id`.
333
+ // A tool result inherits the hint of its call; OpenAI tool messages carry `tool_call_id`, a Responses output
334
+ // (`…_call_output`) its call's `call_id`.
315
335
  const id = (o.tool_use_id ??
316
336
  o.tool_call_id ??
317
- (o.type === "function_call_output" || o.type === "local_shell_call_output" ? o.call_id : undefined));
318
- const hint = typeof id === "string" && hints.has(id) ? hints.get(id) : scope.hint;
337
+ (typeof o.type === "string" && o.type.endsWith("_call_output") ? o.call_id : undefined));
338
+ let hint = typeof id === "string" && hints.has(id) ? hints.get(id) : scope.hint;
339
+ // Gemini's `functionResponse`: its call's, by id. (One with no id gets none: the clients sending these requests,
340
+ // Gemini CLI and pydantic-ai, give each call an id.)
341
+ const gemini = key === "functionCall" || key === "functionResponse";
342
+ if (key === "functionResponse" && typeof o.id === "string" && hints.has(o.id))
343
+ hint = hints.get(o.id);
319
344
  // Base64 media: `{type: "base64", media_type, data}`, `image_url: {url: "data:…"}`.
320
345
  const media = o.type === "base64" ||
321
346
  o.type === "image" ||
@@ -324,7 +349,18 @@ function collect(value, hints, keepThinking) {
324
349
  o.type === "document";
325
350
  // A tool input that writes a file: its content is judged as that file (`.env` content as a secret file).
326
351
  const writes = TOOL_PATH_KEYS.map((k) => o[k]).find((x) => typeof x === "string");
327
- const envelope = !scope.free && isEnvelope(o);
352
+ const envelope = !scope.free &&
353
+ (isEnvelope(o) ||
354
+ gemini ||
355
+ (!!scope.block && (typeof o.type === "string" ? ENUM_VALUE.test(o.type) : key === "resource")));
356
+ // (An object's `input`, `arguments` or `action` is a tool's input when the object is a tool call, which has a name or
357
+ // a type: a Responses request's `input` is its list of messages.)
358
+ const call = typeof o.name === "string" || typeof o.type === "string";
359
+ // (How many members are text that isn't structure: one alone is the envelope's text, a message's or a block's.)
360
+ const texts = envelope
361
+ ? Object.entries(o).filter(([k, x]) => (typeof x === "string" || typeof x === "number") && !isStructure(k, x))
362
+ .length
363
+ : Infinity;
328
364
  // (An attached file's name: Anthropic's `document` block, OpenAI's `input_file` part and Chat Completions' `file`.)
329
365
  const name = !hint.path && (o.type === "document" || o.type === "input_file" || "file_data" in o)
330
366
  ? documentName(o)
@@ -362,7 +398,13 @@ function collect(value, hints, keepThinking) {
362
398
  if ((media && k === "data" && typeof x === "string" && BASE64_VALUE.test(x.replace(/\s/g, ""))) ||
363
399
  (typeof x === "string" && MEDIA_URI.test(x)))
364
400
  return undefined;
365
- return { hint: childHint, free: scope.free || TOOL_INPUT_KEYS.has(k) };
401
+ return {
402
+ hint: childHint,
403
+ free: scope.free || (call && TOOL_INPUT_KEYS.has(k)),
404
+ // (Down through what a block holds: a shell output's `{stdout, outcome: {type: "exit"}}`.)
405
+ block: envelope ? BLOCK_KEYS.has(k) : scope.block,
406
+ lone: texts <= 1 && !isCredentialName(k),
407
+ };
366
408
  };
367
409
  });
368
410
  return leaves;
@@ -489,26 +531,31 @@ export function redactValue(redactor, value, options = {}) {
489
531
  const inParents = new Map();
490
532
  const found = (leaf) => {
491
533
  const alone = redactor.detect(leaf.text, { hint: leaf.hint, guessing: options.guessing });
492
- const ctx = leaf.hint.command === undefined && leaf.hint.path === undefined ? leafContext(leaf, serialized) : undefined;
534
+ const ctx = !leaf.lone && leaf.hint.command === undefined && leaf.hint.path === undefined
535
+ ? leafContext(leaf, serialized)
536
+ : undefined;
493
537
  if (!ctx)
494
538
  return alone;
495
539
  let inParent = inParents.get(ctx.text);
496
540
  if (!inParent)
497
541
  inParents.set(ctx.text, (inParent = redactor.detect(ctx.text, { guessing: options.guessing })));
498
- const end = ctx.at + leaf.text.length;
542
+ const end = ctx.at + ctx.length;
499
543
  const mine = inParent
500
544
  .filter((f) => f.start >= ctx.at && f.end <= end)
501
- .map((f) => ({ ...f, start: f.start - ctx.at, end: f.end - ctx.at }));
545
+ .map((f) => ({ ...f, start: ctx.toLeaf(f.start - ctx.at, false), end: ctx.toLeaf(f.end - ctx.at, true) }));
502
546
  return mine.length ? merge([...alone, ...mine]) : alone;
503
547
  };
504
548
  const judged = (leaf) => {
505
- const ctx = leafContext(leaf, serialized);
549
+ // (Placed once a finding is to be judged: most leaves have none.)
550
+ let ctx = null;
506
551
  // (The key names only a finding that is the whole leaf; inside a longer text, the name is the text's own.)
507
552
  return (f) => {
553
+ if (ctx === null)
554
+ ctx = leafContext(leaf, serialized);
508
555
  const name = f.name ?? (f.start === 0 && f.end === leaf.text.length ? leaf.key : undefined);
509
556
  const g = name ? { ...f, name } : f;
510
557
  return ctx
511
- ? redactor.judge(ctx.text, { ...g, start: f.start + ctx.at, end: f.end + ctx.at })
558
+ ? redactor.judge(ctx.text, { ...g, start: ctx.at + ctx.toContext(f.start), end: ctx.at + ctx.toContext(f.end) })
512
559
  : redactor.judge(leaf.text, g);
513
560
  };
514
561
  };
@@ -516,11 +563,14 @@ export function redactValue(redactor, value, options = {}) {
516
563
  // this value's other leaves only.
517
564
  // (Learned all at once after the pass: adding values rebuilds the short-value matcher, once per batch.)
518
565
  const learned = [];
566
+ const first = [];
519
567
  for (const leaf of leaves) {
520
568
  const keep = judged(leaf);
521
569
  if (isNamedKeySecret(leaf, keep) && !isNeverSecretShape(leaf.text))
522
570
  learned.push([leaf.text, leaf.key]);
523
- for (const f of found(leaf).filter(keep)) {
571
+ const here = found(leaf);
572
+ first.push(here);
573
+ for (const f of here.filter(keep)) {
524
574
  const secret = secretValue(leaf.text, f);
525
575
  if (!isLearnable(f, secret))
526
576
  continue;
@@ -535,12 +585,20 @@ export function redactValue(redactor, value, options = {}) {
535
585
  for (const [value, label] of learned)
536
586
  redactor.addKnown(value, label);
537
587
  const seen = remember ? undefined : new Map(learned.map(([value, kind]) => [value, { kind, value }]));
588
+ // (A leaf is detected again only where a value learned here turns up in it, in any form: elsewhere, what pass 1 found
589
+ // is what detection finds now.)
590
+ let fresh;
591
+ if (remember && learned.length) {
592
+ fresh = new KnownValues();
593
+ for (const [value, label] of learned)
594
+ fresh.add(value, label);
595
+ }
538
596
  // Pass 2: redact, into one deep copy of the value.
539
597
  let out = leaves.length ? cloneJson(value) : value;
540
598
  const findings = [];
541
599
  // (Object keys hidden: the key's text, and what it is replaced by.)
542
600
  const renames = new Map();
543
- for (const leaf of leaves) {
601
+ for (const [i, leaf] of leaves.entries()) {
544
602
  const keep = judged(leaf);
545
603
  if (isNamedKeySecret(leaf, keep)) {
546
604
  findings.push({
@@ -555,7 +613,7 @@ export function redactValue(redactor, value, options = {}) {
555
613
  out = setAt(out, leaf.path, redactor.placeholderFor({ start: 0, end: leaf.text.length, kind: leaf.key, rule: "transcript:named-key", confidence: 0.95 }, leaf.text));
556
614
  continue;
557
615
  }
558
- const here = found(leaf).filter(keep);
616
+ const here = (fresh?.find(leaf.text).length ? found(leaf) : first[i]).filter(keep);
559
617
  const r = redactor.apply(leaf.text, seen?.size ? merge([...here, ...occurrences(leaf.text, seen, here)]) : here);
560
618
  if (!r.findings.length)
561
619
  continue;
@@ -582,14 +640,42 @@ export function redactValue(redactor, value, options = {}) {
582
640
  renameKeys(out, renames);
583
641
  return { value: out, findings };
584
642
  }
643
+ /** A leaf's text placed at `at` in its context, as JSON writes it (`written`, without its quotes). */
644
+ function placed(text, context, at, written) {
645
+ if (written.length === text.length)
646
+ return { text: context, at, length: text.length, toContext: (i) => i, toLeaf: (i) => i };
647
+ // (Where each character is written: a code point as JSON writes it, a valid surrogate pair as it is, a lone one escaped.)
648
+ const starts = [];
649
+ let w = 0;
650
+ for (const ch of text) {
651
+ starts.push(w);
652
+ if (ch.length === 2)
653
+ starts.push(w + 1);
654
+ w += JSON.stringify(ch).length - 2;
655
+ }
656
+ starts.push(w);
657
+ return {
658
+ text: context,
659
+ at,
660
+ length: written.length,
661
+ toContext: (i) => starts[i],
662
+ // (The last leaf position written at or before `i`; with `after`, the first at or after it.)
663
+ toLeaf: (i, after) => {
664
+ const j = starts.findLastIndex((s) => s <= i);
665
+ return after && starts[j] < i ? j + 1 : j;
666
+ },
667
+ };
668
+ }
585
669
  /**
586
670
  * The leaf's object serialized, or its list of strings (an argv: `["docker", "login", "-p", "…"]`), and where the leaf's
587
- * text starts in it; undefined when it can't be placed exactly.
671
+ * text stands in it; undefined when it can't be placed exactly.
588
672
  */
589
673
  function leafContext(leaf, serialized) {
590
674
  const parent = leaf.parent;
591
- if (!parent || JSON.stringify(leaf.text) !== `"${leaf.text}"`)
675
+ if (!parent)
592
676
  return undefined;
677
+ const literal = JSON.stringify(leaf.text);
678
+ const written = literal.slice(1, -1);
593
679
  if (Array.isArray(parent)) {
594
680
  const index = leaf.path.at(-1);
595
681
  if (typeof index !== "number" || parent.length > 200 || !parent.every((x) => typeof x === "string"))
@@ -598,7 +684,9 @@ function leafContext(leaf, serialized) {
598
684
  serialized.set(parent, JSON.stringify(parent));
599
685
  const text = serialized.get(parent);
600
686
  const at = 1 + parent.slice(0, index).reduce((n, x) => n + JSON.stringify(x).length + 1, 0) + 1;
601
- return text.length <= 20_000 && text.slice(at, at + leaf.text.length) === leaf.text ? { text, at } : undefined;
687
+ return text.length <= 20_000 && text.slice(at, at + written.length) === written
688
+ ? placed(leaf.text, text, at, written)
689
+ : undefined;
602
690
  }
603
691
  if (leaf.key === undefined)
604
692
  return undefined;
@@ -623,9 +711,9 @@ function leafContext(leaf, serialized) {
623
711
  const text = serialized.get(parent);
624
712
  if (text === undefined)
625
713
  return undefined;
626
- const needle = `${JSON.stringify(leaf.key)}:"${leaf.text}"`;
714
+ const needle = `${JSON.stringify(leaf.key)}:${literal}`;
627
715
  const i = text.indexOf(needle);
628
- return i < 0 ? undefined : { text, at: i + needle.length - 1 - leaf.text.length };
716
+ return i < 0 ? undefined : placed(leaf.text, text, i + needle.length - 1 - written.length, written);
629
717
  }
630
718
  /**
631
719
  * The `arguments` or `output` JSON string `original` with the scalars that redaction changed (`redacted` is the value it parsed
package/dist/unnamed.js CHANGED
@@ -16,7 +16,7 @@ import { base64Binary, base64Bytes } from "./base64.js";
16
16
  import { naturalness } from "./charlm.js";
17
17
  import { contextOf, Doc } from "./context.js";
18
18
  import { isCredentialName, isVendorClient, mentionsCredential } from "./names.js";
19
- import { isPlaceholder } from "./public.js";
19
+ import { isPlaceholder, isTruncated } from "./public.js";
20
20
  import { alphabetOf, entropyZ, shape } from "./random.js";
21
21
  import { trailingRun } from "./lines.js";
22
22
  import { ENCRYPTION_WRAPPER, ANALYTICS_COOKIE, isIdentifierShaped, isInert, isPodName, PASSWORD_HASH_IN_TEXT, isWordsValue, readsAsWords, } from "./values.js";
@@ -36,7 +36,7 @@ const API_OBJECT_ID = /^(?:(?:vpc|subnet|sg|i|ami|vol|snap|eni|igw|rtb|acl|nat|e
36
36
  * Prefixes of payment, messaging and SaaS object IDs (Stripe `cus_`, `pi_`, `ch_`, Twilio, Svix…). Some
37
37
  * are also prefixes of keys (`re_` is Resend's), so these are public only outside a key's place (keyPlace).
38
38
  */
39
- const OBJECT_ID = /^(?:env|usr|svc|srv|db|setl|order|job|raysubmit|ins|client|sia|sms|idn|pri|pro|sub|txn|ctm|add|biz|dsc|dpl|prj|team|conn|directory|user|ap|in|ta|fu|vol|sb|sess(?=_)|we|promo|qt|cs|dp|cus|pi|ch|il|ii|price|prod|evt|acct|tr|po|pm|seti|src|card|ba|btok|tok|re|plink|si|pmc|ipi|cbtxn|ich|ic|fee|link|req|test_clock|whe|wh|ca|cn|du|py|mandate|pyr|trr|setatt|sched|subi|ri|gsk_live_?)[-_]/;
39
+ const OBJECT_ID = /^(?:env|usr|svc|srv|db|setl|order|org|job|raysubmit|ins|client|sia|sms|idn|pri|pro|sub|txn|ctm|add|biz|dsc|dpl|prj|team|conn|directory|user|ap|in|ta|fu|vol|sb|sess(?=_)|we|promo|qt|cs|dp|cus|pi|ch|il|ii|price|prod|evt|acct|tr|po|pm|seti|src|card|ba|btok|tok|re|plink|si|pmc|ipi|cbtxn|ich|ic|fee|link|req|test_clock|whe|wh|ca|cn|du|py|mandate|pyr|trr|setatt|sched|subi|ri|gsk_live_?)[-_]/;
40
40
  /** Right after an encryption wrapper (values.ts ENCRYPTION_WRAPPER), or SOPS's `,iv:` and `,tag:` fields, or Splunk's `$7$`. */
41
41
  const AFTER_ENCRYPTION_WRAPPER = new RegExp(String.raw `(?:${ENCRYPTION_WRAPPER}|,iv:|,tag:|(?:^|[\s=:"'])\$[17]\$)$`);
42
42
  /** Shapes that are IDs or encodings, not keys. */
@@ -59,6 +59,10 @@ const CANDIDATE = new RegExp(`^[A-Za-z0-9+/=_-]{${MIN_TOKEN},}$`);
59
59
  * `%40scope%2Fname`). A base64 value's `/`, `+` and `=` are escaped in one.
60
60
  */
61
61
  const PERCENT_RUN = /(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2}){16,}/g;
62
+ /** A run of token characters with JSON's escaped slashes in it (PHP's and Laravel's `Zx9…\/Hd1…`). */
63
+ const ESCAPED_SLASH_RUN = /(?:[A-Za-z0-9+=_-]|\\\/){16,}/g;
64
+ /** A letter beyond ASCII in an alphabet with cases (Latin's, Greek's, Cyrillic's): `ð`, `é`, `ж`. */
65
+ const NON_ASCII_CASED = /(?![\x00-\x7f])[\p{Lu}\p{Ll}]/u;
62
66
  /**
63
67
  * Runs of token characters long enough to be a key. A run that is a path (`/api/v1/<id>/users`) or
64
68
  * dotted (`a.b.c`) is split into its segments, each judged alone. A percent-encoded run that decodes to one token is
@@ -83,6 +87,20 @@ export function candidates(text) {
83
87
  encoded.push([m.index, m.index + m[0].length]);
84
88
  }
85
89
  }
90
+ // A base64 value whose slashes JSON escaped, as PHP's json_encode does every one, is judged whole too, as its
91
+ // decoded value; a path's (`api.example.io\/v1\/me`) is split as a path is.
92
+ if (text.includes("\\/")) {
93
+ for (const m of text.matchAll(ESCAPED_SLASH_RUN)) {
94
+ if (!m[0].includes("\\/") || backslashesBefore(text, m.index) % 2)
95
+ continue;
96
+ const token = m[0].replace(/\\\//g, "/");
97
+ if (token.length > 256 || !CANDIDATE.test(token) || !isBase64Run(token, text, m.index))
98
+ continue;
99
+ out.push({ start: m.index, end: m.index + m[0].length, token });
100
+ encoded.push([m.index, m.index + m[0].length]);
101
+ }
102
+ encoded.sort((x, y) => x[0] - y[0]);
103
+ }
86
104
  // (Both in the text's order: the encoded run a token run may be in is the first not ended before it.)
87
105
  let k = 0;
88
106
  for (const [runStart, runEnd] of longRuns(text, CANDIDATE_SET, MIN_TOKEN)) {
@@ -92,6 +110,11 @@ export function candidates(text) {
92
110
  continue;
93
111
  if (runEnd - runStart > 256)
94
112
  continue; // blobs: images, certificates, bundles; longer tokens are caught by format or header
113
+ // The ASCII part of a word in a cased alphabet, cut at its first letter beyond ASCII: Icelandic's
114
+ // `Endurheimtulykilorð`, `endurheimtulykilorðið`. (Not after a letter of a script without cases: Chinese writes a
115
+ // token against its words, `令牌是Xk9…`.)
116
+ if (NON_ASCII_CASED.test(text[runStart - 1] ?? "") || NON_ASCII_CASED.test(text[runEnd] ?? ""))
117
+ continue;
95
118
  const run = text.slice(runStart, runEnd);
96
119
  // `name=value` inside a run (query strings, form bodies): split at an `=` that isn't padding.
97
120
  for (const [eqOff, piece] of splitKeep(run, /=(?=[A-Za-z0-9+/_-])/g)) {
@@ -310,21 +333,30 @@ const isUserinfo = (f) => /^[^\s"'<>@?#/]*@[\w-]+(?:\.[\w-]+)+/.test(f.ctx.right
310
333
  /**
311
334
  * A URL path segment that names an object: under a collection (`/v1/predictions/<id>`), or anywhere when its shape
312
335
  * says ID (a file name, `<key>~noop.jpeg`, `<id>.png`; short; one case). A mixed-case random run of 20 or more is a
313
- * token whatever path it's in, unless the host serves media (`inews.gtimg.com`) or a content word is its parent
314
- * (`/article/<id>`). Never a hook's URL, the user part before an `@`, or a secret file's value.
336
+ * token whatever path it's in (24 or more under a collection: `/integrations/<token>`, `/endpoints/<token>`), unless
337
+ * the host serves media (`inews.gtimg.com`) or a content word is its parent (`/article/<id>`). Never a hook's URL, the
338
+ * user part before an `@`, or a secret file's value.
315
339
  */
316
340
  function urlPathId(f) {
317
341
  if (f.ctx.syntax !== "path" || f.secretFile || isHookUrl(f) || isUserinfo(f))
318
342
  return undefined;
319
- if (/^[a-z][a-z-]{2,30}s$/i.test(f.ctx.pathParent) && !isHookSegment(f.ctx.pathParent))
320
- return "collection-id";
321
- if (f.ctx.nameClass === "credential")
343
+ const collection = /^[a-z][a-z-]{2,30}s$/i.test(f.ctx.pathParent) && !isHookSegment(f.ctx.pathParent);
344
+ if (f.ctx.nameClass === "credential" && !collection)
322
345
  return undefined;
323
346
  const mediaHost = /:\/\/(?:[\w-]+\.)*[\w-]*(?:img|image|images|cdn|static|media|pic|pics|photo|photos|thumb|thumbs|resiz|assets?)[\w-]*\.[\w.-]+\/[^\s"']*$/i.test(f.ctx.left);
324
347
  const contentParent = /^(?:article|articles|post|posts|p|story|stories|news|item|items|video|videos|watch|status|statuses|question|questions|answer|issue|issues|pull|commit|blob|tree|wiki|doc|docs|d|album|track|playlist|episode|product|products)$/i.test(f.ctx.pathParent);
325
348
  const t = f.token;
326
349
  const tokenShaped = t.length >= 20 && /[a-z]/.test(t) && /[A-Z]/.test(t) && /\d/.test(t) && !mediaHost && !contentParent;
327
- return /^[~.][A-Za-z0-9]{2,6}\b|^~|^=[swhc]\d+/.test(f.ctx.right) || !tokenShaped ? "url-path-id" : undefined;
350
+ // (Under a collection, longer than the IDs services generate in that alphabet: Firestore's 20, nanoid's 21, Spotify's
351
+ // 22, ElevenLabs' voice 20.)
352
+ if (collection)
353
+ return tokenShaped && t.length >= 24 ? undefined : "collection-id";
354
+ // (A file name: an image's or a page asset's, named by its ID on any host, `<key>~noop.jpeg`, `<id>.png`, an image
355
+ // server's sizing, `<id>=s1200`. Not a feed, a calendar or a download, which a private link names by its token:
356
+ // `/subscribe/<token>.ics`, `/feeds/<token>.rss`.)
357
+ if (/^~|^=[swhc]\d+|^\.(?:jpe?g|png|gif|webp|avif|svg|ico|bmp|tiff?|heic|mp[34]|webm|mov|m4[av]|ogg|wav|m3u8|ts|js|mjs|css|map|woff2?|ttf|html?)\b/i.test(f.ctx.right))
358
+ return "url-path-id";
359
+ return tokenShaped ? undefined : "url-path-id";
328
360
  }
329
361
  /**
330
362
  * Hex, hashes and IDs until a key context says otherwise: under a credential's name, or (32 characters or more) a
@@ -389,6 +421,15 @@ const VETOES = [
389
421
  // The token's own shape.
390
422
  ["digits", (_, t) => /^\d+$/.test(t)],
391
423
  ["placeholder", (_, t) => isPlaceholder(t)],
424
+ // A key shown cut short, its front before an ellipsis (`sk-ant-api03-Zx9K...`, `…QAAA` after it): the token stops at
425
+ // the dots, the display doesn't.
426
+ [
427
+ "placeholder",
428
+ (f, t) => {
429
+ const cut = /^(?:\.\.\.|…)[\w+/=-]{0,40}/.exec(f.ctx.right)?.[0];
430
+ return cut !== undefined && isTruncated(`${f.lead ?? ""}${t}${cut}`);
431
+ },
432
+ ],
392
433
  // Hex of one short unit written over and over (`Key = ef012345ef012345ef012345ef012345`): a filler someone typed, not
393
434
  // key bytes, which never repeat in step. (Other alphabets' fillers, `foobarfoobar…`, are in the model's training
394
435
  // negatives, which a veto would change.)
@@ -416,6 +457,44 @@ const VETOES = [
416
457
  (f) => /[?&](?:pf_rd_\w+|pd_rd_\w+|utm_\w+|fbclid|gclid|msclkid|dclid|mc_eid|igshid|_hsenc|crid|qid|sprefix|ved|usg|ei|sei|iflsig|refRID|sca_esv|sca_upv|sxsrf|oq|aqs|gs_lcp|gs_lp|uact|rlz|ictx|vin)=$/i.test(f.ctx.left),
417
458
  ],
418
459
  ["build-id", (f) => /\bBuildID=\S*$/.test(f.ctx.left)],
460
+ // The name a declaration gives, an identifier in its letters' case: C++'s `class XiaomiLYWSD03MMC final`, `struct
461
+ // QMI8658Config {`, `enum`, `interface`.
462
+ [
463
+ "declared-name",
464
+ (f, t) => /(?:^|[\s;{])(?:class|struct|enum|union|interface|trait|namespace|record)[ \t]+$/.test(f.ctx.left) &&
465
+ /^[A-Z][A-Za-z0-9_]*$/.test(t) &&
466
+ /^[ \t]*(?:final\b|[:{<(]|extends\b|implements\b|\r?$)/m.test(f.ctx.right),
467
+ ],
468
+ // A cell of a row of a command's table, six cells or more and three of the others counts, sizes or dates, with no
469
+ // credential's word: an ID (Elasticsearch's `_cat/indices`, `green open chunks-2026.10 <uuid> 3 1 1820331 0 14.2gb`).
470
+ [
471
+ "table-row",
472
+ (f, t, cred) => {
473
+ if (cred || f.ctx.lineCredential)
474
+ return false;
475
+ const cells = `${f.ctx.left}${f.lead ?? ""}${t}${f.ctx.right}`.trim().split(/[ \t]+/);
476
+ if (cells.length < 6 || !/(?:^|[ \t])$/.test(f.ctx.left) || !/^(?:[ \t]|$)/.test(f.ctx.right))
477
+ return false;
478
+ const counts = cells.filter((c) => /^\d+(?:\.\d+)?(?:[kmgtp]?i?b|[kmg]|%|ms|s|m|h|d)?$|^\d{4}-\d\d-\d\d(?:T[\d:.]+Z?)?$/i.test(c));
479
+ return counts.length >= 3 && !/key|token|secret|passw|auth|credential/i.test(f.ctx.left + f.ctx.right);
480
+ },
481
+ ],
482
+ // The object a lookup is given, its first argument: `Subscription::retrieve("<id>")`, `stripe.customers.retrieve(…)`,
483
+ // `Model.findById(…)`. (Not a secret's lookup, `get_secret`, `Secret.fetch`: there the argument names the secret.)
484
+ [
485
+ "lookup-id",
486
+ (f, _, cred) => !cred &&
487
+ f.ctx.syntax === "call" &&
488
+ /(?:^|[.:])(?:retrieve|find|findById|findOne|fetch|lookup|describe|delete|destroy|cancel|getById|get_by_id|find_by_id)$/i.test(f.ctx.name) &&
489
+ /(?:::|\.|->)\w+\([ \t]*["'`]$/.test(f.ctx.left) &&
490
+ !/secret|token|key|passw|credential|vault/i.test(/[\w$\\:.>-]*\([ \t]*["'`]$/.exec(f.ctx.left)?.[0] ?? ""),
491
+ ],
492
+ // A static file's cache buster, a digest of its content: `site.css?v=<base64url SHA-256>` (ASP.NET's
493
+ // `asp-append-version`), and as IIS logs it, the query after the path's field, `/css/site.css v=…`.
494
+ [
495
+ "cache-buster",
496
+ (f) => /\.(?:css|js|mjs|png|svg|jpe?g|gif|webp|avif|woff2?|ttf|otf|eot|ico|map)(?:\?|[ \t]+)(?:[\w.-]+=[^\s&=]*&)*(?:v|ver|version|hash|h|cb|rev|_)=$/i.test(f.ctx.left),
497
+ ],
419
498
  ["public-id", (f) => inPublicNamespace(f)],
420
499
  // A hashed host in `~/.ssh/known_hosts` (`|1|<salt>|<HMAC-SHA1 of the host name>`, both base64 of 20 bytes): public, as the host names are.
421
500
  // (With the `+` a salt can start with, which the token drops: `|1|+bsj…=|`.)
@@ -528,12 +607,15 @@ const VETOES = [
528
607
  /^(?:data|embeddings?|vectors?|payload|blob|frame(?:_?\d+)?|thumbnail|bytes|mac|cnonce|opaque|protected|aad)$/i.test(f.ctx.name),
529
608
  ],
530
609
  // Where it sits: a file path, a PEM block of public material, a line of a wrapped blob.
531
- ["fs-path", (f, _, cred) => f.ctx.fsPath && !cred],
610
+ // (Not a request target in a log under a hook's segment, `"POST /hooks/<token> HTTP/1.1"`: a URL's path.)
611
+ ["fs-path", (f, _, cred) => f.ctx.fsPath && !cred && !isHookSegment(f.ctx.pathParent)],
532
612
  ["armored-public", (f) => f.ctx.armored],
533
613
  // A PEM block's label on the line above it, a name of parts (`ALICE_cf_c2pnb163v1`, `PRIME192V1_RFC5114-Peer`): the
534
614
  // block holds the key.
535
615
  ["block-label", (f, t) => f.ctx.labelsBlock && /^[A-Za-z][A-Za-z0-9]*(?:[_-][A-Za-z0-9]+)+$/.test(t)],
536
- ["blob-line", (f, _, cred) => f.ctx.blobLine && !cred],
616
+ // (Not lines a handover heads, keys listed one per line: "Here are your new API keys:" over three `openssl rand
617
+ // -base64 32`s, of one width as a blob's lines are.)
618
+ ["blob-line", (f, _, cred) => f.ctx.blobLine && !cred && !f.ctx.handover],
537
619
  ["ssh-public-key", (_, t) => /^AAAA[A-Z]\d?Nza|^AAAAE2Vj/.test(t)],
538
620
  // (After a version's number, `1.2.`, not after the last digit of a token cut at a dot: `<id>7.<secret>`.)
539
621
  ["version", (f, t) => /(?:^|[^\w.])v?\d{1,4}(?:\.\d{1,6})*\.$/.test(f.ctx.left) || /\+g[0-9a-f]{7,40}$/.test(t)],
@@ -559,7 +641,12 @@ const VETOES = [
559
641
  !cred &&
560
642
  !keyPlace(f),
561
643
  ],
562
- ["filename", (f, _, cred) => f.ctx.fileish && !cred],
644
+ // (A web link's path segment is read by urlPathId, which knows a private feed's or download's link from a file's
645
+ // name; a `file://` URL's are a file's path.)
646
+ [
647
+ "filename",
648
+ (f, _, cred) => f.ctx.fileish && !cred && (f.ctx.syntax !== "path" || /file:\/\/[^\s"']*$/i.test(f.ctx.left)),
649
+ ],
563
650
  // (Elsewhere a UUID is hex in groups: hexVeto.)
564
651
  ["uuid", (f, t, cred) => cred && UUID.test(t) && /session[_-]?id$|sessionId$/i.test(f.ctx.name)],
565
652
  // (A ULID's and a cuid's own timestamp says ID wherever it stands; a key's random characters rarely make one.)
@@ -605,10 +692,13 @@ const VETOES = [
605
692
  ],
606
693
  // The local part of an email address or Message-ID (`<CAOS…@mail.gmail.com>`, its first piece when an `=` in it
607
694
  // split it, `CAPig+…=JtAj…@mail.gmail.com`), a push token's body. (Not a URL's userinfo, `https://<token>@host`: a
608
- // password or a token there.)
695
+ // password or a token there; nor the password after a user and a colon with the scheme left off, as proxy variables
696
+ // are written, `HTTPS_PROXY=svc:<password>@proxy:8080`: an address's local part holds no colon.)
609
697
  [
610
698
  "local-part",
611
- (f) => (/^(?:=[\w+=.-]*)?@/.test(f.ctx.right) && !/:\/\/[^\s/@]*$/.test(f.ctx.left)) ||
699
+ (f) => (/^(?:=[\w+=.-]*)?@/.test(f.ctx.right) &&
700
+ !/:\/\/[^\s/@]*$/.test(f.ctx.left) &&
701
+ !/(?:^|[\s="'])[\w.%+-]{1,64}:$/.test(f.ctx.left)) ||
612
702
  /ExponentPushToken\[$|ExpoPushToken\[$/.test(f.ctx.left),
613
703
  ],
614
704
  // An Android app's signature hash in MSAL's redirect URI (`msauth://com.contoso.app/<base64 SHA-1>`, encoded or in
@@ -632,12 +722,16 @@ const VETOES = [
632
722
  // A fragment of a percent-encoded value (`…%2fAx%2bTHQ…`): the whole value, not a key, if anything.
633
723
  ["percent-fragment", (f) => /%[0-9A-Fa-f]{2}$/.test(f.ctx.left) || /^%[0-9A-Fa-f]{2}/.test(f.ctx.right)],
634
724
  // Cloud resource IDs: `/subscriptions/<uuid>/resourceGroups/<name>/providers/…`, ARNs.
635
- // (The path ends at a query: a parameter after `?` or `&` is its own, and a credential's name keeps it.)
725
+ // (The path ends at a query: a parameter after `?` or `&` is its own, and a credential's name keeps it. And a hook's
726
+ // segment after the resource starts a secret's path: `/projects/42/webhooks/<token>`.)
636
727
  [
637
728
  "resource-id",
638
- (f, _, cred) => !cred &&
639
- (/\/(?:subscriptions|resourceGroups|resourcegroups|providers|projects|locations|instances|clusters|namespaces)\/[^\s"'?&#]*$/.test(f.ctx.left) ||
640
- /\barn:aws[\w-]*:[^\s"'?&#]*$/.test(f.ctx.left)),
729
+ (f, _, cred) => {
730
+ if (cred)
731
+ return false;
732
+ const path = /\/(?:subscriptions|resourceGroups|resourcegroups|providers|projects|locations|instances|clusters|namespaces)\/([^\s"'?&#]*)$/.exec(f.ctx.left)?.[1];
733
+ return ((path !== undefined && !path.split("/").some(isHookSegment)) || /\barn:aws[\w-]*:[^\s"'?&#]*$/.test(f.ctx.left));
734
+ },
641
735
  ],
642
736
  // IDs that frameworks generate: cuid2 (24 lowercase characters starting with a letter, random all through: only
643
737
  // where IDs are written says it's one).
@@ -649,10 +743,15 @@ const VETOES = [
649
743
  "public-browser-key",
650
744
  (f) => f.ctx.syntax === "call" && (PUBLIC_KEY_CALLEE.test(f.ctx.name) || FULLSTORY_GLOBAL.test(f.ctx.name)),
651
745
  ],
652
- // A Solana address near Solana's words. (Other blockchain addresses and content IDs are `inert`'s: values.ts.)
746
+ // A Solana address near Solana's words. (Other blockchain addresses and content IDs are `inert`'s: values.ts.) Not
747
+ // under a credential's name, nor where an RPC provider's URL takes its key: after the API's version
748
+ // (`solana-devnet.g.alchemy.com/v2/<key>`). (A key's parameter, `?api-key=`, is a credential's name.)
653
749
  [
654
750
  "public-address",
655
- (f, t) => /^[1-9A-HJ-NP-Za-km-z]{32,44}$/.test(t) && /solana|devnet|mainnet-beta|lamports|phantom|pubkey/i.test(f.ctx.near),
751
+ (f, t, cred) => /^[1-9A-HJ-NP-Za-km-z]{32,44}$/.test(t) &&
752
+ /solana|devnet|mainnet-beta|lamports|phantom|pubkey/i.test(f.ctx.near) &&
753
+ !cred &&
754
+ !(f.ctx.inUrl && /\/v\d{1,2}\/$/.test(f.ctx.left)),
656
755
  ],
657
756
  ["timestamp", (_, t) => /^\d{8}T\d{6}Z$/.test(t)],
658
757
  // A Google Drive file's ID near Drive's words.
@@ -666,7 +765,16 @@ const VETOES = [
666
765
  ["name-hash", (f) => f.ctx.nameClass === "hash"],
667
766
  ["name-public", (f) => f.ctx.nameClass === "public"],
668
767
  ["name-identifier", (f) => f.ctx.nameClass === "identifier"],
669
- ["base64-text", (f, t, cred) => f.decodes === "text" && f.length >= 24 && !cred && decodedIsNatural(t)],
768
+ // (Shorter, only words or a JWT's header: a tokenizer's vocabulary, `IG5lY2Vzc2FyaWx5` for ` necessarily`;
769
+ // `eyJhbGciOiJFUzI1NiJ9`, `{"alg":"ES256"}`. A pair, `user:pass`, is a credential's, and other JSON a cookie's,
770
+ // Laravel's `{"iv":…}`.)
771
+ [
772
+ "base64-text",
773
+ (f, t, cred) => f.decodes === "text" &&
774
+ !cred &&
775
+ decodedIsNatural(t) &&
776
+ (f.length >= 24 || /^(?:[A-Za-z]+(?: [A-Za-z]+)*|\{"(?:alg|typ)":"[\w-]+"\})$/.test(base64Binary(t).trim())),
777
+ ],
670
778
  // Words, and words around a digit or two: package and module names (`qtquick3dphysics`, `oauth2client`).
671
779
  [
672
780
  "words",
package/dist/urls.d.ts CHANGED
@@ -15,9 +15,11 @@ export declare const URL_SCHEME: string;
15
15
  * Where a URL that starts at `at` ends: at a blank, a quote, an angle bracket, a backtick, a backslash (a JSON
16
16
  * string's escaped quote), or a closing bracket it didn't open (a Markdown link's `)`); sentence punctuation at its
17
17
  * end isn't part of it (`…/reset/<token>.`). A password's brackets are (`app:p)dgbs@db`, `admin:<Harbor2291x>@host`).
18
- * However long the password: each look-ahead for its `@` is remembered to where it stopped (`reachesAt`).
18
+ * However long the password: each look-ahead for its `@` is remembered to where it stopped (`reachesAt`). With `enough`,
19
+ * it stops once it has read past that point (the answer is then only known to be past it): urlAround asks whether a
20
+ * URL reaches a position, and a line with no blank in it (a comma-separated list of hosts) reads, as one URL, to its end.
19
21
  */
20
- export declare function urlEnd(text: string, at: number): number;
22
+ export declare function urlEnd(text: string, at: number, enough?: number): number;
21
23
  /** Whether `value` starts with a URL with a scheme. */
22
24
  export declare const isUrl: (value: string) => boolean;
23
25
  /** The URL with a scheme that `value` starts with, read to where urlEnd ends it; undefined if it starts with none. */
@@ -49,7 +51,10 @@ export interface Param {
49
51
  name: string;
50
52
  value: Span;
51
53
  }
52
- /** A parameter's start and name: after `?`, `&` (or HTML's `&amp;`) or `#`, a name with PHP's `[]` allowed. */
54
+ /**
55
+ * A parameter's start and name: after `?`, `&` (or HTML's `&amp;`, or the `\u0026` and `\x26` that JSON encoders and JS
56
+ * write for it: Go's encoding/json escapes every `&`) or `#`, a name with PHP's `[]` allowed.
57
+ */
53
58
  export declare const PARAM_START: string;
54
59
  export declare const PARAM_NAME: string;
55
60
  /**