redac---n 0.0.0-stage → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +323 -2
  3. package/THIRD_PARTY_NOTICES.md +59 -0
  4. package/dist/aho.d.ts +25 -0
  5. package/dist/aho.js +116 -0
  6. package/dist/base64.d.ts +10 -0
  7. package/dist/base64.js +29 -0
  8. package/dist/bip39-data.d.ts +2 -0
  9. package/dist/bip39-data.js +2 -0
  10. package/dist/cache.d.ts +14 -0
  11. package/dist/cache.js +40 -0
  12. package/dist/calls.d.ts +64 -0
  13. package/dist/calls.js +269 -0
  14. package/dist/charlm-data.d.ts +2 -0
  15. package/dist/charlm-data.js +3 -0
  16. package/dist/charlm.d.ts +7 -0
  17. package/dist/charlm.js +65 -0
  18. package/dist/checksums.d.ts +10 -0
  19. package/dist/checksums.js +98 -0
  20. package/dist/commands.d.ts +72 -0
  21. package/dist/commands.js +787 -0
  22. package/dist/context.d.ts +98 -0
  23. package/dist/context.js +469 -0
  24. package/dist/encoded.d.ts +11 -0
  25. package/dist/encoded.js +336 -0
  26. package/dist/environment.d.ts +10 -0
  27. package/dist/environment.js +62 -0
  28. package/dist/formats-data.d.ts +2 -0
  29. package/dist/formats-data.js +405 -0
  30. package/dist/formats.d.ts +5 -0
  31. package/dist/formats.js +470 -0
  32. package/dist/hmac.d.ts +3 -0
  33. package/dist/hmac.js +87 -0
  34. package/dist/index.d.ts +101 -0
  35. package/dist/index.js +488 -0
  36. package/dist/json.d.ts +82 -0
  37. package/dist/json.js +392 -0
  38. package/dist/known.d.ts +34 -0
  39. package/dist/known.js +340 -0
  40. package/dist/lines.d.ts +28 -0
  41. package/dist/lines.js +69 -0
  42. package/dist/lookup.d.ts +5 -0
  43. package/dist/lookup.js +7 -0
  44. package/dist/model-data.d.ts +3 -0
  45. package/dist/model-data.js +4 -0
  46. package/dist/model.d.ts +2 -0
  47. package/dist/model.js +19 -0
  48. package/dist/named.d.ts +20 -0
  49. package/dist/named.js +3257 -0
  50. package/dist/names.d.ts +106 -0
  51. package/dist/names.js +1517 -0
  52. package/dist/normalize.d.ts +25 -0
  53. package/dist/normalize.js +529 -0
  54. package/dist/pem.d.ts +32 -0
  55. package/dist/pem.js +269 -0
  56. package/dist/policy.d.ts +3 -0
  57. package/dist/policy.js +470 -0
  58. package/dist/public.d.ts +44 -0
  59. package/dist/public.js +362 -0
  60. package/dist/publickeys.d.ts +41 -0
  61. package/dist/publickeys.js +232 -0
  62. package/dist/random.d.ts +29 -0
  63. package/dist/random.js +165 -0
  64. package/dist/repeats.d.ts +21 -0
  65. package/dist/repeats.js +80 -0
  66. package/dist/rules.d.ts +42 -0
  67. package/dist/rules.js +58 -0
  68. package/dist/runs.d.ts +10 -0
  69. package/dist/runs.js +36 -0
  70. package/dist/secretfile.d.ts +52 -0
  71. package/dist/secretfile.js +722 -0
  72. package/dist/shell.d.ts +127 -0
  73. package/dist/shell.js +703 -0
  74. package/dist/spans.d.ts +61 -0
  75. package/dist/spans.js +287 -0
  76. package/dist/sql.d.ts +20 -0
  77. package/dist/sql.js +66 -0
  78. package/dist/stream.d.ts +46 -0
  79. package/dist/stream.js +150 -0
  80. package/dist/structure.d.ts +24 -0
  81. package/dist/structure.js +1625 -0
  82. package/dist/syntax.d.ts +156 -0
  83. package/dist/syntax.js +446 -0
  84. package/dist/transcript.d.ts +30 -0
  85. package/dist/transcript.js +713 -0
  86. package/dist/types.d.ts +81 -0
  87. package/dist/types.js +1 -0
  88. package/dist/unnamed.d.ts +59 -0
  89. package/dist/unnamed.js +939 -0
  90. package/dist/urls.d.ts +70 -0
  91. package/dist/urls.js +303 -0
  92. package/dist/values.d.ts +178 -0
  93. package/dist/values.js +1621 -0
  94. package/dist/yaml.d.ts +40 -0
  95. package/dist/yaml.js +174 -0
  96. package/docs/creddata.svg +318 -0
  97. package/docs/guessing.md +139 -0
  98. package/docs/holdout.svg +85 -0
  99. package/docs/layers.md +80 -0
  100. package/docs/models.svg +81 -0
  101. package/docs/real-text.svg +112 -0
  102. package/package.json +57 -5
@@ -0,0 +1,939 @@
1
+ /**
2
+ * Random tokens with no recognizable format: the key passed straight to `Client("…")`, the token in a
3
+ * neutral `X-Lumen:` header or a webhook URL's path, the value pasted into chat. Two questions decide
4
+ * each candidate:
5
+ *
6
+ * 1. Is it random? Measured against what a random string of the same length and alphabet would show
7
+ * (random.ts), and against a model of the identifiers and words people write (charlm.ts).
8
+ * 2. Does its context make a random string a secret or a hash? `api_key=`, `Client(`, `/hooks/` and
9
+ * "here's your token" say secret; `sha256:`, `"integrity":`, `commit `, `cursor=` say hash or ID.
10
+ *
11
+ * Hard rules sit outside the score: hex, UUIDs and hash-prefixed values never count without a
12
+ * credential name; API object IDs (`toolu_`, `chatcmpl-`) never count, payment and SaaS IDs (`cus_`, `pi_`)
13
+ * only when passed to a call, a header or a command. Values public by design are decided in policy.ts.
14
+ */
15
+ import { base64Binary, base64Bytes } from "./base64.js";
16
+ import { naturalness } from "./charlm.js";
17
+ import { contextOf, Doc } from "./context.js";
18
+ import { isCredentialName, isVendorClient, mentionsCredential } from "./names.js";
19
+ import { isPlaceholder } from "./public.js";
20
+ import { alphabetOf, entropyZ, shape } from "./random.js";
21
+ import { trailingRun } from "./lines.js";
22
+ import { ENCRYPTION_WRAPPER, ANALYTICS_COOKIE, isIdentifierShaped, isInert, isPodName, PASSWORD_HASH_IN_TEXT, isWordsValue, readsAsWords, } from "./values.js";
23
+ import { predict } from "./model.js";
24
+ import { backslashesBefore, escapeLength } from "./syntax.js";
25
+ import { asciiSet, longRuns } from "./runs.js";
26
+ import { urlAround } from "./urls.js";
27
+ /**
28
+ * IDs that LLM APIs and GitHub put everywhere in transcripts: public by design, whatever the context. Each prefix with
29
+ * its own separator: OpenAI's function calls are `fc_…`, and Firecrawl's API keys `fc-…`. (And Fly.io's image tags,
30
+ * `deployment-<ULID>`; GitHub's advisory IDs, `GHSA-m7j5-r2p5-c39r`, in their own alphabet; GitHub's GraphQL node IDs,
31
+ * `PRR_kwDOarJHIC…`, a type and the base64 of a MessagePack array of a version and integers; AWS's resource IDs,
32
+ * `vpc-0bddc3783c4aae5fc`, a type and 8 or 17 hex digits; Anthropic's request IDs, `req_011CwAGnE86XcDAUWRVjP5fG`.)
33
+ */
34
+ const API_OBJECT_ID = /^(?:(?:vpc|subnet|sg|i|ami|vol|snap|eni|igw|rtb|acl|nat|eipalloc|tgw|tgw-attach|lt|pcx|vpce|dopt|cgw|vgw|fs|fsmt)-[0-9a-f]{8}(?:[0-9a-f]{9})?$|[OI]v23li[A-Za-z0-9]{14}$|req_011[A-Za-z0-9]{21}$|GHSA(?:-[23456789cfghjmpqrvwx]{4}){3}$|[A-Z]{1,6}_(?:k[gw]|l[AQ])D[M-P]|(?:toolu|srvtoolu|msg|msgbatch|call|fc|resp|asst|thread|run|step|batch|vs|proj|file)_|(?:chatcmpl|cmpl|modr|ftjob|ft|img|org|file|deployment)-)/;
35
+ /**
36
+ * Prefixes of payment, messaging and SaaS object IDs (Stripe `cus_`, `pi_`, `ch_`, Twilio, Svix…). Some
37
+ * are also prefixes of keys (`re_` is Resend's), so these are public only outside a key's place (keyPlace).
38
+ */
39
+ const OBJECT_ID = /^(?:env|usr|svc|srv|db|setl|order|job|raysubmit|ins|client|sia|sms|idn|pri|pro|sub|txn|ctm|add|biz|dsc|dpl|prj|team|conn|directory|user|ap|in|ta|fu|vol|sb|sess(?=_)|we|promo|qt|cs|dp|cus|pi|ch|il|ii|price|prod|evt|acct|tr|po|pm|seti|src|card|ba|btok|tok|re|plink|si|pmc|ipi|cbtxn|ich|ic|fee|link|req|test_clock|whe|wh|ca|cn|du|py|mandate|pyr|trr|setatt|sched|subi|ri|gsk_live_?)[-_]/;
40
+ /** Right after an encryption wrapper (values.ts ENCRYPTION_WRAPPER), or SOPS's `,iv:` and `,tag:` fields, or Splunk's `$7$`. */
41
+ const AFTER_ENCRYPTION_WRAPPER = new RegExp(String.raw `(?:${ENCRYPTION_WRAPPER}|,iv:|,tag:|(?:^|[\s=:"'])\$[17]\$)$`);
42
+ /** Shapes that are IDs or encodings, not keys. */
43
+ const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
44
+ const ULID = /^[0-7][0-9A-HJKMNP-TV-Z]{25}$|^[0-7][0-9a-hjkmnp-tv-z]{25}$/;
45
+ /** Words of a path segment a secret sits under: a webhook's, a bot's, a token's, a share link's or an invite's. */
46
+ const HOOK_WORD = /^(?:hooks?|webhooks?|incomingwebhooks?|bots?|catch|tokens?|keys?|secrets?|apikeys?|invites?|invitations?|resets?|verify|magic|unsubscribe|shares?|links?|push|ping|triggers?|deploy|ingest|notify|callbacks?|incoming|dl|downloads?|send|services)$/i;
47
+ /** A path segment a secret sits under (`/hooks/<token>`, `/deploy-hooks/<id>`, `/api_keys/<key>`, Slack's `/t/<id>`, `/s/<share>`). */
48
+ function isHookSegment(segment) {
49
+ return (/^[st]$/i.test(segment) || segment.split(/[-_.]/).some((w) => HOOK_WORD.test(w)) || /^api[-_]?keys?$/i.test(segment));
50
+ }
51
+ /** The shortest token judged: 16 characters, about 95 bits in base62, the shortest key length vendors issue. */
52
+ const MIN_TOKEN = 16;
53
+ /** The characters a candidate run is made of: base64 and base64url, `=` for padding and `name=value`. */
54
+ const CANDIDATE_SET = asciiSet("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/=_-");
55
+ /** A value that is one such run, whole. */
56
+ const CANDIDATE = new RegExp(`^[A-Za-z0-9+/=_-]{${MIN_TOKEN},}$`);
57
+ /**
58
+ * A run of what `encodeURIComponent` leaves as it is, with `%XX` escapes: a percent-encoded value (`nPdn…%2FFuUO…`,
59
+ * `%40scope%2Fname`). A base64 value's `/`, `+` and `=` are escaped in one.
60
+ */
61
+ const PERCENT_RUN = /(?:[A-Za-z0-9_.~-]|%[0-9A-Fa-f]{2}){16,}/g;
62
+ /**
63
+ * Runs of token characters long enough to be a key. A run that is a path (`/api/v1/<id>/users`) or
64
+ * dotted (`a.b.c`) is split into its segments, each judged alone. A percent-encoded run that decodes to one token is
65
+ * judged whole, as its decoded value (`token` then differs from the text it spans); its pieces between the escapes
66
+ * aren't judged apart.
67
+ */
68
+ export function candidates(text) {
69
+ const out = [];
70
+ const encoded = [];
71
+ if (text.includes("%")) {
72
+ for (const m of text.matchAll(PERCENT_RUN)) {
73
+ if (!m[0].includes("%"))
74
+ continue;
75
+ let token = "";
76
+ try {
77
+ token = decodeURIComponent(m[0]);
78
+ }
79
+ catch { } // an escape that isn't UTF-8
80
+ if (token.length > 256 || !CANDIDATE.test(token))
81
+ continue;
82
+ out.push({ start: m.index, end: m.index + m[0].length, token });
83
+ encoded.push([m.index, m.index + m[0].length]);
84
+ }
85
+ }
86
+ // (Both in the text's order: the encoded run a token run may be in is the first not ended before it.)
87
+ let k = 0;
88
+ for (const [runStart, runEnd] of longRuns(text, CANDIDATE_SET, MIN_TOKEN)) {
89
+ while (k < encoded.length && encoded[k][1] <= runStart)
90
+ k++;
91
+ if (k < encoded.length && runStart >= encoded[k][0] && runEnd <= encoded[k][1])
92
+ continue;
93
+ if (runEnd - runStart > 256)
94
+ continue; // blobs: images, certificates, bundles; longer tokens are caught by format or header
95
+ const run = text.slice(runStart, runEnd);
96
+ // `name=value` inside a run (query strings, form bodies): split at an `=` that isn't padding.
97
+ for (const [eqOff, piece] of splitKeep(run, /=(?=[A-Za-z0-9+/_-])/g)) {
98
+ const parts = /\//.test(piece) && !isBase64Run(piece, text, runStart + eqOff)
99
+ ? splitKeep(piece, "/")
100
+ : [[0, piece]];
101
+ for (const [off, part] of parts) {
102
+ let lead = part.match(/^[=_+-]*/)[0].length;
103
+ const dropped = part.slice(0, lead);
104
+ // The tail of an escape before the run: `\n<token>`, `\u001b[32m<token>`, `ESC[0m<token>`, `%2b<token>`. (A
105
+ // backslash's tail can be what the token drops: `\_<token>`.)
106
+ const at = runStart + eqOff + off;
107
+ const before = text.slice(Math.max(0, at - 12), at);
108
+ if (backslashesBefore(text, at) % 2)
109
+ lead = Math.max(lead, escapeLength(text, at - 1) - 1);
110
+ else if (/(?:\x1b|\\u001b|\\x1b|\\e|\\033)\[$/i.test(before))
111
+ lead += part.match(/^[\d;]*[A-Za-z]/)?.[0].length ?? 0;
112
+ else if (text[at - 1] === "%" && /^[0-9A-Fa-f]{2}/.test(part))
113
+ lead += 2;
114
+ const body = part
115
+ .slice(lead)
116
+ .replace(/=+$/, "")
117
+ .replace(/[_-]+$/, "");
118
+ if (body.length < MIN_TOKEN)
119
+ continue;
120
+ const pad = part.slice(lead + body.length).match(/^={0,2}/)[0];
121
+ const start = runStart + eqOff + off + lead;
122
+ out.push({
123
+ start,
124
+ end: start + body.length + pad.length,
125
+ token: body + pad,
126
+ ...(dropped ? { lead: dropped } : {}),
127
+ });
128
+ }
129
+ }
130
+ }
131
+ return out;
132
+ }
133
+ /**
134
+ * A run with `/` that is base64 rather than a path: of base64's alphabet, not inside a URL or after a path's characters,
135
+ * and with `+` or padding, or with no segment that reads as a word (`src`, `heads`, `Users`), wherever it stands (a
136
+ * quoted value, a shell argument, a line of its own). `/tmp/tmpab12cd34`, `refs/heads/main` and `api/v1/users` are paths.
137
+ */
138
+ function isBase64Run(run, text, at) {
139
+ if (/[\w.~-]$/.test(text[at - 1] ?? "") || urlAround(text, at, at + run.length) >= 0)
140
+ return false;
141
+ // (With a prefix: `sha512-<base64>==`. Otherwise `_` and `-` are base64url's, which has no `/`.)
142
+ if (/[+]|=$/.test(run))
143
+ return true;
144
+ if (/[_-]/.test(run))
145
+ return false;
146
+ // (A path has a segment that is a word, `tmp`, `heads`, `Users`; a key's segments are random, wherever it stands.)
147
+ return !run.split("/").some((seg) => /^(?:[a-z]{3,}|[A-Z][a-z]{2,})$/.test(seg) && readsAsWords(seg));
148
+ }
149
+ /** `run` cut at each `sep` (a string, or a global regex), as [offset, piece], the separators dropped. */
150
+ function splitKeep(run, sep) {
151
+ const out = [];
152
+ let at = 0;
153
+ if (typeof sep === "string") {
154
+ for (let i = run.indexOf(sep); i >= 0; i = run.indexOf(sep, at)) {
155
+ out.push([at, run.slice(at, i)]);
156
+ at = i + sep.length;
157
+ }
158
+ }
159
+ else {
160
+ for (const m of run.matchAll(sep)) {
161
+ out.push([at, run.slice(at, m.index)]);
162
+ at = m.index + m[0].length;
163
+ }
164
+ }
165
+ out.push([at, run.slice(at)]);
166
+ return out;
167
+ }
168
+ /** Bytes a base64 or hex token decodes to, or 0. Hash outputs are 16, 20, 28, 32, 48 or 64 bytes. */
169
+ function decodedBytes(token, alphabet) {
170
+ if (alphabet === "hex" || alphabet === "HEX")
171
+ return token.length % 2 ? 0 : token.length / 2;
172
+ if (alphabet !== "b64" && alphabet !== "b64url" && alphabet !== "b62")
173
+ return 0;
174
+ const body = token.replace(/=+$/, "");
175
+ return Math.floor((body.length * 3) / 4);
176
+ }
177
+ /** The byte lengths of digests: MD5, SHA-1, SHA-224, SHA-256, SHA-384, SHA-512. */
178
+ const DIGEST_BYTES = [16, 20, 28, 32, 48, 64];
179
+ /** The unpadded base64 lengths of the digests that padding doesn't mark: MD5, SHA-1, SHA-256, SHA-512 (a SHA-384 or SHA-224 is as long as a random key of that size, so no length vouches for it). */
180
+ const UNPADDED_DIGEST_LENGTHS = [22, 27, 43, 86];
181
+ /** Whether a token is the size of a digest: hex of one, or base64 of one (padded, or of a length only a digest has). */
182
+ function isHashSize(token, alphabet) {
183
+ if (!DIGEST_BYTES.includes(decodedBytes(token, alphabet)))
184
+ return false;
185
+ return alphaIsHex(alphabet) || /=$/.test(token) || UNPADDED_DIGEST_LENGTHS.includes(token.length);
186
+ }
187
+ /** Whether a byte is text's: printable ASCII, a tab or a line break. */
188
+ const isTextByte = (b) => (b >= 32 && b < 127) || b === 9 || b === 10 || b === 13;
189
+ /** What base64 decodes to: printable text, binary, or not base64 at all. */
190
+ function decodesTo(token) {
191
+ const bin = /^[A-Za-z0-9+/_-]+={0,2}$/.test(token) ? base64Binary(token) : undefined;
192
+ if (!bin)
193
+ return "none";
194
+ let printable = 0;
195
+ for (let i = 0; i < bin.length; i++)
196
+ if (isTextByte(bin.charCodeAt(i)))
197
+ printable++;
198
+ return printable >= bin.length * 0.95 ? "text" : "binary";
199
+ }
200
+ /** Whether base64 decodes to ordinary text (words, JSON, a path) rather than another random token. */
201
+ function decodedIsNatural(token) {
202
+ // (Printable text first: random bytes hold a space or a slash by chance.)
203
+ if (decodesTo(token) !== "text")
204
+ return false;
205
+ const d = base64Binary(token)?.trim();
206
+ if (d === undefined)
207
+ return false;
208
+ // A token of its own, decoded (a key encoded twice, as a Kubernetes Secret holds a base64 key): not text, though its
209
+ // own base64 has a slash.
210
+ if (/^[A-Za-z0-9+/_-]{16,}={0,2}$/.test(d) && naturalness(d) < 0.5)
211
+ return false;
212
+ return /\s/.test(d) || /^[{[<"]/.test(d) || naturalness(d) > 0.5 || /^[\w.-]+@|^https?:|\//.test(d);
213
+ }
214
+ function alphaIsHex(a) {
215
+ return a === "hex" || a === "HEX";
216
+ }
217
+ export function features(doc, c, secretFile) {
218
+ const text = doc.text;
219
+ const s = shape(c.token);
220
+ const alphabet = alphabetOf(c.token);
221
+ return {
222
+ token: c.token,
223
+ alphabet,
224
+ length: c.token.length,
225
+ natural: naturalness(c.token),
226
+ entropyZ: entropyZ(c.token, alphabet),
227
+ switches: s.expectedSwitches > 0 ? s.switches / s.expectedSwitches : 0,
228
+ digits: s.digit / s.length,
229
+ upper: s.upper / s.length,
230
+ lower: s.lower / s.length,
231
+ // `acme_live_8f7Qx…`, `tok-9fK2…`: a short lowercase word, a separator, then a random body.
232
+ vendorPrefix: /^[a-z]{2,10}(?:_(?:live|test|prod|dev|sk|pk|key|api))?[_-][A-Za-z0-9]{16,}$/.test(c.token) &&
233
+ naturalness(c.token.slice(c.token.search(/[_-][A-Za-z0-9]{16,}$/) + 1)) < 0,
234
+ decodes: alphabet === "b64" || alphabet === "b64url" || alphabet === "b62" ? decodesTo(c.token) : "none",
235
+ hashSize: isHashSize(c.token, alphabet),
236
+ leftHashy: /hash|digest|checksum|sha\d|md5|integrity|fingerprint|etag|nonce|commit|revision|cursor|ksuid|ulid|uuid|guid|playlist|video|chain|mask|example|signature|\bsig\b|\btx\b|txid|transaction|block|address|public/i.test(text.slice(Math.max(0, c.start - 80), c.start)),
237
+ ctx: contextOf(doc, c.start - (c.lead?.length ?? 0), c.end),
238
+ lead: c.lead ?? "",
239
+ secretFile,
240
+ };
241
+ }
242
+ /** Whether `token` is the salt or digest of a password hash: one found whole in the word around it covers it. */
243
+ function isPasswordHashPart(left, token, right) {
244
+ const before = trailingRun(left, /[\s"'`;()]/);
245
+ const word = before + token + /^[^\s"'`;()]*/.exec(right)[0];
246
+ for (const m of word.matchAll(PASSWORD_HASH_IN_TEXT))
247
+ if (m.index <= before.length && m.index + m[0].length >= before.length + token.length)
248
+ return true;
249
+ return false;
250
+ }
251
+ /** The start of a URL, `://` on, up to a host that serves public IDs, and what follows it (a host alone ends at a path, a port, a query or a fragment). */
252
+ const PUBLIC_ID_URL = /^:\/\/(?:[\w-]+\.)*(?:(?:(?:image|stream)\.mux\.com|lh\d\.googleusercontent\.com|kroki\.io|lore\.kernel\.org|(?:docs|drive|sheets|slides|forms|script|sites|colab\.research)\.google\.com|youtube\.com|youtu\.be|figma\.com|notion\.so|notion\.site|loom\.com|vimeo\.com|miro\.com|canva\.com|mp\.weixin\.qq\.com|1drv\.ms|gist\.github(?:usercontent)?\.com|forms\.gle|goo\.gl|bit\.ly|t\.co|calendly\.com|medallia\.com|surveymonkey\.com|qualtrics\.com|typeform\.com)(?=[/:?#]|$)|join\.slack\.com\/t\/[^/\s]+\/shared_invite|vercel\.com\/[^/\s]+\/[^/\s]+\/|airtable\.com\/(?:app|shr|tbl)|(?:x|twitter)\.com\/\w+\/status\/\d+\?|nel\.heroku\.com\/reports)/i;
253
+ /**
254
+ * Public namespaces that IDs sit in: the text just before a token (`spotify:track:`, `pkg:npm/`, `pin-sha256="`), just
255
+ * after it (`.oastify.com`, `._domainkey`), or a host earlier in its URL whose paths and queries carry public
256
+ * document and video IDs (`docs.google.com`, `youtube.com`). An ID there is the namespace's, not a credential.
257
+ */
258
+ const PUBLIC_ID_CONTEXTS = [
259
+ // SDP stream identifiers (`a=ssrc:… cname:…`, `msid:`), Google image thumbnails (`tbn:`).
260
+ { left: /\b(?:cname|msid|label|mslabel):$|\btbn:$/ },
261
+ // LVM UUIDs (`PV UUID 3rsJgY-…`, `Found duplicate PV <32>`).
262
+ { left: /\b(?:[PVL]V|VG)(?: UUID)?[ \t]+$|duplicate PV[ \t]+$/ },
263
+ // DKIM selectors, HPKP pins, ACME challenge files.
264
+ { right: /^\._domainkey\b/ },
265
+ { left: /\bpin-sha256=["']?$|acme-challenge\/$/ },
266
+ // A Snowflake storage integration's external ID, given to AWS as public config: `<ACCOUNT>_SFCRole=<n>_<base64>`.
267
+ { left: /_SFCRole=\d+_$/ },
268
+ // Out-of-band testing hosts and canary tokens (`<id>.oastify.com`), a malicious package's name in an advisory.
269
+ {
270
+ right: /^\.(?:oastify\.com|burpcollaborator\.net|interact\.sh|oast\.(?:fun|pro|live|site|online|me)|canarytokens\.com)\b|^ \((?:npm|pypi|PyPI|crates\.io|RubyGems|NuGet|Go|Maven)\)/,
271
+ },
272
+ { left: /pkg:(?:npm|pypi|gem|cargo|nuget)\/$|npmjs\.com\/package\/$|pypi\.org\/project\/$/ },
273
+ // A Firebase app ID's tail: `1:123456789012:web:3f2a8c1e…`.
274
+ { left: /(?:^|[^\w:])\d:\d{6,15}:(?:web|android|ios):$/ },
275
+ { left: /\bspotify:(?:track|album|artist|playlist|show|episode|user):$/ },
276
+ // A domain's verification record in DNS (`atlassian-domain-verification=…`, `google-site-verification=…`): public.
277
+ { left: /\b[\w-]*-verification=$/ },
278
+ // GitHub's push protection bypass links, Go's checksum database tree head, GitHub Actions' OIDC request URL.
279
+ {
280
+ left: /secret-scanning\/unblock-secret\/$|go\.sum database tree\s*(?:\\n|\n)\d+\s*(?:\\n|\n)$|actions\.githubusercontent\.com\/$/,
281
+ },
282
+ // Services that put public document, image, diagram, video and invite IDs in their URLs (the host of the token's own URL, any subdomain).
283
+ { url: PUBLIC_ID_URL },
284
+ { left: /\bgdrive:\/\/\S*$/ },
285
+ ];
286
+ /** The calls whose argument is a public key or ID: a vendor's loader (`analytics.load`, `window.posthog.init`, `fbq`) or a database reference's path (`doc`, `getDoc`). A name of its own: `setPrefs` is not `FS`. */
287
+ const PUBLIC_KEY_CALLEE = /^(?:doc|collection|child)$|(?:^|\.)(?:getDoc|setDoc|authenticatedContext|branch\.init|analytics|ttq|posthog|mixpanel|amplitude|heap|hj|clarity|rudderanalytics|plausible|intercom|crisp|grecaptcha|fbq|gtag|Sentry|LogRocket|FullStory)(?:\.(?:load|init|execute|identify|render|config))?$/i;
288
+ /** FullStory's global, upper case only (`fs` is Node's). */
289
+ const FULLSTORY_GLOBAL = /(?:^|\.)FS(?:\.(?:load|init|execute|identify|render|config))?$/;
290
+ /** Whether the token sits in a public namespace (PUBLIC_ID_CONTEXTS). */
291
+ function inPublicNamespace(f) {
292
+ const left = f.ctx.left.replace(/\\\//g, "/");
293
+ // (The token's own URL: from the last `://` before it, so a host earlier in the line, or in a parameter before this one, doesn't count.)
294
+ const url = f.ctx.inUrl ? left.slice(left.lastIndexOf("://")) : "";
295
+ return PUBLIC_ID_CONTEXTS.some((c) => (!c.url || c.url.test(url)) && (!c.left || c.left.test(left)) && (!c.right || c.right.test(f.ctx.right)));
296
+ }
297
+ /** A hook's, a ping's or a download's URL, whose path segments are secrets: `/hooks/<token>`, `hc-ping.com/<uuid>`. */
298
+ function isHookUrl(f) {
299
+ if (!f.ctx.inUrl)
300
+ return false;
301
+ // (The parent or a segment up to three above it, `/services/T…/B…/<token>`; or the host, `hc-ping.com`, `hooks.slack.com`.)
302
+ const ancestors = /\/((?:[^/\s]+\/){1,4})$/.exec(f.ctx.left)?.[1].split("/").filter(Boolean) ?? [];
303
+ return (ancestors.some(isHookSegment) ||
304
+ /^bot\d/.test(ancestors.at(-1) ?? "") ||
305
+ // (The token's own URL's host: an earlier URL on the line is another's.)
306
+ /:\/\/(?:[\w-]+\.)*(?:hooks?|webhooks?|hc-ping|ping)[.-]/i.test(f.ctx.left.slice(f.ctx.left.lastIndexOf("://"))));
307
+ }
308
+ /** The token is the user information of a URL: `https://<token>@host`. */
309
+ const isUserinfo = (f) => /^[^\s"'<>@?#/]*@[\w-]+(?:\.[\w-]+)+/.test(f.ctx.right);
310
+ /**
311
+ * A URL path segment that names an object: under a collection (`/v1/predictions/<id>`), or anywhere when its shape
312
+ * says ID (a file name, `<key>~noop.jpeg`, `<id>.png`; short; one case). A mixed-case random run of 20 or more is a
313
+ * token whatever path it's in, unless the host serves media (`inews.gtimg.com`) or a content word is its parent
314
+ * (`/article/<id>`). Never a hook's URL, the user part before an `@`, or a secret file's value.
315
+ */
316
+ function urlPathId(f) {
317
+ if (f.ctx.syntax !== "path" || f.secretFile || isHookUrl(f) || isUserinfo(f))
318
+ return undefined;
319
+ if (/^[a-z][a-z-]{2,30}s$/i.test(f.ctx.pathParent) && !isHookSegment(f.ctx.pathParent))
320
+ return "collection-id";
321
+ if (f.ctx.nameClass === "credential")
322
+ return undefined;
323
+ const mediaHost = /:\/\/(?:[\w-]+\.)*[\w-]*(?:img|image|images|cdn|static|media|pic|pics|photo|photos|thumb|thumbs|resiz|assets?)[\w-]*\.[\w.-]+\/[^\s"']*$/i.test(f.ctx.left);
324
+ const contentParent = /^(?:article|articles|post|posts|p|story|stories|news|item|items|video|videos|watch|status|statuses|question|questions|answer|issue|issues|pull|commit|blob|tree|wiki|doc|docs|d|album|track|playlist|episode|product|products)$/i.test(f.ctx.pathParent);
325
+ const t = f.token;
326
+ const tokenShaped = t.length >= 20 && /[a-z]/.test(t) && /[A-Z]/.test(t) && /\d/.test(t) && !mediaHost && !contentParent;
327
+ return /^[~.][A-Za-z0-9]{2,6}\b|^~|^=[swhc]\d+/.test(f.ctx.right) || !tokenShaped ? "url-path-id" : undefined;
328
+ }
329
+ /**
330
+ * Hex, hashes and IDs until a key context says otherwise: under a credential's name, or (32 characters or more) a
331
+ * neutral name with a credential word, a handover or an argument to a vendor's client (`algoliasearch(appId,
332
+ * '8f2a…')`). In a secret file a whole value is suspect, but not hex inside a larger value (an ARN, a URL, a path).
333
+ */
334
+ function hexVeto(f) {
335
+ const t = f.token;
336
+ const cred = f.ctx.nameClass === "credential";
337
+ const wholeValue = f.secretFile && /(?:[=:][ \t]*["']?|^[ \t]*)$/.test(f.ctx.left);
338
+ // Hex in dashed groups: traceparent `00-<32>-<16>-01`, X-Ray `1-<8>-<24>`, UUID variants. IDs, not keys.
339
+ if (/^[0-9a-f]{1,8}(?:-[0-9a-f]{2,32}){2,}$/i.test(t) && !cred)
340
+ return "hex-groups";
341
+ const keyContext = f.ctx.nameClass === "neutral" &&
342
+ (mentionsCredential(f.ctx.name) || f.ctx.handover || (f.ctx.syntax === "call" && isVendorClient(f.ctx.name))) &&
343
+ !f.ctx.hashPhrase;
344
+ // A word prefix before hex (`build-9f3a…`, `etag-…`, `jb-env-…`): judged as the hex it carries.
345
+ const hexTail = /^(?:[a-z][a-z0-9]*[-_])+([0-9a-f]{16,})$/i.exec(t)?.[1];
346
+ if (hexTail && !cred && !wholeValue && !((f.ctx.lineCredential || keyContext) && hexTail.length >= 32))
347
+ return "hex-suffix";
348
+ if (!alphaIsHex(f.alphabet) || cred || wholeValue)
349
+ return undefined;
350
+ return t.length < 32 || !keyContext ? "hex" : undefined;
351
+ }
352
+ /**
353
+ * A place keys are passed, where an ID seldom stands alone, so an ID's shape (a ULID, a cuid, an object ID's prefix, a
354
+ * slug) says nothing there: an HTTP header (`X-Lumen: …`, not a log's `Error: Post …`), a handover ("your key: …"), a
355
+ * command's arguments after a prompt or a script's path (`$ ./notify.sh prod …`, not a chat's `job id …`), or a
356
+ * client's constructor or a vendor's SDK (`new Client("k3j9…")`).
357
+ */
358
+ function keyPlace({ ctx }) {
359
+ return ((ctx.syntax === "header" && (/-/.test(ctx.name) || /^authorization$/i.test(ctx.name))) ||
360
+ ctx.handover ||
361
+ (ctx.shellArg && /^[ \t]*(?:[$#>][ \t]|sudo[ \t]|\.{0,2}\/)/.test(ctx.left)) ||
362
+ (ctx.syntax === "call" &&
363
+ (/client|auth|token|key|secret|credential|api|sdk/i.test(ctx.name) || isVendorClient(ctx.name))));
364
+ }
365
+ /**
366
+ * A place IDs are written: under a name of an identifier's class or ending in `id` (`job_id`, `orderId`), as a URL's
367
+ * or a REST path's segment (`PATCH /api/posts/<id>`; not a query's value, where keys go too), or after the word `id`
368
+ * (`job id <id>`, `ID: <id>`).
369
+ */
370
+ function idPlace({ ctx }) {
371
+ return (ctx.nameClass === "identifier" ||
372
+ /(?:^|[^A-Za-z])(?:id|ID|Id)s?$|[a-z](?:Id|ID)s?$|^uids?$/.test(ctx.name) ||
373
+ (ctx.inUrl && /\/$/.test(ctx.left)) ||
374
+ /(?:^|[\s"'(])(?:\/[\w.~-]+)+\/$/.test(ctx.left) ||
375
+ /\b(?:id|uid|ref)[ \t]*[:=#]?[ \t]*["'`]?$/i.test(ctx.left));
376
+ }
377
+ /** Crockford's base32, as a ULID's timestamp is written. */
378
+ function crockford(s) {
379
+ let n = 0;
380
+ for (const c of s.toUpperCase())
381
+ n = n * 32 + "0123456789ABCDEFGHJKMNPQRSTVWXYZ".indexOf(c);
382
+ return n;
383
+ }
384
+ /** Whether milliseconds since the epoch fall between 2015 and 2035, as an ID made now has. */
385
+ function isRecentTime(ms) {
386
+ return ms >= Date.UTC(2015, 0) && ms < Date.UTC(2035, 0);
387
+ }
388
+ const VETOES = [
389
+ // The token's own shape.
390
+ ["digits", (_, t) => /^\d+$/.test(t)],
391
+ ["placeholder", (_, t) => isPlaceholder(t)],
392
+ // Hex of one short unit written over and over (`Key = ef012345ef012345ef012345ef012345`): a filler someone typed, not
393
+ // key bytes, which never repeat in step. (Other alphabets' fillers, `foobarfoobar…`, are in the model's training
394
+ // negatives, which a veto would change.)
395
+ ["repeating", (f, t) => alphaIsHex(f.alphabet) && /^(.{2,8}?)\1{2,}$/.test(t)],
396
+ ["public-id", (_, t) => API_OBJECT_ID.test(t)],
397
+ // Ciphertext inside an encryption wrapper: sops `ENC[AES256_GCM,data:…`, Spring Cloud Config `{cipher}…`, jasypt
398
+ // `ENC(…)`, WildFly `${VAULT::…}`, IIS `[enc:IISCngProvider:…`, Splunk's `$7$…`, Pulumi's passphrase-encrypted
399
+ // values (`v1:<salt>:v1:<nonce>:<ciphertext>`, `v1:<nonce>:<ciphertext>`). (Not Jenkins' or Maven's `{…}`: see
400
+ // policy.ts.)
401
+ [
402
+ "encrypted",
403
+ (f) => /(?:^|["'\s])v1:(?:[A-Za-z0-9+/=]{8,}:v1:)?(?:[A-Za-z0-9+/=]{8,}:)?$/.test(f.ctx.left) &&
404
+ /^(?::[A-Za-z0-9+/=]+)*(?:["'\s,]|$)/.test(f.ctx.right),
405
+ ],
406
+ // (And DigitalOcean App Platform's `EV[1:<nonce>:<ciphertext>]`.)
407
+ ["encrypted", (f) => AFTER_ENCRYPTION_WRAPPER.test(f.ctx.left)],
408
+ // Analytics and load-balancer cookies: `__cf_bm=`, `cf_clearance=`, `AWSALB=`, `ai_user=`, PostHog `ph_…`.
409
+ [
410
+ "analytics-cookie",
411
+ (f) => ANALYTICS_COOKIE.test(/(?:^|[\s;"'])([\w.-]{1,60})[ \t]*=[ \t]*$/.exec(f.ctx.left)?.[1] ?? ""),
412
+ ],
413
+ // Tracking parameters in URLs (Amazon `pf_rd_r=`, `utm_*`, click IDs) and Go build IDs.
414
+ [
415
+ "tracking-param",
416
+ (f) => /[?&](?:pf_rd_\w+|pd_rd_\w+|utm_\w+|fbclid|gclid|msclkid|dclid|mc_eid|igshid|_hsenc|crid|qid|sprefix|ved|usg|ei|sei|iflsig|refRID|sca_esv|sca_upv|sxsrf|oq|aqs|gs_lcp|gs_lp|uact|rlz|ictx|vin)=$/i.test(f.ctx.left),
417
+ ],
418
+ ["build-id", (f) => /\bBuildID=\S*$/.test(f.ctx.left)],
419
+ ["public-id", (f) => inPublicNamespace(f)],
420
+ // A hashed host in `~/.ssh/known_hosts` (`|1|<salt>|<HMAC-SHA1 of the host name>`, both base64 of 20 bytes): public, as the host names are.
421
+ // (With the `+` a salt can start with, which the token drops: `|1|+bsj…=|`.)
422
+ [
423
+ "hashed-host",
424
+ (f, t) => /^[A-Za-z0-9+/]{27}=$/.test(f.lead + t) && /(?:^|[\s,])\|1\|(?:[A-Za-z0-9+/]{27}=\|)?$/.test(f.ctx.left),
425
+ ],
426
+ // Facebook post IDs, LVM's PV UUIDs, OpenRouter generations, Cloudflare rays, a Snowflake external ID as a whole token.
427
+ [
428
+ "public-id",
429
+ (_, t) => /^pfbid0[1-9A-HJ-NP-Za-km-z]{30,}$|^[A-Za-z0-9]{6}(?:-[A-Za-z0-9]{4}){5}-[A-Za-z0-9]{6}$|^gen-\d{10}-[A-Za-z0-9]{12,40}$|^[0-9a-f]{16}-[A-Z]{3}$|^[A-Z0-9]+_SFCRole=/.test(t),
430
+ ],
431
+ // A protobuf message in base64 (grpcurl, gRPC-web, Google API page tokens): it parses to the last byte and
432
+ // carries readable text. Random key bytes almost never parse, let alone with a readable field.
433
+ ["encoded message", (_, t) => /^[A-Za-z0-9+/_-]+={0,2}$/.test(t) && isProtobufText(t)],
434
+ // A file in base64, by its format's magic number: gzip (`H4sI`: Google's `stick=`, kroki's diagrams), PNG, JPEG, GIF,
435
+ // PDF, zip, RIFF (WebP, WAV), WebAssembly. A key's random bytes start so once in millions at most.
436
+ [
437
+ "encoded file",
438
+ (_, t) => /^(?:H4sI|iVBORw0KGg|\/9j\/|R0lGOD[dl]h|JVBERi|UEsDB|UklGR|AGFzb)[A-Za-z0-9+/_-]{12}/.test(t),
439
+ ],
440
+ // DNSSEC record data; OpenPGP fingerprints and key IDs where an OpenPGP tool or a fingerprint is mentioned.
441
+ [
442
+ "public-id",
443
+ (f) => /\bIN[ \t]+(?:RRSIG|DNSKEY|CDNSKEY|DS|NSEC3?|NSEC3PARAM|TLSA|SSHFP)\b/.test(`${f.ctx.left} ${f.ctx.right}`),
444
+ ],
445
+ [
446
+ "public-id",
447
+ (f, t) => /^(?:[0-9A-Fa-f]{40}|[0-9A-Fa-f]{16})$/.test(t) &&
448
+ /\b(?:gpg2?|gpgv|pacman-key|apt-key)\b|\bfpr\b|fingerprint/i.test(`${f.ctx.left} ${f.ctx.right}`),
449
+ ],
450
+ // The argument of an option that takes OpenPGP key IDs (`gpg --sign-key <40 hex>`, `pacman-key --lsign-key`,
451
+ // `--recv-keys <id> <id>`, `--default-key <16 hex>`): what the option says, whatever else is on the line.
452
+ [
453
+ "key-id-option",
454
+ (f, t) => /^(?:0x)?(?:[0-9A-Fa-f]{40}|[0-9A-Fa-f]{16}|[0-9A-Fa-f]{8})$/.test(t) && KEY_ID_OPTION.test(f.ctx.left),
455
+ ],
456
+ // Model names (`diffusiongemma-26B-A4B-it-GGUF`, `Llama-3.1-8B-Instruct`), Ollama tags, and long hyphenated slugs.
457
+ [
458
+ "model-name",
459
+ (_, t) => /^[A-Za-z][\w.]*(?:-[\w.]+){2,}$/.test(t) &&
460
+ t
461
+ .split("-")
462
+ .some((p) => /^\d+(?:\.\d+)?[BbMmKk]$|^(?:A\d+B|GGUF|AWQ|GPTQ|EXL2|MLX|it|Instruct|instruct|chat|Chat|base|Base|hf|HF|Q\d_K(?:_[SML])?|fp16|bf16|int[48])$/.test(p)),
463
+ ],
464
+ [
465
+ "slug",
466
+ (_, t) => /^[a-z][a-z0-9]*(?:-[a-z0-9]+){4,}$/.test(t) &&
467
+ t.split("-").filter((p) => naturalness(p) > 0.3 || /^\d+$/.test(p)).length * 2 >= t.split("-").length,
468
+ ],
469
+ // A JWK's public members (`n`, `e`, `x`, `y`, `x5c`, `x5t`); a shell's `-n` isn't one.
470
+ [
471
+ "data-name",
472
+ (f) => /^(?:n|e|x|y|x5c|x5t)$/.test(f.ctx.name) && f.ctx.jwk && !/(?:^|\s)-[a-z]\s*["']?$/.test(f.ctx.left),
473
+ ],
474
+ // An object storage key (`'Bucket': 'b', 'Key': '<sha>/dist/…'`, `<Key>` in S3's XML): a path, not a credential.
475
+ ["object-key", (f) => /^key$/i.test(f.ctx.name) && (/\bbucket\b/i.test(f.ctx.near) || /^\//.test(f.ctx.right))],
476
+ // C's 64-bit literals (`0x1f911b719147c712ull`), rustc's codegen units (`uv_auth.<hash>-cgu.0`) and its v0 mangled
477
+ // symbols (`_RNvCs30UZn7PijB2_3uvx4main`: `_R`, then path tags and length-prefixed identifiers).
478
+ [
479
+ "number",
480
+ (f, t) => /^0x[0-9a-fA-F]+(?:u?ll?|ULL?|LL?)$/.test(t) || (/^[0-9a-f]{16}-cgu$/.test(t) && /^\.\d/.test(f.ctx.right)),
481
+ ],
482
+ [
483
+ "mangled-symbol",
484
+ (f, t) => /^R[CMXYNIBS][A-Za-z0-9_]*\d[a-z_]{2,}\w*$/.test(t) && /(?:^|[^A-Za-z0-9])_$/.test(f.ctx.left + f.lead),
485
+ ],
486
+ // A function's name, called, or an object's or a class's whose method is: `EVP_PKEY_get1_DH(pkey)`,
487
+ // `X25519PrivateKey.from_private_bytes(raw)`.
488
+ ["callee", (f, t) => /^(?:\.[A-Za-z_$][\w$]*)*\(/.test(f.ctx.right) && /^[A-Za-z_$][\w$]*$/.test(t)],
489
+ // A type name: `H225_ArrayOf_CryptoH323Token`, its parts words, acronyms and numbers (each lower-case run follows a
490
+ // capital: no digit before one).
491
+ ["identifier-shaped", (_, t) => /^[A-Z][A-Za-z0-9]*(?:_[A-Z][A-Za-z0-9]*){2,}$/.test(t) && !/\d[a-z]/.test(t)],
492
+ // Codecov's report anchors (`#diff-<base64 of a file path>`), GitHub's diff anchors (`/files#diff-<sha256>`, to a
493
+ // line `…R55`), Alpine's APK checksums (`Z:Q1<base64 SHA-1>=`, 30 characters; in a diff, `+Z:…`).
494
+ [
495
+ "hash-prefix",
496
+ (f, t) => (/^diff-[A-Za-z0-9+/=_-]{8,}$/.test(t) && decodedIsNatural(t.slice(5))) ||
497
+ /^diff-[0-9a-f]{40,64}(?:[LR]\d+)?$/.test(t) ||
498
+ (/^Q1[A-Za-z0-9+/]{27}=$/.test(t) && /(?:^|[\s"'+-])[CZ]:$/.test(f.ctx.left)),
499
+ ],
500
+ // Part of a password hash (bcrypt, argon2, crypt, Django, LDAP): kept, as the name layer keeps them.
501
+ ["password-hash", (f, t) => isPasswordHashPart(f.ctx.left, t, f.ctx.right)],
502
+ // AWS unique IDs of roles, users, groups, policies and instance profiles, and CloudFront key pair IDs.
503
+ ["aws-unique-id", (_, t) => /^(?:AROA|AIDA|AGPA|ANPA|ANVA|AIPA|APKA)[A-Z0-9]{12,20}$/.test(t)],
504
+ // (In a key's place the same prefixes are credentials: Resend `re_…`, Wompi `prod_events_…`.)
505
+ ["object-id", (f, t) => OBJECT_ID.test(t) && !keyPlace(f)],
506
+ ["hash-prefix", (f) => f.ctx.hashPrefix],
507
+ // Signatures by name, Gemini's `thoughtSignature` among them; a `sig` only when longer than an HMAC key; a mail's
508
+ // DKIM or ARC signature (`b=`), which every receiving server checks.
509
+ [
510
+ "public-signature",
511
+ (f, t) => /signatures?$/i.test(f.ctx.name) ||
512
+ (/(?:^|[^a-z])sigs?$/i.test(f.ctx.name) && t.length > 64) ||
513
+ (f.ctx.name === "b" && /\b(?:DKIM-Signature|ARC-Message-Signature|ARC-Seal):/i.test(f.ctx.near)),
514
+ ],
515
+ // Accessors, page tokens, hashes and IDs by name: Vault `token_accessor`, `nextPageToken`, detect-secrets'
516
+ // `hashed_secret`, `node_id`, `job_id`.
517
+ [
518
+ "id-name",
519
+ (f, _, cred) => !cred &&
520
+ (/accessor$|^(?:open|union)_?id$|(?:^|[_-]|next|prev)page_?token$|^(?:cursor|continuation(?:_?token)?)$|^(?:app_?id|to_?user|from_?user|key_?id|kid|InResponseTo|AssertionID|SessionIndex|slug)$|^hash(?:ed)?_|(?:^|[_\s-])id$|(?:request|trace|correlation|span|message|event|transaction|amzn-request)[-_]?id$|(?:^|[_-])(?:prefix|hint|last4|suffix)$/i.test(f.ctx.name) ||
521
+ /[a-z](?:Id|ID)$/.test(f.ctx.name)),
522
+ ],
523
+ // Named as data: `"embedding": "…"`, `frame_796=…`, `mac = …`, `cnonce=…`. (Digests, nonces and IVs by name are
524
+ // hashes: `name-hash` below.)
525
+ [
526
+ "data-name",
527
+ (f, _, cred) => !cred &&
528
+ /^(?:data|embeddings?|vectors?|payload|blob|frame(?:_?\d+)?|thumbnail|bytes|mac|cnonce|opaque|protected|aad)$/i.test(f.ctx.name),
529
+ ],
530
+ // Where it sits: a file path, a PEM block of public material, a line of a wrapped blob.
531
+ ["fs-path", (f, _, cred) => f.ctx.fsPath && !cred],
532
+ ["armored-public", (f) => f.ctx.armored],
533
+ // A PEM block's label on the line above it, a name of parts (`ALICE_cf_c2pnb163v1`, `PRIME192V1_RFC5114-Peer`): the
534
+ // block holds the key.
535
+ ["block-label", (f, t) => f.ctx.labelsBlock && /^[A-Za-z][A-Za-z0-9]*(?:[_-][A-Za-z0-9]+)+$/.test(t)],
536
+ ["blob-line", (f, _, cred) => f.ctx.blobLine && !cred],
537
+ ["ssh-public-key", (_, t) => /^AAAA[A-Z]\d?Nza|^AAAAE2Vj/.test(t)],
538
+ // (After a version's number, `1.2.`, not after the last digit of a token cut at a dot: `<id>7.<secret>`.)
539
+ ["version", (f, t) => /(?:^|[^\w.])v?\d{1,4}(?:\.\d{1,6})*\.$/.test(f.ctx.left) || /\+g[0-9a-f]{7,40}$/.test(t)],
540
+ ["hash-word", (_, t) => /(?:^|[-_./])(?:md5|sha\d*|etag|hash|checksum|digest|crc32|blake\d*)(?:[-_./]|$)/i.test(t)],
541
+ // (Under a credential's name only codes of short parts, `KAS-ECC-CDH_K-233_C7`: a product key's groups are
542
+ // longer, `VKXQR-8B2MP-7WJ4T`, and so are a made password's words, `Summer-Breeze-2024`.)
543
+ [
544
+ "identifier-shaped",
545
+ (_, t, cred) => !cred &&
546
+ (/^[A-Z0-9]{1,10}(?:[_-][A-Z0-9]{1,10}){2,}$/.test(t) ||
547
+ /^[A-Z]{2,}(?:_[A-Za-z0-9]+){2,}$/.test(t) ||
548
+ isIdentifierShaped(t)),
549
+ ],
550
+ ["pod-name", (_, t, cred) => !cred && isPodName(t)],
551
+ [
552
+ "identifier-shaped",
553
+ (_, t, cred) => cred && isIdentifierShaped(t) && t.split(/[-_.:]/).every((p) => p.length <= 3 || /^\d+$/.test(p)),
554
+ ],
555
+ [
556
+ "slug",
557
+ (f, t, cred) => /^(?:[a-z][a-z0-9]{0,15}[-_.]){2,}[a-z0-9]{1,16}$/.test(t) &&
558
+ t.split(/[-_.]/).every((p) => p.length <= 16 && (naturalness(p) > 0 || p.length <= 4)) &&
559
+ !cred &&
560
+ !keyPlace(f),
561
+ ],
562
+ ["filename", (f, _, cred) => f.ctx.fileish && !cred],
563
+ // (Elsewhere a UUID is hex in groups: hexVeto.)
564
+ ["uuid", (f, t, cred) => cred && UUID.test(t) && /session[_-]?id$|sessionId$/i.test(f.ctx.name)],
565
+ // (A ULID's and a cuid's own timestamp says ID wherever it stands; a key's random characters rarely make one.)
566
+ [
567
+ "ulid",
568
+ (f, t, cred) => ULID.test(t) && !cred && !keyPlace(f) && (idPlace(f) || isRecentTime(crockford(t.slice(0, 10)))),
569
+ ],
570
+ [
571
+ "cuid",
572
+ (f, t, cred) => /^c[a-z0-9]{24}$/.test(t) && !cred && !keyPlace(f) && (idPlace(f) || isRecentTime(parseInt(t.slice(1, 9), 36))),
573
+ ],
574
+ // A minisign public key: base64 of `Ed`, a key ID and 32 bytes of key, 42 bytes in all (`RWQ…`, 56 characters);
575
+ // or whatever minisign is given as one (`minisign -Vm <file> -P <key>`).
576
+ [
577
+ "minisign-public-key",
578
+ (f, t) => /^RW[QRST][A-Za-z0-9+/]{53}$/.test(t) || /\bminisign\b[^\n|;&]*[ \t]-P[ \t]+["']?$/.test(f.ctx.left),
579
+ ],
580
+ ["inert", (_, t) => isInert(t)],
581
+ // Pure hex or a hash under a bare `key`: an object key, a cache key, a test vector. (Not a key handed over: "your
582
+ // key: …".)
583
+ [
584
+ "bare-key-hash",
585
+ (f, t) => /^keys?$/i.test(f.ctx.name) &&
586
+ !f.ctx.handover &&
587
+ ((/[0-9a-f]{16,}/i.test(t) && /[-_/.:]/.test(t)) ||
588
+ (alphaIsHex(f.alphabet) &&
589
+ /"(?:Bucket|StorageClass|LastModified|ETag|Contents|VersionId)"|\bcache\b/i.test(f.ctx.near))),
590
+ ],
591
+ // Test vectors, examples and test output: Wycheproof, CAVP, RFC samples, "for example", pytest IDs, Jest diffs.
592
+ [
593
+ "test-vector",
594
+ (f) => alphaIsHex(f.alphabet) &&
595
+ /\btcId\b|\bCOUNT = \d|Test Case \d|test vectors?|\bKAT\b|wycheproof|\[(?:EN|DE)CRYPT\]|for example|e\.g\./i.test(f.ctx.near),
596
+ ],
597
+ // What a test compares its result to (`want := "<HMAC digest>"`, `expected = …`, `const golden = …`): output, not input.
598
+ [
599
+ "test-expectation",
600
+ (f, _, cred) => !cred && /^(?:want|wanted|expect|expected|golden)(?:[_-]?[A-Za-z0-9]+)?$/i.test(f.ctx.name),
601
+ ],
602
+ [
603
+ "test-output",
604
+ (f) => /::test\w*\[$/.test(f.ctx.left) || /\b(?:Expected|Received)(?: value)?:[ \t]*["']?$/.test(f.ctx.left),
605
+ ],
606
+ // The local part of an email address or Message-ID (`<CAOS…@mail.gmail.com>`, its first piece when an `=` in it
607
+ // split it, `CAPig+…=JtAj…@mail.gmail.com`), a push token's body. (Not a URL's userinfo, `https://<token>@host`: a
608
+ // password or a token there.)
609
+ [
610
+ "local-part",
611
+ (f) => (/^(?:=[\w+=.-]*)?@/.test(f.ctx.right) && !/:\/\/[^\s/@]*$/.test(f.ctx.left)) ||
612
+ /ExponentPushToken\[$|ExpoPushToken\[$/.test(f.ctx.left),
613
+ ],
614
+ // An Android app's signature hash in MSAL's redirect URI (`msauth://com.contoso.app/<base64 SHA-1>`, encoded or in
615
+ // the manifest's `android:path`): the app's identity, public.
616
+ [
617
+ "app-signature",
618
+ (f, t) => /^[A-Za-z0-9+/]{26,27}(?:=|%3D)?$/.test(t) &&
619
+ (/msauth(?::\/\/|%3A%2F%2F)[\w.]+(?:\/|%2F)(?:\+|%2B)?$/i.test(f.ctx.left) ||
620
+ (/android:path=["']\/\+?$/.test(f.ctx.left) && /android:scheme=["']msauth["']/.test(f.ctx.left))),
621
+ ],
622
+ // Telegram file IDs, nostr notes: public handles.
623
+ [
624
+ "public-handle",
625
+ (_, t) => /^(?:AgAC|BQAC|CAAC|BAAC|AwAC|DQAC|CQAC)[A-Za-z0-9_-]{20,}$/.test(t) || /^note1[a-z0-9]{50,}$/.test(t),
626
+ ],
627
+ // A value in an SQL row whose table and column don't name a credential: row data and IDs.
628
+ [
629
+ "sql-row",
630
+ (f, _, cred) => !cred && !!f.ctx.sqlCell && ![f.ctx.sqlCell.table, f.ctx.sqlCell.column].some(isCredentialName),
631
+ ],
632
+ // A fragment of a percent-encoded value (`…%2fAx%2bTHQ…`): the whole value, not a key, if anything.
633
+ ["percent-fragment", (f) => /%[0-9A-Fa-f]{2}$/.test(f.ctx.left) || /^%[0-9A-Fa-f]{2}/.test(f.ctx.right)],
634
+ // Cloud resource IDs: `/subscriptions/<uuid>/resourceGroups/<name>/providers/…`, ARNs.
635
+ // (The path ends at a query: a parameter after `?` or `&` is its own, and a credential's name keeps it.)
636
+ [
637
+ "resource-id",
638
+ (f, _, cred) => !cred &&
639
+ (/\/(?:subscriptions|resourceGroups|resourcegroups|providers|projects|locations|instances|clusters|namespaces)\/[^\s"'?&#]*$/.test(f.ctx.left) ||
640
+ /\barn:aws[\w-]*:[^\s"'?&#]*$/.test(f.ctx.left)),
641
+ ],
642
+ // IDs that frameworks generate: cuid2 (24 lowercase characters starting with a letter, random all through: only
643
+ // where IDs are written says it's one).
644
+ ["cuid2", (f, t, cred) => /^[a-z][a-z0-9]{23}$/.test(t) && !cred && idPlace(f)],
645
+ // A label of a URL's host (`https://<id>.appsync-api.…`).
646
+ ["hostname", (f) => /:\/\/[\w-]*$/.test(f.ctx.left) && /^\./.test(f.ctx.right)],
647
+ // Browser analytics and widget loaders take public keys: `analytics.load("…")`, `posthog.init('…')`.
648
+ [
649
+ "public-browser-key",
650
+ (f) => f.ctx.syntax === "call" && (PUBLIC_KEY_CALLEE.test(f.ctx.name) || FULLSTORY_GLOBAL.test(f.ctx.name)),
651
+ ],
652
+ // A Solana address near Solana's words. (Other blockchain addresses and content IDs are `inert`'s: values.ts.)
653
+ [
654
+ "public-address",
655
+ (f, t) => /^[1-9A-HJ-NP-Za-km-z]{32,44}$/.test(t) && /solana|devnet|mainnet-beta|lamports|phantom|pubkey/i.test(f.ctx.near),
656
+ ],
657
+ ["timestamp", (_, t) => /^\d{8}T\d{6}Z$/.test(t)],
658
+ // A Google Drive file's ID near Drive's words.
659
+ [
660
+ "public-url-id",
661
+ (f, t, cred) => /^1[A-Za-z0-9_-]{32}$|^1[A-Za-z0-9_-]{43}$/.test(t) &&
662
+ /drive|docs?\b|sheets?|spreadsheet|slides|folder/i.test(f.ctx.near) &&
663
+ !cred,
664
+ ],
665
+ // What the name says, when it isn't a credential's.
666
+ ["name-hash", (f) => f.ctx.nameClass === "hash"],
667
+ ["name-public", (f) => f.ctx.nameClass === "public"],
668
+ ["name-identifier", (f) => f.ctx.nameClass === "identifier"],
669
+ ["base64-text", (f, t, cred) => f.decodes === "text" && f.length >= 24 && !cred && decodedIsNatural(t)],
670
+ // Words, and words around a digit or two: package and module names (`qtquick3dphysics`, `oauth2client`).
671
+ [
672
+ "words",
673
+ (_, t) => (/[_-]/.test(t) && isWordsValue(t)) ||
674
+ (/^[a-z]{3,}\d{1,2}[a-z]{3,}$/.test(t) && t.split(/\d+/).every((p) => naturalness(p) > 0.3)),
675
+ ],
676
+ ["natural", (f) => f.natural > 1.2],
677
+ urlPathId,
678
+ hexVeto,
679
+ ];
680
+ /** An option that takes OpenPGP key IDs, and the IDs already given it, just before a token. */
681
+ const KEY_ID_OPTION = /(?:^|[ \t])--(?:(?:recv|send|delete|delete-secret|delete-secret-and-public|edit|export|export-secret|export-secret-sub|list|list-secret|list-public|fingerprint|l?sign|default|trusted|refresh)-keys?|quick-(?:sign|lsign|add|revoke|set-expire|set-primary-uid|revoke-sig|add-adsk|add-uid|revoke-uid)(?:-key)?|local-user|recipient|hidden-recipient|encrypt-to)(?:[ \t]+(?:0x)?[0-9A-Fa-f]{8,40})*[ \t]+["']?(?:0x)?$/;
682
+ /**
683
+ * The vetoes that say a value is public by design, whatever names it: the same in every layer, so policy.ts asks them
684
+ * of every named and structural proposal too (publicReason). Each reads the value's own format or its own place (an
685
+ * encryption wrapper, a public armored block, the option that takes it), which no name can contradict. The others
686
+ * are guesses about a token nothing names, from its shape or from words elsewhere on its line (a model's name, a
687
+ * wallet address near `solana`, a hex ID near `gpg`): a credential's name or a password's flag outvotes those.
688
+ */
689
+ const PUBLIC_VETOES = new Set([
690
+ "encrypted",
691
+ "aws-unique-id",
692
+ "hash-prefix",
693
+ "armored-public",
694
+ "ssh-public-key",
695
+ "minisign-public-key",
696
+ "key-id-option",
697
+ ]);
698
+ /**
699
+ * Why the value at `[start, end)` is public by design, by the vetoes in PUBLIC_VETOES; undefined if none says so. Only
700
+ * of a value the random layer would judge, one run of token characters as long as MIN_TOKEN, which they're written for.
701
+ */
702
+ export function publicReason(doc, start, end) {
703
+ const token = doc.text.slice(start, end);
704
+ if (!CANDIDATE.test(token))
705
+ return undefined;
706
+ const f = features(doc, { start, end, token }, false);
707
+ const cred = f.ctx.nameClass === "credential";
708
+ for (const v of VETOES)
709
+ if (typeof v !== "function" && PUBLIC_VETOES.has(v[0]) && v[1](f, token, cred))
710
+ return v[0];
711
+ return undefined;
712
+ }
713
+ /** The reason `veto` gives for the candidate, if it applies. */
714
+ function reasonOf(veto, f, cred) {
715
+ return typeof veto === "function" ? veto(f) : veto[1](f, f.token, cred) ? veto[0] : undefined;
716
+ }
717
+ /** Every hard rule that rules the candidate out (see VETOES), by its reason: for tracing, where vetoed() stops at the first. */
718
+ export function vetoes(f) {
719
+ const cred = f.ctx.nameClass === "credential";
720
+ return [...new Set(VETOES.map((v) => reasonOf(v, f, cred)).filter((r) => r !== undefined))];
721
+ }
722
+ /** The first hard rule that rules the candidate out, or undefined. */
723
+ export function vetoed(f, { statistical = true } = {}) {
724
+ const cred = f.ctx.nameClass === "credential";
725
+ for (const v of VETOES) {
726
+ const reason = reasonOf(v, f, cred);
727
+ if (reason && (statistical || !STATISTICAL_VETOES.has(reason)))
728
+ return reason;
729
+ }
730
+ return undefined;
731
+ }
732
+ /**
733
+ * The vetoes that are a judgment about how a token looks, which the model's features measure too (its naturalness,
734
+ * its alphabet, its name's class), rather than a fact about what it is: without them (`without: ["statistical-vetoes"]`),
735
+ * the model decides those tokens alone. For measuring whether a model trained on them can (scripts/unnamed-data.ts
736
+ * `--statistical`).
737
+ */
738
+ const STATISTICAL_VETOES = new Set([
739
+ "natural",
740
+ "words",
741
+ "slug",
742
+ "identifier-shaped",
743
+ "hex",
744
+ "hex-groups",
745
+ "hex-suffix",
746
+ "name-identifier",
747
+ "name-hash",
748
+ "id-name",
749
+ "model-name",
750
+ "pod-name",
751
+ ]);
752
+ /** The model's probability that the candidate is a secret: boosted trees trained by scripts/train-unnamed.py. */
753
+ export function score(f) {
754
+ return predict(vector(f));
755
+ }
756
+ // The syntaxes and alphabets that tell the model something. (Not prose, which no tree split on, nor hex, which
757
+ // hexVeto decides before the model sees it.)
758
+ const SYNTAXES = ["assign", "flag", "header", "query", "path", "call", "xml", "bare", "list", "other"];
759
+ const ALPHABETS = ["b32", "b36", "B36", "b62", "b64", "b64url", "other"];
760
+ const flag = (b) => (b ? 1 : 0);
761
+ /** The model's inputs, in order: each feature's name (for inspecting the model) and its value. */
762
+ const FEATURES = [
763
+ ["natural", (f) => f.natural],
764
+ ["natural_neg", (f) => Math.max(0, -f.natural)],
765
+ ["entropyZ", (f) => Math.max(-6, Math.min(4, f.entropyZ))],
766
+ ["entropyZ_low", (f) => flag(f.entropyZ < -2.5)],
767
+ // (Past the longest key it was trained on, the model has seen only blobs: a longer token of base62 or URL-safe base64,
768
+ // as OAuth and session tokens are, is judged as that longest. Standard base64 keeps its length: its keys run to
769
+ // `openssl rand -base64 128`'s 172 characters in training, and longer is a blob. So does a URL path's segment, an
770
+ // image's or a payload's ID, but for a hook's.)
771
+ [
772
+ "logLength",
773
+ (f) => Math.log2(f.alphabet === "b64" || (f.ctx.inUrl && /\/$/.test(f.ctx.left) && !isHookUrl(f))
774
+ ? f.length
775
+ : Math.min(f.length, LONGEST_TRAINED)),
776
+ ],
777
+ ["len_lt20", (f) => flag(f.length < 20)],
778
+ ["len_ge32", (f) => flag(f.length >= 32)],
779
+ ["len_ge64", (f) => flag(f.length >= 64)],
780
+ ["switches", (f) => Math.min(2, f.switches)],
781
+ ["digits", (f) => f.digits],
782
+ ["upper", (f) => f.upper],
783
+ ["lower", (f) => f.lower],
784
+ ["vendorPrefix", (f) => flag(f.vendorPrefix)],
785
+ ["decodes_binary", (f) => flag(f.decodes === "binary")],
786
+ ["hashSize", (f) => flag(f.hashSize)],
787
+ ["leftHashy", (f) => flag(f.leftHashy)],
788
+ ...ALPHABETS.map((a) => [`alpha_${a}`, (f) => flag(f.alphabet === a)]),
789
+ ["name_neutral", (f) => flag(f.ctx.nameClass === "neutral" && !!f.ctx.name)],
790
+ ["name_none", (f) => flag(!f.ctx.name)],
791
+ ["lineCredential", (f) => flag(f.ctx.lineCredential)],
792
+ ["lineHash", (f) => flag(f.ctx.lineHash)],
793
+ ["handover", (f) => flag(f.ctx.handover)],
794
+ ["quoted", (f) => flag(f.ctx.quoted)],
795
+ ["inUrl", (f) => flag(f.ctx.inUrl)],
796
+ ["path_hooky", (f) => flag(isHookSegment(f.ctx.pathParent))],
797
+ ["shellArg", (f) => flag(f.ctx.shellArg)],
798
+ ...SYNTAXES.map((s) => [`syntax_${s}`, (f) => flag(f.ctx.syntax === s)]),
799
+ ["neutral_x_long", (f) => flag(f.ctx.nameClass === "neutral" && f.length >= 32)],
800
+ ];
801
+ export function vector(f) {
802
+ return FEATURES.map(([, value]) => value(f));
803
+ }
804
+ /** The model's inputs by name, as vector() gives them. */
805
+ export function namedVector(f) {
806
+ return Object.fromEntries(FEATURES.map(([name, value]) => [name, value(f)]));
807
+ }
808
+ /** The longest random key in the model's training positives but for standard base64's (scripts/unnamed-data.ts `randomKey`). */
809
+ const LONGEST_TRAINED = 90;
810
+ /** The score a token needs, by how hard to guess (RedactOptions.guessing); in a secret file, the eager one. */
811
+ const THRESHOLD = { confident: 0.8, eager: 0.5 };
812
+ /** The score a token needs to be hidden; undefined where nothing is guessed ("off", outside a secret file). */
813
+ export function thresholdFor(guessing, secretFile) {
814
+ if (secretFile || guessing === "eager")
815
+ return THRESHOLD.eager;
816
+ return guessing === "off" ? undefined : THRESHOLD.confident;
817
+ }
818
+ /**
819
+ * A long mixed-case random value quoted on its own (`"X1Am…"`, `` `X1Am…` `` in prose), with no name: the model learned
820
+ * quoted strings from code, where they're rarely keys, but this shape is one (0.01 per MB of real text). Hidden in the
821
+ * balanced mode whatever the model scores it.
822
+ */
823
+ function isLoneQuotedKey(f) {
824
+ return (f.ctx.quoted &&
825
+ !f.ctx.name &&
826
+ (f.ctx.syntax === "bare" || f.ctx.syntax === "list") &&
827
+ /^[A-Za-z0-9]{32,64}$/.test(f.token) &&
828
+ /[a-z]/.test(f.token) &&
829
+ /[A-Z]/.test(f.token) &&
830
+ /\d/.test(f.token) &&
831
+ f.natural < -0.3 &&
832
+ !/(?:\b[A-Z][A-Z0-9_]{2,}|\b[a-z][a-z0-9]*_[a-z0-9_]+)[ \t:=]+["'`]$/.test(f.ctx.left));
833
+ }
834
+ /** The vetoes that say a token is an ID by its shape where it stands, and the shapes they read (another veto may come first). */
835
+ const ID_VETOES = new Set(["ulid", "cuid", "cuid2", "public-id", "object-id"]);
836
+ const ID_SHAPED = /^[a-z][a-z0-9]{23,24}$|^[0-7][0-9A-Za-z]{25}$|^[a-z]{2,12}_/;
837
+ export function findUnnamed(text, taken, guessing, secretFile, without = new Set()) {
838
+ const out = [];
839
+ // (In a secret file a generated-looking value is a secret whatever it's named: the eager threshold, whatever the
840
+ // guessing; outside one, "off" guesses nothing.)
841
+ const threshold = thresholdFor(guessing, secretFile);
842
+ if (threshold === undefined)
843
+ return out;
844
+ let doc;
845
+ // (A token the text shows as an ID somewhere, `/api/posts/<id>`, is that ID where it stands alone: `Post <id> not found`;
846
+ // not where a key goes, `Client("<id>")`, nor where a credential's name has it, `config.password = "<id>"`.)
847
+ const ids = new Set();
848
+ const keyed = new Set();
849
+ for (const c of candidates(text)) {
850
+ if (taken(c.start, c.end))
851
+ continue;
852
+ // Cheap randomness gate before the context work: most candidates are identifiers and words.
853
+ if (naturalness(c.token) > 1.2 && !/\d/.test(c.token))
854
+ continue;
855
+ doc ??= Doc.of(text);
856
+ // (Measuring what context buys: the token scored where nothing surrounds it.)
857
+ const f = without.has("token-context")
858
+ ? features(new Doc(c.token), { start: 0, end: c.token.length, token: c.token }, secretFile)
859
+ : features(doc, c, secretFile);
860
+ const reason = without.has("vetoes") ? undefined : vetoed(f, { statistical: !without.has("statistical-vetoes") });
861
+ if (reason) {
862
+ if (ID_VETOES.has(reason) || (ID_SHAPED.test(c.token) && vetoes(f).some((r) => ID_VETOES.has(r))))
863
+ ids.add(c.token);
864
+ continue;
865
+ }
866
+ let p = score(f);
867
+ if (isLoneQuotedKey(f))
868
+ p = Math.max(p, THRESHOLD.confident);
869
+ if (p < threshold)
870
+ continue;
871
+ const finding = {
872
+ start: c.start,
873
+ end: c.end,
874
+ kind: "random token",
875
+ rule: "unnamed",
876
+ confidence: p,
877
+ ...(f.ctx.name ? { name: f.ctx.name } : {}),
878
+ };
879
+ if (keyPlace(f) || (f.ctx.name && isCredentialName(f.ctx.name)))
880
+ keyed.add(finding);
881
+ out.push(finding);
882
+ }
883
+ return ids.size ? out.filter((f) => !ids.has(text.slice(f.start, f.end)) || keyed.has(f)) : out;
884
+ }
885
+ /** Whether base64 decodes to a complete protobuf message with at least one readable string field (4+ characters, mostly printable). */
886
+ function isProtobufText(b64) {
887
+ const bytes = base64Bytes(b64);
888
+ return !!bytes && parsesAsMessage(bytes, 0, bytes.length, 0) === true;
889
+ }
890
+ /** The highest field number read as a schema's: messages number their fields densely from 1, and a larger "tag" is random bytes. */
891
+ const MAX_FIELD_NUMBER = 536;
892
+ /** `true` if bytes[from, to) is a message with a readable string somewhere in it; `false` if it parses without one; `undefined` if it doesn't parse. */
893
+ function parsesAsMessage(bytes, from, to, depth) {
894
+ let i = from;
895
+ let readable = false;
896
+ // (Up to 5 bytes for a key or a length, 10 for a value: a negative int64 takes all 10.)
897
+ const varint = (longest = 5) => {
898
+ let v = 0;
899
+ for (let shift = 0; shift < 7 * longest && i < to; shift += 7) {
900
+ const b = bytes[i++];
901
+ v += (b & 127) * 2 ** shift;
902
+ if (!(b & 128))
903
+ return v;
904
+ }
905
+ return undefined;
906
+ };
907
+ while (i < to) {
908
+ const key = varint();
909
+ if (key === undefined)
910
+ return undefined;
911
+ // (A tag is `field << 3 | wire type`, up to 2^35 here: `>>` would wrap past 32 bits. `& 7` keeps the low bits.)
912
+ const field = Math.floor(key / 8);
913
+ if (field === 0 || field > MAX_FIELD_NUMBER)
914
+ return undefined;
915
+ const wire = key & 7;
916
+ if (wire === 0) {
917
+ if (varint(10) === undefined)
918
+ return undefined;
919
+ }
920
+ else if (wire === 1 || wire === 5) {
921
+ i += wire === 1 ? 8 : 4;
922
+ }
923
+ else if (wire === 2) {
924
+ const len = varint();
925
+ if (len === undefined || i + len > to)
926
+ return undefined;
927
+ const s = bytes.subarray(i, i + len);
928
+ const printable = s.filter(isTextByte).length;
929
+ if (len >= 4 && printable >= 0.8 * len && /[ -~]{4}/.test(String.fromCharCode(...s)))
930
+ readable = true;
931
+ else if (len >= 2 && depth < 4 && parsesAsMessage(bytes, i, i + len, depth + 1))
932
+ readable = true;
933
+ i += len;
934
+ }
935
+ else
936
+ return undefined;
937
+ }
938
+ return i === to ? readable : undefined;
939
+ }