redac---n 0.0.1 → 0.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/commands.js +5 -2
- package/dist/index.js +7 -3
- package/dist/names.d.ts +12 -0
- package/dist/names.js +70 -2
- package/dist/repeats.js +5 -1
- package/dist/secretfile.js +59 -13
- package/dist/spans.d.ts +8 -0
- package/dist/spans.js +15 -0
- package/dist/structure.js +13 -2
- package/dist/transcript.js +7 -3
- package/docs/creddata.svg +12 -12
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -7,7 +7,7 @@ up in the context sent to the provider, any router in between, and your logs. re
|
|
|
7
7
|
text and replaces them with placeholders, and leaves alone the commit SHAs, digests, IDs and code the agent needs to
|
|
8
8
|
keep working.
|
|
9
9
|
|
|
10
|
-
TypeScript, no dependencies, no Node-only APIs
|
|
10
|
+
TypeScript, no dependencies, no Node-only APIs: it runs on Node 20+, Bun and in the browser.
|
|
11
11
|
|
|
12
12
|
```sh
|
|
13
13
|
npm install redac---n
|
package/dist/commands.js
CHANGED
|
@@ -24,7 +24,7 @@ const SECRET_FILE = [
|
|
|
24
24
|
/^\.?vault[_-]?pass(?:word)?(?:\.txt)?$/i,
|
|
25
25
|
/(?:^|[_.-])(?:pass(?:word|wd|phrase|code)?|secret|token|pin(?:[_-]?code)?)(?:\.txt)?$/i,
|
|
26
26
|
/^(?:pap|chap)-secrets(?:[-_.][\w-]+)?$/,
|
|
27
|
-
/^(_netrc|rclone\.conf|cookies\.txt|keystore\.properties|wp-config\.php|\.?htpasswd|redis\.conf|sentinel\.conf|keyfile|userlist\.txt)$/i,
|
|
27
|
+
/^(_netrc|rclone\.conf|cookies\.txt|keystore\.properties|wp-config\.php|\.?htpasswd|redis\.conf|sentinel\.conf|users\.acl|keyfile|userlist\.txt)$/i,
|
|
28
28
|
/(^|[-_.])secrets?(\.[\w-]{1,40})?\.(json|ya?ml|toml|env|ini|properties)$/i,
|
|
29
29
|
/\.(tfvars|tfstate)(\.json|\.backup)?$/,
|
|
30
30
|
/^kubeconfig(\..+)?$|\.kubeconfig$/,
|
|
@@ -35,6 +35,8 @@ const SECRET_FILE = [
|
|
|
35
35
|
/realm\.properties$/,
|
|
36
36
|
];
|
|
37
37
|
const SECRET_DIR = /(^|\/)\.(aws|ssh|gnupg|docker|kube|azure)\/|(^|\/)\.config\/(gcloud|gh|rclone|sops)\/|(^|\/)\.?secrets\//;
|
|
38
|
+
/** Secret files by their directory and name: Bundler's config, which holds gem servers' credentials (names.ts isBundlerHostName). */
|
|
39
|
+
const SECRET_PATH = /(^|\/)\.bundle\/config$/;
|
|
38
40
|
// (Also a key's public half by its name, `cache-pub-key.pem`, `server.pub.pem`, `public.key`; and a certificate's,
|
|
39
41
|
// `fullchain.pem`, `cert.pem`, but not a private key's container named for its certificate, `cert.key`, `server.crt.p12`,
|
|
40
42
|
// `cert-key.pem`.
|
|
@@ -46,7 +48,7 @@ export function isSecretFile(path) {
|
|
|
46
48
|
const name = path.slice(path.lastIndexOf("/") + 1);
|
|
47
49
|
if (!name || EXAMPLE_FILE.test(name) || NOT_SECRET_PATH.test(path))
|
|
48
50
|
return false;
|
|
49
|
-
return SECRET_FILE.some((r) => r.test(name)) || SECRET_DIR.test(path);
|
|
51
|
+
return SECRET_FILE.some((r) => r.test(name)) || SECRET_DIR.test(path) || SECRET_PATH.test(path);
|
|
50
52
|
}
|
|
51
53
|
/**
|
|
52
54
|
* Whether a value is the name of a file that is a secret file or its template, not a password: `.env.example`, `.env.local`,
|
|
@@ -708,6 +710,7 @@ const TOOLS = [
|
|
|
708
710
|
[/^(?:supabase|altool)$/, { flags: new Set(["-p", "--password"]), next: "always", attached: false }],
|
|
709
711
|
[/^mosquitto_(?:pub|sub|rr)$/, { flags: new Set(["-P", "--pw"]), next: "always", attached: false }],
|
|
710
712
|
[/^beeline$/, { flags: new Set(["-p"]), next: "always", attached: false }],
|
|
713
|
+
[/^cypher-shell$/, { flags: new Set(["-p", "--password"]), next: "always", attached: false }],
|
|
711
714
|
[/^az$/, { flags: new Set(["-p", "--password"]), next: "always", attached: false, subcommand: "login" }],
|
|
712
715
|
[/^gh$/, { flags: new Set(["-b", "--body"]), next: "always", attached: false, subcommand: "secret" }],
|
|
713
716
|
[/^(?:zip|unzip|zipcloak)$/, { flags: new Set(["-P", "--password"]), next: "always", attached: false }],
|
package/dist/index.js
CHANGED
|
@@ -21,7 +21,7 @@ import { findEncoded, findHexDumps } from "./encoded.js";
|
|
|
21
21
|
import { jsonLiteralSpans, jsonStringLiterals, mayHoldJsonText, pythonReprLiterals } from "./json.js";
|
|
22
22
|
import { repeats } from "./repeats.js";
|
|
23
23
|
import { RedactionStream } from "./stream.js";
|
|
24
|
-
import { covered, isLearnable, toWordEnd, trimmedToValue, merge, overlaps, secretValue, wrapped } from "./spans.js";
|
|
24
|
+
import { covered, isLearnable, loginPassword, toWordEnd, trimmedToValue, merge, overlaps, secretValue, wrapped, } from "./spans.js";
|
|
25
25
|
import { isGuess, isVouched, layerOf, needsJudging, PLACEHOLDER_RULE } from "./rules.js";
|
|
26
26
|
import { lowerBound } from "./lines.js";
|
|
27
27
|
import { findEmbeddedOutputs, findInSecretFile, findPrintedSecret, firstPrompt, secretFileSections, } from "./secretfile.js";
|
|
@@ -454,8 +454,12 @@ export class Redactor {
|
|
|
454
454
|
const before = this.known.size;
|
|
455
455
|
for (const f of this.detect(text, options)) {
|
|
456
456
|
const value = secretValue(text, f);
|
|
457
|
-
if (isLearnable(f, value, options.minConfidence))
|
|
458
|
-
|
|
457
|
+
if (!isLearnable(f, value, options.minConfidence))
|
|
458
|
+
continue;
|
|
459
|
+
this.known.add(value, f.kind);
|
|
460
|
+
const password = loginPassword(f, value);
|
|
461
|
+
if (password)
|
|
462
|
+
this.known.add(password, f.kind);
|
|
459
463
|
}
|
|
460
464
|
return this.known.size - before;
|
|
461
465
|
}
|
package/dist/names.d.ts
CHANGED
|
@@ -3,6 +3,18 @@ export declare const PASSWORD_SPELLINGS: string[];
|
|
|
3
3
|
export declare const FOREIGN_PASSWORD_SPELLINGS: string[];
|
|
4
4
|
/** The words of a name in upper case: `clientSecret` → CLIENT, SECRET; `x-api-key` → X, API, KEY. */
|
|
5
5
|
export declare function words(name: string): string[];
|
|
6
|
+
/**
|
|
7
|
+
* Whether a name names a session or its ID, whoever's session it is: `INSTAGRAM_SESSION_ID`, `LINKEDIN_SESSION`,
|
|
8
|
+
* `tiktokSessionId`; not a terminal's or a desktop's (LOCAL_SESSION_OWNERS). Read so only in a secret file, which says
|
|
9
|
+
* its values are secrets: elsewhere a session's ID under another's name is as often a checkout's or an agent's
|
|
10
|
+
* (`CHECKOUT_SESSION_ID`, `CLAUDE_SESSION_ID`), and only the names isSessionIdName knows are credentials.
|
|
11
|
+
*/
|
|
12
|
+
export declare function namesSession(name: string): boolean;
|
|
13
|
+
/**
|
|
14
|
+
* Whether a name's value may be a login, `user:password` or `user/password` (spans.ts loginPassword): by a word of
|
|
15
|
+
* LOGIN_WORDS, or Bundler's name for a host.
|
|
16
|
+
*/
|
|
17
|
+
export declare function namesLogin(name: string): boolean;
|
|
6
18
|
/** File names' extensions, so a file name that holds a credential word (`has_key_spec.rb`) isn't one; never a qualifier (`.prod`, `.old`, `.value`, `.test`). */
|
|
7
19
|
export declare const FILE_EXTENSION: RegExp;
|
|
8
20
|
export declare const VENDOR_CLIENTS: Set<string>;
|
package/dist/names.js
CHANGED
|
@@ -656,6 +656,74 @@ export function words(name) {
|
|
|
656
656
|
const SESSION_COOKIE = /^_?(?!(?:tmux|screen|zellij|byobu|desktop|xdg|ssh|term|dbus|shell|login|vnc|x11)_)[a-z][a-z0-9]*_session$|^_session_id$|^ASP\.NET_SessionId$|^_gh_sess$/;
|
|
657
657
|
const SESSION_ID = /^(?:x-)?(?:auth[_-]|user[_-]|web[_-]|app[_-])?(?:session|sess)[_-]?id$|^(?:auth|user|web|app)?[sS]essionId$|^(?:php)?sessid$|^connect\.sid$|^(?:__Secure-[13]P)?(?:SID|HSID|APISID|SAPISID|OSID|LSID|SIDTS|SIDCC)$/i;
|
|
658
658
|
const isSessionIdName = (name) => (name !== "Session-ID" && SESSION_ID.test(name)) || SESSION_COOKIE.test(name);
|
|
659
|
+
/**
|
|
660
|
+
* The owners of sessions that are no web login: a terminal's, a desktop's, a remote shell's (`ITERM_SESSION_ID`,
|
|
661
|
+
* `WT_SESSION`, `XDG_SESSION_ID`, `DESKTOP_SESSION`), as an environment's listing shows them.
|
|
662
|
+
*/
|
|
663
|
+
const LOCAL_SESSION_OWNERS = new Set([
|
|
664
|
+
"TMUX",
|
|
665
|
+
"SCREEN",
|
|
666
|
+
"ZELLIJ",
|
|
667
|
+
"BYOBU",
|
|
668
|
+
"DESKTOP",
|
|
669
|
+
"XDG",
|
|
670
|
+
"SSH",
|
|
671
|
+
"TERM",
|
|
672
|
+
"ITERM",
|
|
673
|
+
"TERMINAL",
|
|
674
|
+
"WT",
|
|
675
|
+
"KITTY",
|
|
676
|
+
"WEZTERM",
|
|
677
|
+
"KONSOLE",
|
|
678
|
+
"DBUS",
|
|
679
|
+
"SHELL",
|
|
680
|
+
"LOGIN",
|
|
681
|
+
"VNC",
|
|
682
|
+
"X11",
|
|
683
|
+
]);
|
|
684
|
+
/**
|
|
685
|
+
* Whether a name names a session or its ID, whoever's session it is: `INSTAGRAM_SESSION_ID`, `LINKEDIN_SESSION`,
|
|
686
|
+
* `tiktokSessionId`; not a terminal's or a desktop's (LOCAL_SESSION_OWNERS). Read so only in a secret file, which says
|
|
687
|
+
* its values are secrets: elsewhere a session's ID under another's name is as often a checkout's or an agent's
|
|
688
|
+
* (`CHECKOUT_SESSION_ID`, `CLAUDE_SESSION_ID`), and only the names isSessionIdName knows are credentials.
|
|
689
|
+
*/
|
|
690
|
+
export function namesSession(name) {
|
|
691
|
+
const w = words(name);
|
|
692
|
+
if (w.at(-1) === "ID")
|
|
693
|
+
w.pop();
|
|
694
|
+
return /^(?:SESSION|SESSIONID|SESS|SESSID)$/.test(w.at(-1) ?? "") && !w.some((x) => LOCAL_SESSION_OWNERS.has(x));
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* Words that name a login, a username and its password together: `NEO4J_AUTH`, `BASIC_AUTH`, `REGISTRY_CREDENTIALS`,
|
|
698
|
+
* `CURLOPT_USERPWD`.
|
|
699
|
+
*/
|
|
700
|
+
const LOGIN_WORDS = new Set([
|
|
701
|
+
"AUTH",
|
|
702
|
+
"BASICAUTH",
|
|
703
|
+
"CREDENTIAL",
|
|
704
|
+
"CREDENTIALS",
|
|
705
|
+
"CRED",
|
|
706
|
+
"CREDS",
|
|
707
|
+
"LOGIN",
|
|
708
|
+
"USERPASS",
|
|
709
|
+
"USERPWD",
|
|
710
|
+
]);
|
|
711
|
+
/**
|
|
712
|
+
* Whether a name's value may be a login, `user:password` or `user/password` (spans.ts loginPassword): by a word of
|
|
713
|
+
* LOGIN_WORDS, or Bundler's name for a host.
|
|
714
|
+
*/
|
|
715
|
+
export function namesLogin(name) {
|
|
716
|
+
return isBundlerHostName(name) || words(name).some((w) => LOGIN_WORDS.has(w));
|
|
717
|
+
}
|
|
718
|
+
/**
|
|
719
|
+
* Bundler's setting for a gem server's credentials, named by its host (`.` written `__`, `-` `___`):
|
|
720
|
+
* `BUNDLE_GEMS__CONTRIBSYS__COM`, `BUNDLE_RUBYGEMS__PKG__GITHUB__COM`. Its value is `user:password` or a token. Not
|
|
721
|
+
* Bundler's own settings that take a gem's or a URL's name the same way (`BUNDLE_BUILD__NOKOGIRI`, `BUNDLE_LOCAL__RACK`,
|
|
722
|
+
* `BUNDLE_MIRROR__…`, `BUNDLE_GEM__TEST`).
|
|
723
|
+
*/
|
|
724
|
+
function isBundlerHostName(name) {
|
|
725
|
+
return /^BUNDLE_(?!(?:BUILD|LOCAL|MIRROR|GEM)__)[A-Z0-9]+(?:_{2,3}[A-Z0-9]+)*__[A-Z]{2,6}$/.test(name);
|
|
726
|
+
}
|
|
659
727
|
/**
|
|
660
728
|
* Data files' extensions: as a key, a data file's name maps to its checksum (`"data.csv": "<sha256>"` in a lockfile or a
|
|
661
729
|
* registry). A config file's name maps to its contents (a Kubernetes Secret's `config.yaml`, `default_user.conf`).
|
|
@@ -1185,8 +1253,8 @@ function classify(name, w, core, last) {
|
|
|
1185
1253
|
// (A bare file name is read below, after the PIN's: it may be a Kubernetes Secret's key, `pin.txt`.)
|
|
1186
1254
|
if (name.includes("/") && isFileName(name))
|
|
1187
1255
|
return "neutral";
|
|
1188
|
-
// A vendor's name holds that vendor's key.
|
|
1189
|
-
if (isVendorName(name))
|
|
1256
|
+
// A vendor's name holds that vendor's key; Bundler's name for a gem server's host, that server's credentials.
|
|
1257
|
+
if (isVendorName(name) || isBundlerHostName(name))
|
|
1190
1258
|
return "credential";
|
|
1191
1259
|
// Unity's serialized localization keys and names; a list item's key (Sanity's and React's `_key`); a generated
|
|
1192
1260
|
// fixture's, numbered with zeros (`tagKey000003`); Postman's per-request UUID.
|
package/dist/repeats.js
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
import { KeywordSet } from "./aho.js";
|
|
3
3
|
import { continuesToken } from "./syntax.js";
|
|
4
4
|
import { isBenignValue } from "./values.js";
|
|
5
|
-
import { covered, secretValue } from "./spans.js";
|
|
5
|
+
import { covered, loginPassword, secretValue } from "./spans.js";
|
|
6
6
|
import { isGuess, layerOf, PLACEHOLDER_RULE, REPEATED_GUESS } from "./rules.js";
|
|
7
7
|
/**
|
|
8
8
|
* Other occurrences of confidently found values: a webhook token found in its URL is also hidden in
|
|
@@ -30,6 +30,10 @@ export function repeatable(text, found) {
|
|
|
30
30
|
continue;
|
|
31
31
|
const remembered = { kind: f.kind, value: v, ...(isGuess(f.rule) ? { guess: true } : {}) };
|
|
32
32
|
values.set(v, remembered);
|
|
33
|
+
// (And a login's password, `NEO4J_AUTH=neo4j/<password>`, where it turns up alone.)
|
|
34
|
+
const password = loginPassword(f, v);
|
|
35
|
+
if (password && !isBenignValue(password) && !values.has(password))
|
|
36
|
+
values.set(password, { ...remembered, value: password });
|
|
33
37
|
// (As it is written here too, when that's another form of it, and as a URL writes it: `Delta2018%26` in a URL,
|
|
34
38
|
// `Delta2018&` in prose, and the other way round. Only for 8 or more printable characters and no blank: half a
|
|
35
39
|
// surrogate pair has no URL form.)
|
package/dist/secretfile.js
CHANGED
|
@@ -8,7 +8,7 @@ import { commandsRun, holdsOnlySecrets, isConfigFileName, isSecretFile, printsSe
|
|
|
8
8
|
import { allRedirects, LineIndex, programName, PROMPT, simpleCommand, wordSpan } from "./shell.js";
|
|
9
9
|
import { naturalness } from "./charlm.js";
|
|
10
10
|
import { isPlaceholder } from "./public.js";
|
|
11
|
-
import { hasCredentialWord, isCredentialName, isSettingName, isVendorName, namesPasswordOrPin, words, } from "./names.js";
|
|
11
|
+
import { hasCredentialWord, isCredentialName, isSettingName, isVendorName, namesPasswordOrPin, namesSession, words, } from "./names.js";
|
|
12
12
|
import { lowerBound } from "./lines.js";
|
|
13
13
|
import { COMMON_WORDS, hasThreeClasses, isBenignValue, isCode, isGeneratedSegment, isIdentifierShaped, isInert, isMadePassword, isPath, isPodName, isRandomToken, isWordy, readsAsWords, } from "./values.js";
|
|
14
14
|
import { isSentenceWord } from "./named.js";
|
|
@@ -20,28 +20,33 @@ import { isSessionCookie } from "./structure.js";
|
|
|
20
20
|
/**
|
|
21
21
|
* Under a name the name rules didn't take, in a secret file: a value no setting has, under a vendor's bare word
|
|
22
22
|
* (`STRIPE=<uuid>`, `OPENAI=Tiger1103#x`) or a name with a credential word that the in-text classifier reads otherwise
|
|
23
|
-
* (`BASIC_AUTH=Harbor2291!`, `MY_SECRET_THING=…`)
|
|
24
|
-
* has (`APP_URL`, `SSH_AUTH_SOCK`), nor a value that reads
|
|
25
|
-
* (`DEBUG=true`, `REGION=us-east-1`). These findings
|
|
26
|
-
* keys and other public values.
|
|
23
|
+
* (`BASIC_AUTH=Harbor2291!`, `MY_SECRET_THING=…`) or a session's (`INSTAGRAM_SESSION_ID`: names.ts namesSession): the
|
|
24
|
+
* file says its values are secrets. Not under a name a setting has (`APP_URL`, `SSH_AUTH_SOCK`), nor a value that reads
|
|
25
|
+
* as a setting: words, numbers, versions, paths, hosts and URLs (`DEBUG=true`, `REGION=us-east-1`). These findings
|
|
26
|
+
* still pass the decision stage (policy.ts), which keeps publishable keys and other public values.
|
|
27
27
|
*/
|
|
28
28
|
function isOpaqueSetting(name, v, tokenUnderAnyName) {
|
|
29
|
-
if (
|
|
29
|
+
if (!mayHoldOpaque(name, tokenUnderAnyName) || isPodName(v))
|
|
30
30
|
return false;
|
|
31
31
|
// (Under a name that says nothing, `admin:` in `config/secrets.staging.yml`, only a generated token or a made password
|
|
32
32
|
// is one, and only in a file that holds nothing else: `.env` has settings and public IDs (`GPG_KEY=<fingerprint>`).
|
|
33
33
|
// Not one a word and a hyphen lead, `gradle-<hash>`: a cache's key.)
|
|
34
|
-
if (!(isBareVendorWord(name) || hasCredentialWord(name)))
|
|
35
|
-
return (
|
|
36
|
-
(isRandomToken(v, MIN_UNNAMED_TOKEN) || isMadePassword(v)) &&
|
|
37
|
-
!isSettingShaped(v) &&
|
|
38
|
-
!/^[a-z]{2,}-/i.test(v));
|
|
34
|
+
if (!(isBareVendorWord(name) || hasCredentialWord(name) || namesSession(name)))
|
|
35
|
+
return (isRandomToken(v, MIN_UNNAMED_TOKEN) || isMadePassword(v)) && !isSettingShaped(v) && !/^[a-z]{2,}-/i.test(v);
|
|
39
36
|
if (/^[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}$/i.test(v))
|
|
40
37
|
return true;
|
|
41
38
|
// (Letters alone are a word, `livemode`, unless random, `XGxAdcVIefCUMAVsDhkYhErsQwPz`. Digits with symbols are a
|
|
42
39
|
// password, `90210!@#`: a number a setting holds is setting-shaped.)
|
|
43
40
|
return !isSettingShaped(v) && (/[\d\W]/.test(v.replace(/[-_.]/g, "")) || isRandomToken(v));
|
|
44
41
|
}
|
|
42
|
+
/**
|
|
43
|
+
* Whether isOpaqueSetting can take a value under `name`, by the name alone: a session's, whose `ID` is no setting's;
|
|
44
|
+
* or not a setting's, and either a vendor's or credential's word, or any name in a file that holds only secrets.
|
|
45
|
+
*/
|
|
46
|
+
function mayHoldOpaque(name, tokenUnderAnyName) {
|
|
47
|
+
return (namesSession(name) ||
|
|
48
|
+
(!isSettingName(name) && (tokenUnderAnyName || isBareVendorWord(name) || hasCredentialWord(name))));
|
|
49
|
+
}
|
|
45
50
|
/** The shortest generated token taken as a secret under a name that says nothing of one: a 128-bit hex ID is 32; a 96-bit one, 24. */
|
|
46
51
|
const MIN_UNNAMED_TOKEN = 24;
|
|
47
52
|
/** One upper-case word: `STRIPE`, `OPENAI`. */
|
|
@@ -207,6 +212,12 @@ function assignments(text, out, tokenUnderAnyName) {
|
|
|
207
212
|
const comment = quoted ? -1 : value.search(/[ \t]#/);
|
|
208
213
|
if (comment >= 0)
|
|
209
214
|
value = value.slice(0, comment).trimEnd();
|
|
215
|
+
// (A vendor's name alone holds the vendor's key, or where the vendor is: `NGROK=abc123def456.ngrok.io` is a host.)
|
|
216
|
+
const credential = (isCredentialName(name) && !(isVendorName(name) && HOST.test(value))) || /^pass$/i.test(name);
|
|
217
|
+
// (The name first: most of a large file's lines, a state file's `"name": "svc-0"`, are settings by it alone, and the
|
|
218
|
+
// value's checks cost more.)
|
|
219
|
+
if (!credential && !mayHoldOpaque(name, tokenUnderAnyName))
|
|
220
|
+
continue;
|
|
210
221
|
// (A reference in code, `password: process.env.DB_PASSWORD,` in a line `grep` found in source.)
|
|
211
222
|
if (!value ||
|
|
212
223
|
isNotSecret(value) ||
|
|
@@ -215,8 +226,6 @@ function assignments(text, out, tokenUnderAnyName) {
|
|
|
215
226
|
/^[|>][-+]?$|^[{[]$|^(true|false|null)$/i.test(value) ||
|
|
216
227
|
isSettingNumber(name, value))
|
|
217
228
|
continue;
|
|
218
|
-
// (A vendor's name alone holds the vendor's key, or where the vendor is: `NGROK=abc123def456.ngrok.io` is a host.)
|
|
219
|
-
const credential = (isCredentialName(name) && !(isVendorName(name) && HOST.test(value))) || /^pass$/i.test(name);
|
|
220
229
|
if (!credential && !isOpaqueSetting(name, value, tokenUnderAnyName))
|
|
221
230
|
continue;
|
|
222
231
|
const start = at(m, 4)[0] + (quoted ? 1 : 0);
|
|
@@ -296,6 +305,41 @@ function htpasswd(text, out) {
|
|
|
296
305
|
for (const m of text.matchAll(/^[\w.@-]{1,100}:(\$[\w.+=-]{1,30}\$[\w$./+=-]{8,300}|\{SSHA\}[\w+/=]{10,200})[ \t]*$/dgm))
|
|
297
306
|
out.push({ start: at(m, 1)[0], end: at(m, 1)[1], kind: "password hash", rule: "file:hash", confidence: 1 });
|
|
298
307
|
}
|
|
308
|
+
/**
|
|
309
|
+
* Redis's ACL rules, `user worker on >password ~jobs:* +@all` (redis.conf, an aclfile): a password given or taken away,
|
|
310
|
+
* `>pw` and `<pw`, and one given or taken away as its SHA-256, `#<hash>` and `!<hash>`.
|
|
311
|
+
*/
|
|
312
|
+
function redisUsers(text, out) {
|
|
313
|
+
for (const line of text.matchAll(/^[ \t]*user[ \t]+\S+[ \t]+[^\r\n]+/gm))
|
|
314
|
+
for (const m of line[0].matchAll(/[ \t](?:[<>](\S+)|[#!]([0-9a-f]{64})(?![^ \t]))/dg)) {
|
|
315
|
+
const g = m[1] !== undefined ? 1 : 2;
|
|
316
|
+
const [start, end] = at(m, g);
|
|
317
|
+
out.push({
|
|
318
|
+
start: line.index + start,
|
|
319
|
+
end: line.index + end,
|
|
320
|
+
kind: g === 1 ? "password" : "password hash",
|
|
321
|
+
rule: "file:redis-acl",
|
|
322
|
+
confidence: 1,
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
/**
|
|
327
|
+
* A string literal of escaped bytes: key material as code writes binary, Symfony's vault key (`prod.decrypt.private.php`
|
|
328
|
+
* holds `return "\x8E\x1Fq\xC4…";`), a Python or C byte string. Read from 16 escaped bytes, half the string or more: fewer
|
|
329
|
+
* are a setting's control characters (`PS1="\x1b[32m$ "`).
|
|
330
|
+
*/
|
|
331
|
+
function escapedBytes(text, out) {
|
|
332
|
+
if (!/\\(?:x[0-9A-Fa-f]{2}|[0-7]{3})/.test(text))
|
|
333
|
+
return;
|
|
334
|
+
for (const m of text.matchAll(/"((?:\\.|[^"\\\r\n]){16,8192})"|'((?:\\.|[^'\\\r\n]){16,8192})'/dg)) {
|
|
335
|
+
const g = m[1] !== undefined ? 1 : 2;
|
|
336
|
+
const escaped = m[g].match(/\\(?:x[0-9A-Fa-f]{2}|[0-7]{3})/g)?.length ?? 0;
|
|
337
|
+
const bytes = m[g].replace(/\\(?:x[0-9A-Fa-f]{2}|[0-7]{3}|.)/g, ".").length;
|
|
338
|
+
if (escaped < 16 || escaped * 2 < bytes)
|
|
339
|
+
continue;
|
|
340
|
+
out.push({ start: at(m, g)[0], end: at(m, g)[1], kind: "secret", rule: "file:bytes", confidence: 0.95 });
|
|
341
|
+
}
|
|
342
|
+
}
|
|
299
343
|
/**
|
|
300
344
|
* Lines that are one token (a bare token file, `cut -d= -f2` output), after any line number a file viewer puts in
|
|
301
345
|
* front (`cat -n`, an agent's Read tool ` 3→`). Not a NAME, an assignment, a comment, a section or a rule, nor
|
|
@@ -337,6 +381,8 @@ const SECRET_FILE_READERS = [
|
|
|
337
381
|
userlist,
|
|
338
382
|
cookiesTxt,
|
|
339
383
|
htpasswd,
|
|
384
|
+
redisUsers,
|
|
385
|
+
escapedBytes,
|
|
340
386
|
bareTokens,
|
|
341
387
|
];
|
|
342
388
|
/**
|
package/dist/spans.d.ts
CHANGED
|
@@ -6,6 +6,14 @@ import type { Finding } from "./types.ts";
|
|
|
6
6
|
* (Hex stays learnable: real keys are hex, and a commit SHA taught as one is over-redaction only.)
|
|
7
7
|
*/
|
|
8
8
|
export declare function isNeverSecretShape(value: string): boolean;
|
|
9
|
+
/**
|
|
10
|
+
* The password of a finding that is a login: a username and a password in one value, `user:password` or
|
|
11
|
+
* `user/password`, under a name that says it holds one (names.ts namesLogin: `NEO4J_AUTH=neo4j/…`, Bundler's
|
|
12
|
+
* `BUNDLE_<HOST>: "user:…"`). Learned and repeated on its own too, as a client takes it (`cypher-shell -u neo4j -p
|
|
13
|
+
* <it>`). Not a `:` or `/` inside another secret (a signature's `abc:<id>`), nor a URL's (its password is read in the
|
|
14
|
+
* URL); before a `/`, only a user's lower-case name (`neo4j/…`, not a base64 key's `wJalrXUtnFEMI/K7MDENG…`).
|
|
15
|
+
*/
|
|
16
|
+
export declare function loginPassword(f: Finding, value: string): string | undefined;
|
|
9
17
|
/**
|
|
10
18
|
* Whether a finding is sure enough, and long enough, to be learned as a known value and hidden wherever it appears: one
|
|
11
19
|
* a rule read by a name, a format, a structure or a file, not the unnamed layer's guess about one place; and whose
|
package/dist/spans.js
CHANGED
|
@@ -7,6 +7,8 @@ import { naturalness } from "./charlm.js";
|
|
|
7
7
|
import { backslashesBefore, escapeLength, UNQUOTED_READ } from "./syntax.js";
|
|
8
8
|
import { leadingUrl } from "./urls.js";
|
|
9
9
|
import { isPath } from "./values.js";
|
|
10
|
+
import { isReplaceable } from "./known.js";
|
|
11
|
+
import { namesLogin } from "./names.js";
|
|
10
12
|
import { continuesAcrossWraps, isGuess, isWholeLine, layerOf, PLACEHOLDER_RULE } from "./rules.js";
|
|
11
13
|
/** An Amazon Resource Name: `arn:aws:iam::483920176654:role/deploy`, in any partition. */
|
|
12
14
|
const ARN = /^arn:[a-z][a-z0-9-]*:[a-z0-9-]+:[a-z0-9-]*:\d*:[\w:/.@+=,*-]+$/;
|
|
@@ -28,6 +30,19 @@ export function isNeverSecretShape(value) {
|
|
|
28
30
|
return false;
|
|
29
31
|
return (!/^https?$/i.test(url.scheme) || (!/[?#]/.test(value) && url.segments.every((s) => s.end - s.start < TOKEN_SEGMENT)));
|
|
30
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* The password of a finding that is a login: a username and a password in one value, `user:password` or
|
|
35
|
+
* `user/password`, under a name that says it holds one (names.ts namesLogin: `NEO4J_AUTH=neo4j/…`, Bundler's
|
|
36
|
+
* `BUNDLE_<HOST>: "user:…"`). Learned and repeated on its own too, as a client takes it (`cypher-shell -u neo4j -p
|
|
37
|
+
* <it>`). Not a `:` or `/` inside another secret (a signature's `abc:<id>`), nor a URL's (its password is read in the
|
|
38
|
+
* URL); before a `/`, only a user's lower-case name (`neo4j/…`, not a base64 key's `wJalrXUtnFEMI/K7MDENG…`).
|
|
39
|
+
*/
|
|
40
|
+
export function loginPassword(f, value) {
|
|
41
|
+
if (!f.name || !namesLogin(f.name) || value.includes("://"))
|
|
42
|
+
return undefined;
|
|
43
|
+
const pair = /^(?:[\w.@+-]{1,64}:|[a-z][a-z0-9._-]{0,31}\/)([^\s/:]\S*)$/.exec(value);
|
|
44
|
+
return pair && isReplaceable(pair[1]) ? pair[1] : undefined;
|
|
45
|
+
}
|
|
31
46
|
/**
|
|
32
47
|
* Whether a finding is sure enough, and long enough, to be learned as a known value and hidden wherever it appears: one
|
|
33
48
|
* a rule read by a name, a format, a structure or a file, not the unnamed layer's guess about one place; and whose
|
package/dist/structure.js
CHANGED
|
@@ -689,8 +689,9 @@ function commandPasswords(words, add) {
|
|
|
689
689
|
function toolPasswords(words, add) {
|
|
690
690
|
// The command starts at its first word that names a tool or a wrapper: past a prompt, a timestamp or a CI log's
|
|
691
691
|
// prefix (`$ `, `2025-…Z ##[group]Run`, `xcrun`); a path names one only as the first word (`find /var/lib/mysql`
|
|
692
|
-
// runs find)
|
|
693
|
-
|
|
692
|
+
// runs find), or after a process listing's columns (` 73104 /usr/bin/redis-cli -a …`). Then past wrappers and
|
|
693
|
+
// assignments (shell.ts simpleCommand).
|
|
694
|
+
const names = (w, i) => (i === 0 || !/[/\\]/.test(w.text) || isProcessColumns(words.slice(0, i))) &&
|
|
694
695
|
(isWrapper(w.text) ||
|
|
695
696
|
/^(?:rabbitmqctl|htpasswd|https?|httpie|steamcmd|vault|lftp|pulumi|openssl|mkpasswd)$/.test(programName(w)) ||
|
|
696
697
|
USER_PASSWORD.test(programName(w)) ||
|
|
@@ -838,6 +839,16 @@ function toolPasswords(words, add) {
|
|
|
838
839
|
addWord(rest[i], w.slice(w.indexOf("%") + 1));
|
|
839
840
|
}
|
|
840
841
|
}
|
|
842
|
+
/**
|
|
843
|
+
* A process listing's columns before its command (`ps -eo pid,args`, `ps aux`, `pgrep -a`): a PID, and maybe a user
|
|
844
|
+
* first, then CPU and memory figures, sizes, a terminal, a state, a start time and a CPU time (`root 73104 0.0 0.1 …
|
|
845
|
+
* pts/0 Ss+ Oct10 0:00`).
|
|
846
|
+
*/
|
|
847
|
+
function isProcessColumns(words) {
|
|
848
|
+
return (words.some((w) => /^\d+$/.test(w.text)) &&
|
|
849
|
+
words.every((w, j) => /^(?:\d[\d.:,]*[KMG]?|\?|(?:pts|tty)\/?\w*|[RSDTtZIWXP][<NLsl+]*|[A-Z][a-z]{2}\d{1,2})$/.test(w.text) ||
|
|
850
|
+
(j === 0 && /^[a-z_][\w.-]*\+?$/.test(w.text))));
|
|
851
|
+
}
|
|
841
852
|
/** Adds `value`, the end of a shell word's text, as a command-line password, at the span the word wrote it in. */
|
|
842
853
|
function addTail(add, word, value, quoted, rule = "structure:cli-password") {
|
|
843
854
|
const { start, end } = wordSpan(word, word.text.length - value.length);
|
package/dist/transcript.js
CHANGED
|
@@ -20,7 +20,7 @@ import { base64Binary } from "./base64.js";
|
|
|
20
20
|
import { findFormats } from "./formats.js";
|
|
21
21
|
import { parseJson, TOOL_COMMAND_KEYS, TOOL_CONTENT_KEYS, TOOL_PATH_KEYS } from "./json.js";
|
|
22
22
|
import { shellLine } from "./shell.js";
|
|
23
|
-
import { isLearnable, isNeverSecretShape, merge, secretValue } from "./spans.js";
|
|
23
|
+
import { isLearnable, isNeverSecretShape, loginPassword, merge, secretValue } from "./spans.js";
|
|
24
24
|
import { occurrences } from "./repeats.js";
|
|
25
25
|
import { isCredentialName } from "./names.js";
|
|
26
26
|
import { isNamedSecret } from "./named.js";
|
|
@@ -522,8 +522,12 @@ export function redactValue(redactor, value, options = {}) {
|
|
|
522
522
|
learned.push([leaf.text, leaf.key]);
|
|
523
523
|
for (const f of found(leaf).filter(keep)) {
|
|
524
524
|
const secret = secretValue(leaf.text, f);
|
|
525
|
-
if (isLearnable(f, secret))
|
|
526
|
-
|
|
525
|
+
if (!isLearnable(f, secret))
|
|
526
|
+
continue;
|
|
527
|
+
learned.push([secret, f.kind]);
|
|
528
|
+
const password = loginPassword(f, secret);
|
|
529
|
+
if (password)
|
|
530
|
+
learned.push([password, f.kind]);
|
|
527
531
|
}
|
|
528
532
|
}
|
|
529
533
|
const remember = options.learn !== false;
|
package/docs/creddata.svg
CHANGED
|
@@ -24,7 +24,7 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
24
24
|
<text x="16" y="76">and Nosey Parker, then detect-secrets (the other six). Hollow: CredSweeper, which CredData's lines partly come from and whose model is trained on its labels. √ scale across; up and left is better.</text>
|
|
25
25
|
<text x="16" y="116" class="title">Dev split (the fixes were made on it)</text>
|
|
26
26
|
<text x="16" y="606" class="title">Test split (never seen by the fixes)</text>
|
|
27
|
-
<text x="56" y="148" class="panel">credential-shaped <tspan class="tick">4,560 true, 10,
|
|
27
|
+
<text x="56" y="148" class="panel">credential-shaped <tspan class="tick">4,560 true, 10,669 false</tspan></text>
|
|
28
28
|
<line class="grid" x1="56" x2="436" y1="490" y2="490"/><text class="tick" x="50" y="494" text-anchor="end">0%</text>
|
|
29
29
|
<line class="grid" x1="56" x2="436" y1="407.5" y2="407.5"/><text class="tick" x="50" y="411.5" text-anchor="end">25%</text>
|
|
30
30
|
<line class="grid" x1="56" x2="436" y1="325" y2="325"/><text class="tick" x="50" y="329" text-anchor="end">50%</text>
|
|
@@ -37,7 +37,7 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
37
37
|
<line class="grid" x1="137.3817706875497" x2="137.3817706875497" y1="160" y2="490"/><text class="tick" x="137.3817706875497" y="504" text-anchor="middle">1%</text>
|
|
38
38
|
<line class="grid" x1="56" x2="56" y1="160" y2="490"/><text class="tick" x="56" y="504" text-anchor="middle">0%</text>
|
|
39
39
|
<polyline fill="none" class="chain" stroke-width="2" points="296.23485612070624,307.3421052631579 337.1070378925266,262.0394736842105 347.2020908546517,250.38815789473685 389.28204733667116,227.15789473684208"/>
|
|
40
|
-
<polyline fill="none" class="accent-line" stroke-width="2" points="192.
|
|
40
|
+
<polyline fill="none" class="accent-line" stroke-width="2" points="192.23881939564785,269.56578947368416 194.2740347893205,176.28289473684208 195.83650285579614,174.03947368421052"/>
|
|
41
41
|
<circle cx="296.23485612070624" cy="307.3421052631579" r="4" class="other ring" stroke-width="1.5"><title>gitleaks: 55.4% caught, 8.7% lost</title></circle>
|
|
42
42
|
<circle cx="317.88625179591025" cy="265.73026315789474" r="4" class="other ring" stroke-width="1.5"><title>Betterleaks: 68.0% caught, 10.4% lost</title></circle>
|
|
43
43
|
<circle cx="118.0617486347827" cy="476.5394736842105" r="4" class="other ring" stroke-width="1.5"><title>Kingfisher: 4.1% caught, 0.6% lost</title></circle>
|
|
@@ -48,10 +48,10 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
48
48
|
<circle cx="337.1070378925266" cy="262.0394736842105" r="4" class="other ring" stroke-width="1.5"><title>gitleaks + Betterleaks + Kingfisher: 69.1% caught, 11.9% lost</title></circle>
|
|
49
49
|
<circle cx="347.2020908546517" cy="250.38815789473685" r="4" class="other ring" stroke-width="1.5"><title>the other five but detect-secrets: 72.6% caught, 12.8% lost</title></circle>
|
|
50
50
|
<circle cx="389.28204733667116" cy="227.15789473684208" r="4" class="other ring" stroke-width="1.5"><title>the other six: 79.6% caught, 16.8% lost</title></circle>
|
|
51
|
-
<circle cx="192.
|
|
52
|
-
<circle cx="194.
|
|
53
|
-
<circle cx="195.
|
|
54
|
-
<text x="202.
|
|
51
|
+
<circle cx="192.23881939564785" cy="269.56578947368416" r="5" class="accent ring" stroke-width="1.5"><title>guessing off: 66.8% caught, 2.8% lost</title></circle>
|
|
52
|
+
<circle cx="194.2740347893205" cy="176.28289473684208" r="5" class="accent ring" stroke-width="1.5"><title>guessing confident: 95.1% caught, 2.9% lost</title></circle>
|
|
53
|
+
<circle cx="195.83650285579614" cy="174.03947368421052" r="5" class="accent ring" stroke-width="1.5"><title>guessing eager: 95.7% caught, 3.0% lost</title></circle>
|
|
54
|
+
<text x="202.2740347893205" y="180.28289473684208" text-anchor="start" class="ours" font-size="11">redac---n</text>
|
|
55
55
|
<line class="lead" x1="127.37318697497005" y1="172.39473684210526" x2="135.37318697497005" y2="187.39473684210526"/>
|
|
56
56
|
<text x="135.37318697497005" y="197.39473684210526" text-anchor="start" font-size="11">CredSweeper (tuned on CredData)</text>
|
|
57
57
|
<text x="381.28204733667116" y="219.15789473684208" text-anchor="end" font-size="11">the other six</text>
|
|
@@ -135,7 +135,7 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
135
135
|
<text x="928" y="524" class="tick">Catch under 1%, not drawn (caught / lost):</text>
|
|
136
136
|
<text x="928" y="538" class="tick">gitleaks 0.9% / 0.00%, Kingfisher 0.9% / 0.01%</text>
|
|
137
137
|
<text x="928" y="552" class="tick">TruffleHog 0.1% / 0.16%</text>
|
|
138
|
-
<text x="1364" y="148" class="panel">other <tspan class="tick">859 true, 12,
|
|
138
|
+
<text x="1364" y="148" class="panel">other <tspan class="tick">859 true, 12,420 false</tspan></text>
|
|
139
139
|
<line class="grid" x1="1364" x2="1744" y1="490" y2="490"/><text class="tick" x="1358" y="494" text-anchor="end">0%</text>
|
|
140
140
|
<line class="grid" x1="1364" x2="1744" y1="407.5" y2="407.5"/><text class="tick" x="1358" y="411.5" text-anchor="end">25%</text>
|
|
141
141
|
<line class="grid" x1="1364" x2="1744" y1="325" y2="325"/><text class="tick" x="1358" y="329" text-anchor="end">50%</text>
|
|
@@ -148,7 +148,7 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
148
148
|
<line class="grid" x1="1433.1375440697743" x2="1433.1375440697743" y1="160" y2="490"/><text class="tick" x="1433.1375440697743" y="504" text-anchor="middle">0.5%</text>
|
|
149
149
|
<line class="grid" x1="1364" x2="1364" y1="160" y2="490"/><text class="tick" x="1364" y="504" text-anchor="middle">0%</text>
|
|
150
150
|
<polyline fill="none" class="chain" stroke-width="2" points="1440.9615288509585,460.034924330617 1570.4349403937827,453.8882421420256 1579.0134163971384,448.8940628637951 1697.282047336671,413.9348079161816"/>
|
|
151
|
-
<polyline fill="none" class="accent-line" stroke-width="2" points="1434.
|
|
151
|
+
<polyline fill="none" class="accent-line" stroke-width="2" points="1434.7334476179146,291.7694994179278 1443.929518066188,282.1653084982538 1443.929518066188,267.5669383003492"/>
|
|
152
152
|
<circle cx="1440.9615288509585" cy="460.034924330617" r="4" class="other ring" stroke-width="1.5"><title>gitleaks: 9.1% caught, 0.6% lost</title></circle>
|
|
153
153
|
<circle cx="1566.294530136326" cy="454.2724097788126" r="4" class="other ring" stroke-width="1.5"><title>Betterleaks: 10.8% caught, 4.3% lost</title></circle>
|
|
154
154
|
<circle cx="1462.8394191060975" cy="481.9324796274738" r="4" class="other ring" stroke-width="1.5"><title>Nosey Parker: 2.4% caught, 1.0% lost</title></circle>
|
|
@@ -157,10 +157,10 @@ text { font-family: system-ui, -apple-system, "Segoe UI", sans-serif; fill: #525
|
|
|
157
157
|
<circle cx="1570.4349403937827" cy="453.8882421420256" r="4" class="other ring" stroke-width="1.5"><title>gitleaks + Betterleaks + Kingfisher: 10.9% caught, 4.5% lost</title></circle>
|
|
158
158
|
<circle cx="1579.0134163971384" cy="448.8940628637951" r="4" class="other ring" stroke-width="1.5"><title>the other five but detect-secrets: 12.5% caught, 4.8% lost</title></circle>
|
|
159
159
|
<circle cx="1697.282047336671" cy="413.9348079161816" r="4" class="other ring" stroke-width="1.5"><title>the other six: 23.1% caught, 11.6% lost</title></circle>
|
|
160
|
-
<circle cx="1434.
|
|
161
|
-
<circle cx="1443.
|
|
162
|
-
<circle cx="1443.
|
|
163
|
-
<text x="1451.
|
|
160
|
+
<circle cx="1434.7334476179146" cy="291.7694994179278" r="5" class="accent ring" stroke-width="1.5"><title>guessing off: 60.1% caught, 0.5% lost</title></circle>
|
|
161
|
+
<circle cx="1443.929518066188" cy="282.1653084982538" r="5" class="accent ring" stroke-width="1.5"><title>guessing confident: 63.0% caught, 0.7% lost</title></circle>
|
|
162
|
+
<circle cx="1443.929518066188" cy="267.5669383003492" r="5" class="accent ring" stroke-width="1.5"><title>guessing eager: 67.4% caught, 0.7% lost</title></circle>
|
|
163
|
+
<text x="1451.929518066188" y="286.1653084982538" text-anchor="start" class="ours" font-size="11">redac---n</text>
|
|
164
164
|
<text x="1425.3493565673837" y="194.73341094295694" text-anchor="start" font-size="11">CredSweeper (tuned on CredData)</text>
|
|
165
165
|
<text x="1689.282047336671" y="405.9348079161816" text-anchor="end" font-size="11">the other six</text>
|
|
166
166
|
<line class="lead" x1="1663.7434800728013" y1="430.22817229336437" x2="1671.7434800728013" y2="433.22817229336437"/>
|