redac---n 0.0.4 → 0.0.5

package/dist/environment.d.ts

@@ -1,8 +1,10 @@
 /**
  * The secrets in an environment, as known values: variables whose name has a credential word (`*_API_KEY`, `*_TOKEN`,
- * `BW_SESSION`, read widely), a vendor's token under any name, and URL passwords inside any variable. Only values a
- * Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value is left out.
- * `new Redactor({ known: fromEnvironment(process.env) })`.
+ * `BW_SESSION`, read widely), a vendor's token under any name, and the passwords and keys inside any variable that
+ * the detectors read there (a URL's, a connection string's: `DATABASE_URL`, `password=` in a DSN). Not what the
+ * decision stage keeps on purpose: a key public by design, or under a browser's prefix (`NEXT_PUBLIC_STRIPE_KEY=pk_live_…`).
+ * Only values a Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value
+ * is left out. `new Redactor({ known: fromEnvironment(process.env) })`.
  */
 export declare function fromEnvironment(env: Record<string, string | undefined>): {
     value: string;

package/dist/environment.js

@@ -1,21 +1,38 @@
 import { findFormats } from "./formats.js";
 import { isReplaceable } from "./known.js";
 import { hasCredentialWord, isCredentialName, isSettingName, namesPasswordOrPin } from "./names.js";
-import { isLiteralPassword } from "./structure.js";
-import { urls } from "./urls.js";
+import { findNamed } from "./named.js";
+import { keepReason } from "./policy.js";
+import { findStructure, isLiteralPassword } from "./structure.js";
 import { naturalness } from "./charlm.js";
 import { hasThreeClasses, isInert, isRandomToken } from "./values.js";
 /**
  * The secrets in an environment, as known values: variables whose name has a credential word (`*_API_KEY`, `*_TOKEN`,
- * `BW_SESSION`, read widely), a vendor's token under any name, and URL passwords inside any variable. Only values a
- * Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value is left out.
- * `new Redactor({ known: fromEnvironment(process.env) })`.
+ * `BW_SESSION`, read widely), a vendor's token under any name, and the passwords and keys inside any variable that
+ * the detectors read there (a URL's, a connection string's: `DATABASE_URL`, `password=` in a DSN). Not what the
+ * decision stage keeps on purpose: a key public by design, or under a browser's prefix (`NEXT_PUBLIC_STRIPE_KEY=pk_live_…`).
+ * Only values a Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value
+ * is left out. `new Redactor({ known: fromEnvironment(process.env) })`.
  */
 export function fromEnvironment(env) {
     const out = [];
+    const add = (value, label) => {
+        if (!out.some((o) => o.value === value))
+            out.push({ value, label });
+    };
     for (const [name, value] of Object.entries(env)) {
         if (!value || value.length < 6 || value.length > 16_384)
             continue;
+        // (Judged as the decision stage judges the same value on a `NAME=value` line, with the rule that read it.)
+        const line = `${name}=${value}`;
+        const kept = (f) => keepReason(line, {
+            ...f,
+            start: f.start + name.length + 1,
+            end: f.end + name.length + 1,
+            kind: "secret",
+            name,
+            confidence: 1,
+        }) !== undefined;
         // (Read widely: the caller's own environment, where a credential missed goes to a provider. A credential word anywhere
         // in the name or a session's at its end, `BW_SESSION`, `TF_TOKEN_app_terraform_io`, `ACTIONS_ID_TOKEN_REQUEST_TOKEN`,
         // unless the name ends as a setting's or a generator didn't make the value: `TOKEN_ENDPOINT`, `DESKTOP_SESSION=ubuntu-wayland`.
@@ -24,17 +41,24 @@ export function fromEnvironment(env) {
             ((namesPasswordOrPin(name) && !isInert(value)) ||
                 ((hasCredentialWord(name) || /(?:^|_)(?:SESSION|PEPPER)$/i.test(name)) && isGenerated(value)));
         const secretName = isCredentialName(name) || /WEBHOOK/i.test(name) || widened;
-        if (secretName && !isInert(value))
-            out.push({ value, label: `$${name}` });
-        // (A vendor's token under any name: `CI_DEPLOY=ghp_…`, a Slack webhook URL.)
+        if (secretName && !isInert(value) && !kept({ start: 0, end: value.length, rule: "named:assign" }))
+            add(value, `$${name}`);
+        // (A vendor's token under any name, `CI_DEPLOY=ghp_…`, a Slack webhook URL; one public by design is kept, a
+        // publishable key or a browser's Google key.)
         else
             for (const f of findFormats(value))
-                if (!out.some((o) => o.value === value.slice(f.start, f.end)))
-                    out.push({ value: value.slice(f.start, f.end), label: `${f.kind} in $${name}` });
-        // (A literal password, as the URL readers take one: not `${DB_PASSWORD}`, `<password>`, `%PASSWORD%`.)
-        for (const url of value.includes("://") ? urls(value) : [])
-            if (url.password && isLiteralPassword(url.password.value))
-                out.push({ value: decodeSafe(url.password.value), label: `password in $${name}` });
+                if (!kept(f))
+                    add(value.slice(f.start, f.end), `${f.kind} in $${name}`);
+        // (What the structure and name readers find inside the value: a URL's password, a connection string's, a DSN's
+        // `password=`. A literal one, as the URL readers take it: not `${DB_PASSWORD}`, `<password>`, `%PASSWORD%`.)
+        if (!/[=:]/.test(value))
+            continue;
+        for (const f of [...findStructure(value), ...findNamed(value)]) {
+            const secret = value.slice(f.start, f.end);
+            if (f.confidence < 0.9 || !isLiteralPassword(secret) || kept(f))
+                continue;
+            add(/^structure:url/.test(f.rule) ? decodeSafe(secret) : secret, `${f.kind} in $${name}`);
+        }
     }
     // (Only what a Redactor takes as a known value, so the caller isn't told a `JWT_SECRET=devsecret` is protected when it isn't.)
     return out.filter((o) => isReplaceable(o.value.trim()));

package/dist/formats.js

@@ -252,6 +252,18 @@ export const CORE_FORMATS = [
         ["aeaaqwvhqxbps2v5"],
     ],
     ["Amazon Bedrock API key", /\b(bedrock-api-key-[A-Za-z0-9+/_-]{40,2000}={0,2})/, ["bedrock-api-key-"]],
+    // The long-term key: `ABSK` and the base64 of `BedrockAPIKey-<id>-at-<account>:<secret>`.
+    ["Amazon Bedrock API key (long-term)", /\b(ABSKQmVkcm9ja0FQSUtleS[A-Za-z0-9+/]{40,2000}={0,2})/, ["absk"]],
+    // Cloudflare Tunnel's token: the base64 of `{"a":<account>,"t":<tunnel>,"s":<secret>}`.
+    [
+        "Cloudflare Tunnel token",
+        /(?<![A-Za-z0-9+/])(eyJhIjoi[A-Za-z0-9+/]{90,600}={0,2})(?![A-Za-z0-9+/=])/,
+        ["eyjhijoi"],
+    ],
+    ["Inngest signing key", /\b(signkey-(?:prod|test|branch)-[0-9a-f]{64})\b/, ["signkey-"]],
+    ["Firecrawl API key", /(?<![\w-])(fc-[0-9a-f]{32})\b/, ["fc-"]],
+    ["Context7 API key", /\b(ctx7sk-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b/, ["ctx7sk-"]],
+    ["Axiom token", /\b(xa[ap]t-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b/, ["xaat-", "xapt-"]],
     [
         "Heroku API key",
         /\b(HRKU-(?:[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}|AA[0-9A-Za-z_-]{58}))\b/,

package/dist/index.js

@@ -226,7 +226,7 @@ export class Redactor {
                 ...f,
                 start: n.map[f.start],
                 end: f.end > f.start ? n.ends[f.end - 1] : n.map[f.start],
-                value: f.value ?? n.text.slice(f.start, f.end),
+                value: secretValue(n.text, f),
             }));
             // An escaped value in markup (`<password>Vq7m&lt;Zp2</password>`, `value="a&quot;b"`): decoded, its `<` or `"`
             // ends what the XML and attribute readers read, so the names are also read where it's written. (Escaped code in
@@ -282,8 +282,14 @@ export class Redactor {
                 !(f.name !== undefined && f.start === l.map[0] && !l.decoded.includes("\n")));
         };
         const decoded = merge(extra);
+        // (Where both passes read a value from the same place, the decoded one says where it ends: read raw, Go's
+        // `client_secret=…\u0026grant_type=…` runs on through the escaped `&`.)
+        const decodedStarts = new Set(extra.map((f) => f.start));
         return merge([
-            ...found.filter((f) => isVouched(f.rule) || layerOf(f.rule) === "repeat" || !insideLiteral(f) || overlaps(decoded, f.start, f.end)),
+            ...found.filter((f) => isVouched(f.rule) ||
+                layerOf(f.rule) === "repeat" ||
+                !insideLiteral(f) ||
+                (overlaps(decoded, f.start, f.end) && !decodedStarts.has(f.start))),
             ...extra,
         ]);
     }
@@ -297,7 +303,7 @@ export class Redactor {
         const out = [];
         for (const { decoded, map } of decodings) {
             for (const f of this.detectNormalized(decoded, { ...options, hint: {} }, depth + 1))
-                out.push({ ...f, start: map[f.start], end: map[f.end], value: f.value ?? decoded.slice(f.start, f.end) });
+                out.push({ ...f, start: map[f.start], end: map[f.end], value: secretValue(decoded, f) });
         }
         return out;
     }
@@ -376,7 +382,7 @@ export class Redactor {
                 : this.detectNormalized(decoded, { guessing: "off", encoded: encoded + 1 }, 0).some((f) => !isGuess(f.rule));
             found.push(...findEncoded(text, inner), ...findHexDumps(text, inner));
         }
-        found = merge(wrapped(text, found));
+        found = merge(wrapped(text, found), text);
         if (on.has("random")) {
             // (Guessing as asked, but in a secret file, the hint's or one written inside the text, always: there the model
             // isn't guessing whether secrets are present, it tells the file's generated values from its settings.)

package/dist/json.js

@@ -8,7 +8,16 @@
  * (transcript.ts): the shell command a tool runs, the path it reads or writes, and the text it writes there.
  */
 export const TOOL_COMMAND_KEYS = ["command", "cmd", "script", "shell_command", "run"];
-export const TOOL_PATH_KEYS = ["file_path", "path", "filePath", "filename", "file", "target_file", "notebook_path"];
+export const TOOL_PATH_KEYS = [
+    "file_path",
+    "path",
+    "filePath",
+    "absolute_path",
+    "filename",
+    "file",
+    "target_file",
+    "notebook_path",
+];
 export const TOOL_CONTENT_KEYS = [
     "content",
     "contents",
@@ -27,13 +36,19 @@ export const TOOL_CONTENT_KEYS = [
     "code",
     "edits",
 ];
+/**
+ * A key that names a command line by its last word, in any case and joining (`CommandLine`, Windsurf's run_command
+ * input; `commandLine`, `bash_command`, `COMMAND_LINE`, `shellCmd`; vercel.json's `buildCommand`), or one of the tool
+ * keys.
+ */
+const COMMAND_KEY = String.raw `(?:(?:[A-Za-z\d]*[a-z\d](?=[A-Z])|[\w-]*[_-])?(?:command|Command|COMMAND|cmd|Cmd|CMD)(?:[_-]?(?:line|Line|LINE))?|${TOOL_COMMAND_KEYS.join("|")})`;
 /** The keys tools take text under: an editor's `new_string`, a writer's `content`, a shell's `command`, a tool's `input`. */
-const TOOL_TEXT_KEY = String.raw `(?:${[...TOOL_CONTENT_KEYS, ...TOOL_COMMAND_KEYS, "input"].join("|")})`;
+const TOOL_TEXT_KEY = String.raw `(?:${[...TOOL_CONTENT_KEYS, COMMAND_KEY, "input"].join("|")})`;
 const TOOL_TEXT_ARGUMENT = new RegExp(String.raw `"${TOOL_TEXT_KEY}"[ \t]*:[ \t]*"`);
 /** A literal's opening quote right after a tool's text key (`"new_string": "`, escaped any number of times). */
 const AFTER_TOOL_TEXT_KEY = new RegExp(String.raw `(?<![\w$-])${TOOL_TEXT_KEY}\\*"[ \t]*:[ \t]*$`);
 /** A literal's opening quote right after a shell command's key (`"command": "`, `"cmd": "`): a command line, read as one. */
-const AFTER_COMMAND_KEY = new RegExp(String.raw `(?<![\w$-])(?:${TOOL_COMMAND_KEYS.join("|")})\\*"[ \t]*:[ \t]*$`);
+const AFTER_COMMAND_KEY = new RegExp(String.raw `(?<![\w$-])${COMMAND_KEY}\\*"[ \t]*:[ \t]*$`);
 /**
  * Whether a text may hold JSON string literals worth decoding: escapes a document's literals carry (a line break,
  * a tab, a quote, a slash, a Unicode escape), or a tool's text argument, decoded even without escapes (a one-line
@@ -173,6 +188,11 @@ export function jsonStringLiterals(text) {
             decoded.length >= 20 &&
             (isOutputText(decoded) ||
                 decoded.includes('"') ||
+                // (A character escaped as a code or a doubled backslash, Go's `\u0026` for `&`, Python's `\u00e9`, a path's
+                // `\\`: read raw, `-pQm7v\u0026Tx2…` or `postgres://app:Qm7v\u0026Tx2…@db` isn't the value the program saw.
+                // Only where JSON's syntax stands around it, as for the slash: a shell's `"s/\\(token=\\).*/\\1…/"` is no JSON.)
+                (/\\(?:u[0-9a-fA-F]{4}|\\)/.test(text.slice(map[0], map[map.length - 1])) &&
+                    isJsonPlaced(text, open, map[map.length - 1])) ||
                 // (An escaped slash, PHP's `"https:\/\/…"`, only where JSON's syntax stands around the literal: in a sed
                 // script in a shell's string, `"s/^DB_PASSWORD=.*/DB_PASSWORD=a\/b/"`, it escapes the script's delimiter.)
                 (decoded.includes("/") &&
@@ -338,8 +358,12 @@ export function* jsonDocuments(text) {
         yield doc.node;
     }
 }
-/** The `{…}` object around [start, end), braces balanced (braces inside its strings too), within `max` characters on each side; "" if none. */
-function enclosingObject(text, start, end, max = 2000) {
+/**
+ * The `{…}` object around [start, end), braces balanced (braces inside its strings too), within `max` characters on
+ * each side; "" if none. (Enough for an RSA-8192 private JWK, about 6,500 characters, whose last members sit far from
+ * its `{`.)
+ */
+function enclosingObject(text, start, end, max = 8000) {
     let depth = 0;
     let open = -1;
     for (let i = start - 1; i >= Math.max(0, start - max); i--) {

package/dist/known.d.ts

@@ -7,10 +7,9 @@ export declare class KnownValues {
     private shortB64;
     /** Every short form in the order added; `word` if it's matched as a whole word (`short`), else anywhere (`shortB64`). */
     private shortForms;
-    /** The short forms indexed: all up to `mainCount` in one index, the rest in a small one (see rebuild). */
-    private main?;
-    private mainCount;
-    private recent?;
+    /** The short forms indexed, in indexes of decreasing size (see rebuild), and how many they hold. */
+    private levels;
+    private indexed;
     private labels;
     /** The value each form was made from: a form found is the value written another way, and tagged as the value. */
     private origins;
@@ -23,9 +22,10 @@ export declare class KnownValues {
     /** Adds a secret value. Values too short or too ordinary to replace everywhere are ignored. (One already held keeps its first label.) */
     add(value: string, label?: string): boolean;
     /**
-     * Indexes the short forms added since the last find. They go in a small index of their own, so learning a value and
-     * then finding (a session that learns as it goes) costs that small index, not all of them; once it holds more than
-     * RECENT_FORMS, every form is indexed in one again.
+     * Indexes the short forms added since the last find. They go in an index of their own, merged with the indexes
+     * no bigger than it, so each holds at least twice what the next one holds: a session that learns a value and then
+     * finds, over and over, indexes each form a logarithmic number of times, not every form at every few values, and
+     * a find reads a logarithmic number of indexes.
      */
     private rebuild;
     find(text: string): Finding[];

package/dist/known.js

@@ -52,13 +52,25 @@ function variants(value) {
     out.add(json.replace(/\//g, "\\/"));
     out.add(json.replace(/[&<>']/g, (c) => `\\u00${c.charCodeAt(0).toString(16).padStart(2, "0")}`));
     out.add(JSON.stringify(json).slice(1, -1)); // twice-escaped: JSON inside a JSON string
+    // Every character past ASCII as a code, as Python's json.dumps and PHP's json_encode (with `\/`) write it, and .NET's
+    // in capitals.
+    const ascii = json.replace(/[^\x00-\x7f]/g, (c) => `\\u${c.charCodeAt(0).toString(16).padStart(4, "0")}`);
+    out.add(ascii);
+    out.add(ascii.replace(/\//g, "\\/"));
+    out.add(ascii.replace(/\\u[0-9a-f]{4}/g, (m) => `\\u${m.slice(2).toUpperCase()}`));
     let uri = value;
     try {
         uri = encodeURIComponent(value);
     }
     catch { } // lone surrogates
     out.add(uri);
-    out.add(uri.replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`).replace(/%20/g, "+"));
+    // RFC 3986's unreserved characters only (Go's QueryEscape, Python's quote_plus with `+`, quote, which also leaves `/`),
+    // and a form body's (URLSearchParams, Java's URLEncoder: `*` left, `~` escaped, a space as `+`).
+    const strict = uri.replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
+    out.add(strict);
+    out.add(strict.replace(/%20/g, "+"));
+    out.add(strict.replace(/%2F/g, "/"));
+    out.add(strict.replace(/%2A/g, "*").replace(/~/g, "%7E").replace(/%20/g, "+"));
     out.add(uri.toLowerCase() === uri ? uri : uri.replace(/%[0-9A-F]{2}/g, (m) => m.toLowerCase()));
     out.add(value.replace(/([^\w\-.,:/@+=%])/g, "\\$1")); // printf %q
     out.add(value.replace(/(["\\$`])/g, "\\$1")); // inside double quotes
@@ -178,8 +190,6 @@ class LiteralIndex {
     }
 }
 const TOKEN_CHAR = /[A-Za-z0-9+/_-]/;
-/** How many short forms are indexed apart from the rest before all are indexed in one again. */
-const RECENT_FORMS = 256;
 export class KnownValues {
     long = new LiteralIndex();
     ends = new LiteralIndex();
@@ -188,10 +198,9 @@ export class KnownValues {
     shortB64 = new Map();
     /** Every short form in the order added; `word` if it's matched as a whole word (`short`), else anywhere (`shortB64`). */
     shortForms = [];
-    /** The short forms indexed: all up to `mainCount` in one index, the rest in a small one (see rebuild). */
-    main;
-    mainCount = 0;
-    recent;
+    /** The short forms indexed, in indexes of decreasing size (see rebuild), and how many they hold. */
+    levels = [];
+    indexed = 0;
     labels = new Map();
     /** The value each form was made from: a form found is the value written another way, and tagged as the value. */
     origins = new Map();
@@ -248,25 +257,22 @@ export class KnownValues {
         return true;
     }
     /**
-     * Indexes the short forms added since the last find. They go in a small index of their own, so learning a value and
-     * then finding (a session that learns as it goes) costs that small index, not all of them; once it holds more than
-     * RECENT_FORMS, every form is indexed in one again.
+     * Indexes the short forms added since the last find. They go in an index of their own, merged with the indexes
+     * no bigger than it, so each holds at least twice what the next one holds: a session that learns a value and then
+     * finds, over and over, indexes each form a logarithmic number of times, not every form at every few values, and
+     * a find reads a logarithmic number of indexes.
      */
     rebuild() {
-        const indexed = this.mainCount + (this.recent?.forms.length ?? 0);
-        if (indexed === this.shortForms.length)
+        if (this.indexed === this.shortForms.length)
             return;
-        const index = (forms) => ({
+        let forms = this.shortForms.slice(this.indexed);
+        while (this.levels.length && this.levels[this.levels.length - 1].forms.length <= forms.length)
+            forms = [...this.levels.pop().forms, ...forms];
+        this.levels.push({
             set: new KeywordSet(forms.map((f) => f.form), true),
             forms,
         });
-        if (this.shortForms.length - this.mainCount > RECENT_FORMS) {
-            this.main = index(this.shortForms.slice());
-            this.mainCount = this.shortForms.length;
-            this.recent = undefined;
-        }
-        else
-            this.recent = index(this.shortForms.slice(this.mainCount));
+        this.indexed = this.shortForms.length;
     }
     find(text) {
         this.rebuild();
@@ -281,9 +287,7 @@ export class KnownValues {
                 value: this.origins.get(v),
             });
         // Short values whole only (or straight after a flag like `-p`); short base64 pieces anywhere.
-        for (const index of [this.main, this.recent]) {
-            if (!index)
-                continue;
+        for (const index of this.levels) {
             for (const [at, k] of index.set.matches(text)) {
                 const { form: v, word } = index.forms[k];
                 const dash = v.includes("-");
@@ -312,11 +316,15 @@ export class KnownValues {
                 fullEnd = Math.max(fullEnd, full[fi++].end);
             if (fullEnd >= at + W || at < lastEnd)
                 continue;
+            // (Over the characters the value has: a token with no `/` cut short after a path, `…/repos/o/r/ghp_Zq8v…`,
+            // takes none of the path with it.)
+            const value = this.origins.get(v) ?? "";
+            const inValue = (c) => TOKEN_CHAR.test(c) && (/[A-Za-z0-9]/.test(c) || value.includes(c));
             let start = at;
             let end = at + W;
-            while (start > 0 && TOKEN_CHAR.test(text[start - 1]))
+            while (start > 0 && inValue(text[start - 1]))
                 start--;
-            while (end < text.length && TOKEN_CHAR.test(text[end]))
+            while (end < text.length && inValue(text[end]))
                 end++;
             lastEnd = end;
             out.push({

package/dist/named.d.ts

@@ -12,6 +12,12 @@ export declare function isPairLabel(name: string, label: string): boolean;
 export declare function isNamedSecret(name: string, raw: string, literal?: boolean): boolean;
 /** Values under a credential name. (Values public by design, client-side Google keys and DocSearch keys among them, are policy.ts's.) */
 export declare function findNamed(text: string): Finding[];
+/**
+ * The key of the object or mapping the entry at `at` sits in: before the nearest `{` it is inside (`mixpanel: {`,
+ * `"mixpanel": {`), or a YAML line indented less with nothing after its key (`mixpanel:`). Read within 2,000
+ * characters; undefined if none.
+ */
+export declare function enclosingKey(text: string, at: number): string | undefined;
 /**
  * A word a sentence has and a passphrase doesn't: a function word, or a verb's or an adverb's form (`hashed`, `using`,
  * `effectively`) that isn't a word of BIP39's list, which passphrases are drawn from (`jelly`, `bring`, `speed`).