redac---n 0.0.0-stage → 0.0.2

package/dist/encoded.js

@@ -0,0 +1,336 @@
+/**
+ * Encodings that hold a secret, decoded and judged by what they hold: base64 blobs (`base64 .env`, a Kubernetes
+ * Secret's data, a docker config `auth`, a Basic header built by hand), base64 wrapped over lines, and hex dumps.
+ * The whole encoding is the secret.
+ */
+import { base64Prefix } from "./base64.js";
+import { naturalness } from "./charlm.js";
+import { lineEnd, lineStart } from "./lines.js";
+import { asciiSet, longRuns } from "./runs.js";
+import { keepReason } from "./policy.js";
+import { hasThreeClasses } from "./values.js";
+const UTF8 = new TextDecoder("utf-8");
+/** The share of characters that must be printable for bytes to count as text, and of zero bytes for UTF-16. */
+const TEXT_SHARE = 0.95;
+/**
+ * The text some bytes hold, if they hold text: UTF-8 (UTF-16 when every other byte is zero, as
+ * `powershell -EncodedCommand`'s is, with or without a byte-order mark), 95% of it printable (ASCII, tabs and line
+ * breaks, and every character from U+00A0 on but U+FFFD, which marks bytes that aren't UTF-8), and at least `min`
+ * characters long. Binary data, keys and hashes aren't text.
+ */
+function textOfBytes(bytes, min = 8) {
+    if (!bytes)
+        return undefined;
+    let b = bytes;
+    if ((b[0] === 0xff && b[1] === 0xfe) || (b[0] === 0xfe && b[1] === 0xff))
+        b = b.subarray(2);
+    let text;
+    for (const parity of [1, 0]) {
+        if (b.length < 16 || b[parity] !== 0)
+            continue;
+        let zeros = 0;
+        for (let i = parity; i < b.length; i += 2)
+            if (b[i] === 0)
+                zeros++;
+        if (zeros >= (b.length >> 1) * TEXT_SHARE) {
+            text = new TextDecoder(parity ? "utf-16le" : "utf-16be").decode(b);
+            break;
+        }
+    }
+    text ??= UTF8.decode(b);
+    if (text.length < min)
+        return undefined;
+    let printable = 0;
+    for (const ch of text) {
+        const c = ch.codePointAt(0);
+        if ((c >= 32 && c < 127) || c === 9 || c === 10 || c === 13 || (c >= 0xa0 && c !== 0xfffd))
+            printable++;
+    }
+    return printable >= [...text].length * TEXT_SHARE ? text : undefined;
+}
+/** Hex digits as bytes, or undefined for an odd count or a character that isn't a hex digit. */
+function hexBytes(hex) {
+    if (hex.length % 2 || !/^[0-9a-f]*$/i.test(hex))
+        return undefined;
+    const out = new Uint8Array(hex.length / 2);
+    for (let i = 0; i < out.length; i++)
+        out[i] = parseInt(hex.slice(i * 2, i * 2 + 2), 16);
+    return out;
+}
+const BASE64_SET = asciiSet("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/_-");
+/** The fewest characters of a base64 value worth decoding (12 bytes). */
+const MIN_BASE64_RUN = 16;
+const WRAPPED_SET = asciiSet("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/");
+const HEX_SET = asciiSet("0123456789abcdefABCDEF");
+/** The fewest hex digits of a run read as a hex-encoded value (12 bytes), and the most. */
+const MIN_HEX_RUN = 24;
+const MAX_HEX_RUN = 100000;
+/**
+ * The base64 values in `text`, whole: a run of the alphabet (base64 or base64url) with up to two `=` of padding, bounded
+ * by other characters, 16 characters with its padding (`V2ludGVyMjAyNCE=`, an 11-character password's).
+ */
+function* base64Runs(text) {
+    for (const [start, runEnd] of longRuns(text, BASE64_SET, MIN_BASE64_RUN - 2)) {
+        let end = runEnd;
+        while (end < runEnd + 2 && text[end] === "=")
+            end++;
+        if (end - start < MIN_BASE64_RUN || /[A-Za-z0-9+/=_-]/.test(text[end] ?? ""))
+            continue;
+        yield [start, end];
+    }
+}
+/**
+ * Base64 blobs whose decoded text holds a secret (`inner` decides), on one line or wrapped over several; and a line
+ * that is only base64 of one generated-looking token (`echo "$KEY" | base64`).
+ */
+export function findEncoded(text, inner) {
+    const out = [];
+    const wrapped = wrappedBase64(text);
+    // (A line of a blob wrapped over lines is no token of its own, however random its piece of the blob reads. Runs and
+    // blobs both come in text order: one pass over each.)
+    let w = 0;
+    const inBlob = (start) => {
+        while (w < wrapped.length && wrapped[w].end <= start)
+            w++;
+        return w < wrapped.length && wrapped[w].start <= start;
+    };
+    for (const [start, end] of base64Runs(text)) {
+        const value = text.slice(start, end);
+        const decoded = textOfBytes(base64Prefix(value));
+        if (decoded === undefined)
+            continue;
+        if (inner(decoded))
+            out.push({ start, end, kind: "base64 of a secret", rule: "encoded:base64", confidence: 0.95 });
+        else if (/^[!-~]{8,200}\n?$/.test(decoded) &&
+            /[=]$|^[A-Za-z0-9+/]+$/.test(value) &&
+            isLineAlone(text, start, end) &&
+            !inBlob(start) &&
+            isGeneratedToken(decoded.trim())) {
+            out.push({ start, end, kind: "base64 of a secret", rule: "encoded:base64-token", confidence: 0.8 });
+        }
+    }
+    for (const run of wrapped) {
+        const decoded = textOfBytes(base64Prefix(run.text));
+        if (decoded !== undefined && inner(decoded))
+            out.push({
+                start: run.start,
+                end: run.end,
+                kind: "base64 of a secret",
+                rule: "encoded:base64-wrapped",
+                confidence: 0.95,
+            });
+    }
+    return out;
+}
+/**
+ * A token a generator made, or a password a person made with symbols in it; not one public by design or never a key
+ * (which the decision stage keeps: `pk_live_…`), a UUID, hex, an email or a URL.
+ */
+function isGeneratedToken(token) {
+    if (/\s/.test(token) ||
+        /^[0-9a-f]+$/i.test(token) ||
+        /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(token) ||
+        /^[\w.-]+@[\w.-]+$|^https?:/.test(token))
+        return false;
+    if (keepReason(token, { start: 0, end: token.length, kind: "random token", rule: "unnamed", confidence: 0.8 }) !==
+        undefined)
+        return false;
+    return naturalness(token) < 0.2 || (hasThreeClasses(token) && /[^A-Za-z0-9]/.test(token));
+}
+const isBase64Char = (ch) => /[A-Za-z0-9+/]/.test(ch);
+/** Where the base64 (and `=` padding) at the end of `line` starts. */
+function base64Start(line) {
+    let k = line.length;
+    while (k > 0 && line[k - 1] === "=" && line.length - k < 2)
+        k--;
+    while (k > 0 && isBase64Char(line[k - 1]))
+        k--;
+    return k;
+}
+/** Where the base64 (and `=` padding) at the start of `line` ends. */
+function base64End(line) {
+    let k = 0;
+    while (k < line.length && isBase64Char(line[k]))
+        k++;
+    for (let padding = 0; k < line.length && line[k] === "=" && padding < 2; padding++)
+        k++;
+    return k;
+}
+/**
+ * Base64 wrapped over lines (`base64 .env`, `base64 -w 76`, PEM-like bodies, JSON with `\n`): full-width lines of
+ * 16 to 200 base64 characters, then one shorter last line or a line ending in `=`. Their characters joined, and
+ * the span from the first to the last. The first line may have text before its base64 (`data: `, `"d":"`) and the
+ * last text after it (`"}`, `")`): of two lines or more, a line's base64 at its end opens a run, and a line's
+ * base64 at its start, followed by a quote or a closing bracket, ends it.
+ */
+function wrappedBase64(text) {
+    // (A run has a line of 16 base64 characters at least.)
+    if (longRuns(text, WRAPPED_SET, MIN_BASE64_RUN).next().done)
+        return [];
+    const out = [];
+    let run;
+    const close = () => {
+        // (One line alone is base64Runs' to judge.)
+        if (run && run.text.length >= 24 && run.lines > 1)
+            out.push(run);
+        run = undefined;
+    };
+    let offset = 0;
+    const parts = text.split(/(\\n|\r?\n)/);
+    for (let i = 0; i < parts.length; offset += parts[i].length, i++) {
+        if (i % 2)
+            continue;
+        // (No run is open, and a line this short opens none: a run's first line has 16 base64 characters.)
+        if (!run && parts[i].length < MIN_BASE64_RUN)
+            continue;
+        const body = parts[i].trim();
+        const at = offset + parts[i].indexOf(body);
+        // (Any other line, a blank one too, ends a run; one with base64 at its end opens the next.)
+        if (!/^[A-Za-z0-9+/]+={0,2}$/.test(body)) {
+            const head = base64End(body);
+            if (run && head > 0 && head <= run.width && /["'`)\]}>]/.test(body[head])) {
+                run.text += body.slice(0, head);
+                run.end = at + head;
+                run.lines++;
+                close();
+                continue;
+            }
+            close();
+            const tail = base64Start(body);
+            const length = body.length - tail;
+            if (tail > 0 && length >= 16 && length <= 200 && /[\s"'`:=(\[>,]/.test(body[tail - 1]))
+                run = { text: body.slice(tail), start: at + tail, end: at + body.length, width: length, lines: 1 };
+            continue;
+        }
+        if (run && body.length <= run.width) {
+            run.text += body;
+            run.end = at + body.length;
+            run.lines++;
+            // (A shorter line, or padding, is the last. A shorter line that more base64 no wider follows may open a blob of
+            // its own instead, after a value printed unwrapped (`base64 -w0`, then `base64`'s 76 columns): it's read both
+            // ways.)
+            if (body.length < run.width || body.endsWith("=")) {
+                close();
+                if (!body.endsWith("=") && body.length >= MIN_BASE64_RUN && continuesAfter(parts, i, body.length))
+                    run = { text: body, start: at, end: at + body.length, width: body.length, lines: 1 };
+            }
+            continue;
+        }
+        close();
+        if (body.length >= 16 && body.length <= 200) {
+            run = { text: body, start: at, end: at + body.length, width: body.length, lines: 1 };
+            if (body.endsWith("="))
+                close();
+        }
+    }
+    close();
+    return out;
+}
+/** Whether the line after `parts[i]` (text split on its line breaks, the breaks kept) is base64 no wider than `width`. */
+function continuesAfter(parts, i, width) {
+    const next = parts[i + 2]?.trim();
+    return !!next && next.length <= width && /^[A-Za-z0-9+/]+={0,2}$/.test(next);
+}
+/** Whether [start, end) is all that's on its line, apart from a log timestamp or prompt prefix. */
+function isLineAlone(text, start, end) {
+    const ls = lineStart(text, start, 200);
+    const le = lineEnd(text, end, 200);
+    const before = text.slice(ls, start);
+    const after = text.slice(end, le);
+    return /^\s*$/.test(after) && /^(?:\s*|\d{4}-\d\d-\d\dT[\d:.]+Z?\s+|[$>+#]\s+)$/.test(before);
+}
+/**
+ * Hex dumps (`xxd`, `xxd -p`, `hexdump -C`, `od -A x -t x1`): the bytes are decoded, and a dump whose
+ * contents hold a secret is redacted whole.
+ */
+export function findHexDumps(text, inner) {
+    const out = [];
+    if (!/[0-9a-f]{4}/i.test(text))
+        return out;
+    // Offset-prefixed dumps: runs of lines `<offset>[:] <hex groups> [ascii]`.
+    for (const m of text.matchAll(/(?:^[0-9a-f]{6,8}:?(?:[ \t]{1,4}[0-9a-f]{2,4}){1,16}[^\n]{0,80}(?:\n|$)){1,100000}/gim)) {
+        const words = [];
+        let bytes = "";
+        const lines = m[0].split("\n").filter((l) => l);
+        // Bytes per line from the offset step (16 for xxd and `hexdump -C`), so an ASCII column that
+        // starts with hex letters (`cafe.txt`, `DEBUG=1.`) isn't read as more bytes.
+        const offsets = lines.slice(0, 2).map((l) => parseInt(l.slice(0, 8), 16));
+        const step = offsets.length === 2 && offsets[1] > offsets[0] && offsets[1] - offsets[0] <= 64 ? offsets[1] - offsets[0] : 16;
+        for (const line of lines) {
+            const body = line.replace(/^[0-9a-f]{6,8}:?[ \t]+/i, "");
+            let digits = 0;
+            for (const t of body.split(/[ \t]+/)) {
+                if (!/^(?:[0-9a-f]{2}|[0-9a-f]{4}|[0-9a-f]{8})$/i.test(t) || digits + t.length > step * 2)
+                    break;
+                if (t.length === 4)
+                    words.push(t.slice(2) + t.slice(0, 2));
+                bytes += t;
+                digits += t.length;
+            }
+        }
+        for (const hex of [bytes, words.join("")]) {
+            if (hex.length < 12)
+                continue;
+            const decoded = textOfBytes(hexBytes(hex), 0);
+            if (decoded !== undefined && inner(decoded)) {
+                out.push({
+                    start: m.index,
+                    end: m.index + m[0].replace(/\n$/, "").length,
+                    kind: "hex dump of a secret",
+                    rule: "encoded:hexdump",
+                    confidence: 0.95,
+                });
+                break;
+            }
+        }
+    }
+    // Plain hex (`xxd -p`): runs of whole lines of hex, or one long run, that decode to text holding a secret.
+    const runs = [];
+    for (const [start, end] of longRuns(text, HEX_SET, MIN_HEX_RUN)) {
+        // (A hex run that a letter or digit touches is part of a longer word; one past the most is no value but a dump. A
+        // literal's prefix is no word: `0x…` (mysqldump's --hex-blob, SQL Server), `\x…` (psql's bytea).)
+        const prefixed = /(?:^|[^0-9a-z])0x$|\\x$/i.test(text.slice(Math.max(0, start - 3), start));
+        if (end - start <= MAX_HEX_RUN &&
+            (prefixed || !/[0-9a-z]/i.test(text[start - 1] ?? "")) &&
+            !/[0-9a-z]/i.test(text[end] ?? ""))
+            runs.push({ index: start, text: text.slice(start, end) });
+    }
+    // (Lines are read together, a dump's last line as short as it is: a secret read line by line leaves its tail on the
+    // next line. A list of hashes, one a line, decodes to no text.)
+    let runStart = -1;
+    let runEnd = -1;
+    let lines = 0;
+    const flush = () => {
+        if (lines > 1)
+            runs.push({ index: runStart, text: text.slice(runStart, runEnd) });
+        runStart = -1;
+        lines = 0;
+    };
+    for (const m of text.matchAll(/^(?:[0-9a-f]{2}){1,100}\r?$/gim)) {
+        if (runStart >= 0 && m.index === runEnd + 1) {
+            runEnd = m.index + m[0].length;
+            lines++;
+        }
+        else {
+            flush();
+            if (m[0].trimEnd().length < 16)
+                continue;
+            runStart = m.index;
+            runEnd = m.index + m[0].length;
+            lines = 1;
+        }
+    }
+    flush();
+    for (const m of runs) {
+        const decoded = textOfBytes(hexBytes(m.text.replace(/\s+/g, "")));
+        if (decoded !== undefined && inner(decoded))
+            out.push({
+                start: m.index,
+                end: m.index + m.text.replace(/\s+$/, "").length,
+                kind: "hex of a secret",
+                rule: "encoded:hex",
+                confidence: 0.95,
+            });
+    }
+    return out;
+}

package/dist/environment.d.ts

@@ -0,0 +1,10 @@
+/**
+ * The secrets in an environment, as known values: variables whose name has a credential word (`*_API_KEY`, `*_TOKEN`,
+ * `BW_SESSION`, read widely), a vendor's token under any name, and URL passwords inside any variable. Only values a
+ * Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value is left out.
+ * `new Redactor({ known: fromEnvironment(process.env) })`.
+ */
+export declare function fromEnvironment(env: Record<string, string | undefined>): {
+    value: string;
+    label: string;
+}[];

package/dist/environment.js

@@ -0,0 +1,62 @@
+import { findFormats } from "./formats.js";
+import { isReplaceable } from "./known.js";
+import { hasCredentialWord, isCredentialName, isSettingName, namesPasswordOrPin } from "./names.js";
+import { isLiteralPassword } from "./structure.js";
+import { urls } from "./urls.js";
+import { naturalness } from "./charlm.js";
+import { hasThreeClasses, isInert, isRandomToken } from "./values.js";
+/**
+ * The secrets in an environment, as known values: variables whose name has a credential word (`*_API_KEY`, `*_TOKEN`,
+ * `BW_SESSION`, read widely), a vendor's token under any name, and URL passwords inside any variable. Only values a
+ * Redactor accepts as known values (the rule `addKnown` applies) are returned; a short or plain-word value is left out.
+ * `new Redactor({ known: fromEnvironment(process.env) })`.
+ */
+export function fromEnvironment(env) {
+    const out = [];
+    for (const [name, value] of Object.entries(env)) {
+        if (!value || value.length < 6 || value.length > 16_384)
+            continue;
+        // (Read widely: the caller's own environment, where a credential missed goes to a provider. A credential word anywhere
+        // in the name or a session's at its end, `BW_SESSION`, `TF_TOKEN_app_terraform_io`, `ACTIONS_ID_TOKEN_REQUEST_TOKEN`,
+        // unless the name ends as a setting's or a generator didn't make the value: `TOKEN_ENDPOINT`, `DESKTOP_SESSION=ubuntu-wayland`.
+        // A password's name takes any value, `MATCH_PASSWORD`; a webhook's URL is its secret whole.)
+        const widened = !isSettingName(name) &&
+            ((namesPasswordOrPin(name) && !isInert(value)) ||
+                ((hasCredentialWord(name) || /(?:^|_)(?:SESSION|PEPPER)$/i.test(name)) && isGenerated(value)));
+        const secretName = isCredentialName(name) || /WEBHOOK/i.test(name) || widened;
+        if (secretName && !isInert(value))
+            out.push({ value, label: `$${name}` });
+        // (A vendor's token under any name: `CI_DEPLOY=ghp_…`, a Slack webhook URL.)
+        else
+            for (const f of findFormats(value))
+                if (!out.some((o) => o.value === value.slice(f.start, f.end)))
+                    out.push({ value: value.slice(f.start, f.end), label: `${f.kind} in $${name}` });
+        // (A literal password, as the URL readers take one: not `${DB_PASSWORD}`, `<password>`, `%PASSWORD%`.)
+        for (const url of value.includes("://") ? urls(value) : [])
+            if (url.password && isLiteralPassword(url.password.value))
+                out.push({ value: decodeSafe(url.password.value), label: `password in $${name}` });
+    }
+    // (Only what a Redactor takes as a known value, so the caller isn't told a `JWT_SECRET=devsecret` is protected when it isn't.)
+    return out.filter((o) => isReplaceable(o.value.trim()));
+}
+/**
+ * A value a generator made: a random token (`Xk9mQ2pLrTvB7nWz`), or one with symbols in it, of all three other classes
+ * and no word (`Pq7!zR2@xW9#kL4m`); not a URL or a path.
+ */
+function isGenerated(value) {
+    return (isRandomToken(value, 12) ||
+        (value.length >= 12 &&
+            /^[!-~]+$/.test(value) &&
+            hasThreeClasses(value) &&
+            /[^A-Za-z0-9]/.test(value) &&
+            !/:\/\/|^[/~.$]/.test(value) &&
+            naturalness(value) < 0.3));
+}
+function decodeSafe(v) {
+    try {
+        return decodeURIComponent(v);
+    }
+    catch {
+        return v;
+    }
+}

package/dist/formats-data.d.ts

@@ -0,0 +1,2 @@
+/** [name, regex source, flags, keywords (any must appear, lower case), minimum entropy, 1 if anchored on a vendor word]. */
+export declare const IMPORTED_FORMATS: [string, string, string, string[], number, number][];