redac---n 0.0.0-stage → 0.0.2

package/dist/random.js

@@ -0,0 +1,165 @@
+/**
+ * Measures of how random a token looks, from the token alone, for the random layer's model (unnamed.ts): the
+ * alphabet it is drawn from, its entropy as a z-score against a uniformly random string of the same length over that
+ * alphabet (so a 20-character hex string and a 60-character base64 one are judged on one scale), and its character
+ * classes and how often they switch. A random key, a hash and an encrypted blob all look random here; telling them
+ * apart is the context's job.
+ */
+/** Sizes of the alphabets a token is drawn from. (`other` is the 62 letters and digits and about 8 of the symbols keys use; the model's features depend on it.) */
+const ALPHABET_SIZE = {
+    hex: 16,
+    HEX: 16,
+    b32: 32,
+    b36: 36,
+    B36: 36,
+    b62: 62,
+    b64: 64,
+    b64url: 64,
+    other: 70,
+};
+/** The smallest common alphabet that contains every character of the token. (`=` is base64's padding, in none.) */
+export function alphabetOf(token) {
+    let lower = false;
+    let upper = false;
+    let digit = false;
+    let hexOnly = true;
+    let b64 = false;
+    let url = false;
+    let other = false;
+    let b32 = true;
+    for (let i = 0; i < token.length; i++) {
+        const c = token.charCodeAt(i);
+        if (c >= 97 && c <= 122) {
+            lower = true;
+            if (c > 102)
+                hexOnly = false;
+        }
+        else if (c >= 65 && c <= 90) {
+            upper = true;
+            if (c > 70)
+                hexOnly = false;
+        }
+        else if (c >= 48 && c <= 57) {
+            digit = true;
+            if (c < 50 || c > 55)
+                b32 = false;
+        }
+        else if (c === 43 || c === 47)
+            b64 = true;
+        else if (c === 45 || c === 95)
+            url = true;
+        else if (c !== 61)
+            other = true;
+    }
+    if (other || (b64 && url))
+        return "other";
+    if (b64)
+        return "b64";
+    if (url)
+        return "b64url";
+    if (hexOnly && !(lower && upper))
+        return upper ? "HEX" : "hex";
+    if (upper && !lower && b32)
+        return "b32";
+    if (lower && upper)
+        return "b62";
+    if (upper)
+        return "B36";
+    return digit || lower ? "b36" : "other";
+}
+/** Shannon entropy of the characters, in bits per character. */
+export function entropy(token) {
+    const counts = new Map();
+    for (let i = 0; i < token.length; i++) {
+        const c = token.charCodeAt(i);
+        counts.set(c, (counts.get(c) ?? 0) + 1);
+    }
+    let bits = 0;
+    for (const n of counts.values()) {
+        const p = n / token.length;
+        bits -= p * Math.log2(p);
+    }
+    return bits;
+}
+/**
+ * The expected entropy and its standard deviation for a uniformly random string of length `n` (at most 512) over
+ * `k` characters, estimated once by simulation and cached. Lets a threshold be a z-score instead of a
+ * fixed number of bits, which is too strict for short tokens and too loose for long ones.
+ */
+const entropyStats = new Map();
+function randomEntropy(k, n) {
+    const key = k * 10_000 + Math.min(n, 512);
+    let s = entropyStats.get(key);
+    if (s)
+        return s;
+    n = Math.min(n, 512);
+    // Fixed seed, so results never vary between runs.
+    let seed = key;
+    const next = () => {
+        seed = (seed + 0x6d2b79f5) | 0;
+        let t = Math.imul(seed ^ (seed >>> 15), 1 | seed);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+    const trials = 400;
+    const counts = new Uint16Array(k);
+    let sum = 0;
+    let sumSq = 0;
+    for (let t = 0; t < trials; t++) {
+        counts.fill(0);
+        for (let i = 0; i < n; i++)
+            counts[Math.floor(next() * k)]++;
+        let bits = 0;
+        for (let j = 0; j < k; j++)
+            if (counts[j])
+                bits -= (counts[j] / n) * Math.log2(counts[j] / n);
+        sum += bits;
+        sumSq += bits * bits;
+    }
+    const mean = sum / trials;
+    s = [mean, Math.sqrt(Math.max(1e-6, sumSq / trials - mean * mean))];
+    entropyStats.set(key, s);
+    return s;
+}
+/**
+ * How many standard deviations the token's entropy sits from a random string's. Random strings score near 0.
+ * (Measured over the first 512 characters, the longest string randomEntropy simulates.)
+ */
+export function entropyZ(token, alphabet = alphabetOf(token)) {
+    const t = token.slice(0, 512);
+    const [mean, sd] = randomEntropy(ALPHABET_SIZE[alphabet], t.length);
+    return (entropy(t) - mean) / sd;
+}
+function classOf(c) {
+    if (c >= 97 && c <= 122)
+        return 0;
+    if (c >= 65 && c <= 90)
+        return 1;
+    if (c >= 48 && c <= 57)
+        return 2;
+    return 3;
+}
+export function shape(token) {
+    const n = token.length;
+    const cls = [0, 0, 0, 0];
+    let switches = 0;
+    let prev = -1;
+    for (let i = 0; i < n; i++) {
+        const c = classOf(token.charCodeAt(i));
+        cls[c]++;
+        if (prev >= 0 && c !== prev)
+            switches++;
+        prev = c;
+    }
+    let same = 0;
+    for (const k of cls)
+        same += (k / n) ** 2;
+    return {
+        length: n,
+        lower: cls[0],
+        upper: cls[1],
+        digit: cls[2],
+        switches: switches / Math.max(1, n - 1),
+        expectedSwitches: n === 0 ? 0 : 1 - same,
+    };
+}

package/dist/repeats.d.ts

@@ -0,0 +1,21 @@
+import type { Finding } from "./types.ts";
+/**
+ * Other occurrences of confidently found values: a webhook token found in its URL is also hidden in
+ * the `POST /api/webhooks/…/<token>` line below it, a password found under its name also where it's
+ * echoed without one.
+ */
+export declare function repeats(text: string, found: Finding[]): Finding[];
+/** A value worth looking for elsewhere: what its finding called it, and the secret it is (for the placeholder's tag). */
+export interface Remembered {
+    kind: string;
+    value: string;
+    /** Whether the random layer guessed it: its other occurrences are proposed, not vouched for (rules.ts). */
+    guess?: boolean;
+}
+/** The values of confident findings worth looking for elsewhere, each as written (one secret may be written two ways). */
+export declare function repeatable(text: string, found: Finding[]): Map<string, Remembered>;
+/**
+ * Where `values` occur in `text` as whole tokens, outside what `found` covers: each under its first finding's kind and
+ * with its secret, so it has the placeholder that finding has.
+ */
+export declare function occurrences(text: string, values: ReadonlyMap<string, Remembered>, found: Finding[]): Finding[];

package/dist/repeats.js

@@ -0,0 +1,80 @@
+/** Values found once with confidence, hidden wherever else they appear in the same text. */
+import { KeywordSet } from "./aho.js";
+import { continuesToken } from "./syntax.js";
+import { isBenignValue } from "./values.js";
+import { covered, loginPassword, secretValue } from "./spans.js";
+import { isGuess, layerOf, PLACEHOLDER_RULE, REPEATED_GUESS } from "./rules.js";
+/**
+ * Other occurrences of confidently found values: a webhook token found in its URL is also hidden in
+ * the `POST /api/webhooks/…/<token>` line below it, a password found under its name also where it's
+ * echoed without one.
+ */
+export function repeats(text, found) {
+    return occurrences(text, repeatable(text, found), found);
+}
+/** The values of confident findings worth looking for elsewhere, each as written (one secret may be written two ways). */
+export function repeatable(text, found) {
+    const values = new Map();
+    for (const f of found) {
+        if (f.confidence < 0.9 || f.end - f.start < 10 || f.end - f.start > 4096 || f.rule === PLACEHOLDER_RULE)
+            continue;
+        const v = secretValue(text, f);
+        // (A hash's shape, a commit SHA or a SHA-256, spreads only from a rule that knows what it hid: one read by its
+        // name alone (`'key': '<sha>/dist/…'`) would otherwise hide that commit in every line of a CI log.)
+        if (/^[0-9a-f]{40}$|^[0-9a-f]{64}$/.test(v) &&
+            !["format", "known", "file", "command", "structure"].includes(layerOf(f.rule)) &&
+            !(layerOf(f.rule) === "named" && f.name && !/^(?:key|keys|hash|sha|etag|id|value|data)$/i.test(f.name)))
+            continue;
+        // (Only a value that can be a secret wherever it is: not a version range, a date, a count.)
+        if (isBenignValue(v))
+            continue;
+        const remembered = { kind: f.kind, value: v, ...(isGuess(f.rule) ? { guess: true } : {}) };
+        values.set(v, remembered);
+        // (And a login's password, `NEO4J_AUTH=neo4j/<password>`, where it turns up alone.)
+        const password = loginPassword(f, v);
+        if (password && !isBenignValue(password) && !values.has(password))
+            values.set(password, { ...remembered, value: password });
+        // (As it is written here too, when that's another form of it, and as a URL writes it: `Delta2018%26` in a URL,
+        // `Delta2018&` in prose, and the other way round. Only for 8 or more printable characters and no blank: half a
+        // surrogate pair has no URL form.)
+        if (/^[!-~]{8,}$/.test(v))
+            for (const form of [text.slice(f.start, f.end), encodeURIComponent(v)])
+                if (form !== v)
+                    values.set(form, remembered);
+    }
+    return values;
+}
+/**
+ * Where `values` occur in `text` as whole tokens, outside what `found` covers: each under its first finding's kind and
+ * with its secret, so it has the placeholder that finding has.
+ */
+export function occurrences(text, values, found) {
+    const out = [];
+    // (The most recent 50,000 values, past any secrets dump: an automaton over more grows without bound.)
+    const list = [...values.keys()].filter((v) => !(/\s/.test(v) && v.length < 16)).slice(-50_000);
+    if (!list.length)
+        return out;
+    const add = (at, v) => {
+        // Whole tokens only: not a prefix or suffix of a longer run.
+        const dash = v.includes("-");
+        if (continuesToken(text, at - 1, dash) ||
+            continuesToken(text, at + v.length, dash) ||
+            covered(found, at, at + v.length))
+            return;
+        const { kind, value, guess } = values.get(v);
+        out.push({ start: at, end: at + v.length, kind, rule: guess ? REPEATED_GUESS : "repeat", confidence: 0.9, value });
+    };
+    // A few values: a scan each. Many in a long text: every value in one pass.
+    // (Overlapping occurrences too, as the automaton finds them. Each scan is a fast `indexOf`; past about 8 M character comparisons the automaton's one pass is cheaper.)
+    if (list.length * text.length <= 8_000_000) {
+        for (const v of list)
+            for (let at = text.indexOf(v); at >= 0; at = text.indexOf(v, at + 1))
+                add(at, v);
+    }
+    else {
+        const set = new KeywordSet(list, true);
+        for (const [at, k] of set.matches(text))
+            add(at, list[k]);
+    }
+    return out;
+}

package/dist/rules.d.ts

@@ -0,0 +1,42 @@
+/**
+ * What the pipeline knows about a finding's rule beyond its name. A rule is named `layer:name` (`format:GitHub
+ * token`, `named:assign`, `file:opaque`), and the stages that treat findings differently by where they came from ask
+ * here, by property, rather than each matching rule names of its own.
+ */
+/** The detectors and the other sources of findings. */
+export type RuleLayer = "format" | "structure" | "named" | "unnamed" | "file" | "command" | "encoded" | "known" | "repeat" | "safety" | "placeholder" | "transcript";
+/** A placeholder of an earlier pass, found so it's replaced by itself and nothing overlapping it is hidden twice. */
+export declare const PLACEHOLDER_RULE = "placeholder";
+/** The layer a rule belongs to, its name before the colon. */
+export declare function layerOf(rule: string): RuleLayer;
+/**
+ * Whether a finding is vouched for, so the decision stage keeps it whatever it looks like: a value the caller or an
+ * earlier text said is secret (`known`, `repeat`), everything in a secret file or a secret command's output, except
+ * what is there under no credential's name (`file:opaque`, see needsJudging), a text a detector failed on (`safety`),
+ * and an earlier pass's placeholder.
+ */
+export declare function isVouched(rule: string): boolean;
+/**
+ * Proposed, not vouched for: a secret file's value under no credential's name (isOpaqueSetting), and another occurrence
+ * of a guess (a repeat of the random layer's finding), each judged in its own place like any proposal.
+ */
+export declare function needsJudging(rule: string): boolean;
+/** Another occurrence of a value the random layer guessed (repeats.ts). */
+export declare const REPEATED_GUESS = "repeat:guess";
+/**
+ * A guess about one place (the random-token layer's, and its repeats): never learned, judged again wherever it's
+ * repeated, and no reason on its own to call decoded text a secret.
+ */
+export declare function isGuess(rule: string): boolean;
+/** Rules that take a whole line as the secret, which give way to a narrower finding inside it. */
+export declare function isWholeLine(rule: string): boolean;
+/** A private key or a seed phrase, by its own structure: secret whatever the text around it says. */
+export declare function isKeyMaterial(rule: string): boolean;
+/**
+ * Found by where the value stands (a name, a structure's slot, a guess about its place), not by what the value is: the
+ * decision stage's "never a secret" values (counts, versions, words) apply. Not structures that say what the value is
+ * (a private key, a CI mask, a JWK, a password hash, a seed phrase).
+ */
+export declare function readsByPlace(rule: string): boolean;
+/** A token that a terminal wrapping at a fixed width may cut and continue on the next line: a format's, a bearer token, a URL parameter. */
+export declare function continuesAcrossWraps(rule: string): boolean;

package/dist/rules.js

@@ -0,0 +1,58 @@
+/**
+ * What the pipeline knows about a finding's rule beyond its name. A rule is named `layer:name` (`format:GitHub
+ * token`, `named:assign`, `file:opaque`), and the stages that treat findings differently by where they came from ask
+ * here, by property, rather than each matching rule names of its own.
+ */
+/** A placeholder of an earlier pass, found so it's replaced by itself and nothing overlapping it is hidden twice. */
+export const PLACEHOLDER_RULE = "placeholder";
+/** The layer a rule belongs to, its name before the colon. */
+export function layerOf(rule) {
+    const colon = rule.indexOf(":");
+    return (colon < 0 ? rule : rule.slice(0, colon));
+}
+/**
+ * Whether a finding is vouched for, so the decision stage keeps it whatever it looks like: a value the caller or an
+ * earlier text said is secret (`known`, `repeat`), everything in a secret file or a secret command's output, except
+ * what is there under no credential's name (`file:opaque`, see needsJudging), a text a detector failed on (`safety`),
+ * and an earlier pass's placeholder.
+ */
+export function isVouched(rule) {
+    return ["known", "repeat", "file", "command", "safety", "placeholder"].includes(layerOf(rule)) && !needsJudging(rule);
+}
+/**
+ * Proposed, not vouched for: a secret file's value under no credential's name (isOpaqueSetting), and another occurrence
+ * of a guess (a repeat of the random layer's finding), each judged in its own place like any proposal.
+ */
+export function needsJudging(rule) {
+    return rule === "file:opaque" || rule === REPEATED_GUESS;
+}
+/** Another occurrence of a value the random layer guessed (repeats.ts). */
+export const REPEATED_GUESS = "repeat:guess";
+/**
+ * A guess about one place (the random-token layer's, and its repeats): never learned, judged again wherever it's
+ * repeated, and no reason on its own to call decoded text a secret.
+ */
+export function isGuess(rule) {
+    return layerOf(rule) === "unnamed" || rule === REPEATED_GUESS;
+}
+/** Rules that take a whole line as the secret, which give way to a narrower finding inside it. */
+export function isWholeLine(rule) {
+    return rule === "file:whole" || rule === "file:bare" || rule === "command:prints-secret";
+}
+/** A private key or a seed phrase, by its own structure: secret whatever the text around it says. */
+export function isKeyMaterial(rule) {
+    return /^structure:(?:private-key|mnemonic)|^format:private key/.test(rule);
+}
+/**
+ * Found by where the value stands (a name, a structure's slot, a guess about its place), not by what the value is: the
+ * decision stage's "never a secret" values (counts, versions, words) apply. Not structures that say what the value is
+ * (a private key, a CI mask, a JWK, a password hash, a seed phrase).
+ */
+export function readsByPlace(rule) {
+    return (["named", "structure", "unnamed"].includes(layerOf(rule)) &&
+        !/^structure:(?:private-key|ci-mask|jwk|password-hash|mnemonic)/.test(rule));
+}
+/** A token that a terminal wrapping at a fixed width may cut and continue on the next line: a format's, a bearer token, a URL parameter. */
+export function continuesAcrossWraps(rule) {
+    return /^(?:format:|structure:(?:bearer|authorization|url-param))/.test(rule);
+}

package/dist/runs.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Long runs of a character set, found without looking at every character: any `min` characters in a row hold exactly one
+ * position that is a multiple of `min` (less one), so only those are read until one is in the set, and the run around it
+ * is walked. A regex like `[A-Za-z0-9+/=_-]{16,}` tries every position; in prose, logs and pretty-printed JSON, where the
+ * runs are short, this reads a sixteenth of the text.
+ */
+/** A table of the ASCII characters in `chars`, for `longRuns`. */
+export declare function asciiSet(chars: string): Uint8Array;
+/** The maximal runs of `set`'s characters at least `min` long, in order, as [start, end]. */
+export declare function longRuns(text: string, set: Uint8Array, min: number): Generator<[start: number, end: number]>;

package/dist/runs.js

@@ -0,0 +1,36 @@
+/**
+ * Long runs of a character set, found without looking at every character: any `min` characters in a row hold exactly one
+ * position that is a multiple of `min` (less one), so only those are read until one is in the set, and the run around it
+ * is walked. A regex like `[A-Za-z0-9+/=_-]{16,}` tries every position; in prose, logs and pretty-printed JSON, where the
+ * runs are short, this reads a sixteenth of the text.
+ */
+/** A table of the ASCII characters in `chars`, for `longRuns`. */
+export function asciiSet(chars) {
+    const set = new Uint8Array(128);
+    for (let i = 0; i < chars.length; i++)
+        set[chars.charCodeAt(i)] = 1;
+    return set;
+}
+/** The maximal runs of `set`'s characters at least `min` long, in order, as [start, end]. */
+export function* longRuns(text, set, min) {
+    const inSet = (i) => text.charCodeAt(i) < 128 && set[text.charCodeAt(i)] === 1;
+    let from = 0;
+    let i = min - 1;
+    while (i < text.length) {
+        if (!inSet(i)) {
+            i += min;
+            continue;
+        }
+        let start = i;
+        while (start > from && inSet(start - 1))
+            start--;
+        let end = i + 1;
+        while (end < text.length && inSet(end))
+            end++;
+        if (end - start >= min)
+            yield [start, end];
+        // (The next position to read: the first one past the run that is a multiple of `min`, less one.)
+        from = end;
+        i = end + ((((min - 1 - end) % min) + min) % min);
+    }
+}

package/dist/secretfile.d.ts

@@ -0,0 +1,52 @@
+import type { Finding } from "./types.ts";
+/**
+ * In a secret file, every value under a credential name is hidden whatever it looks like, and so is a
+ * line that is a single token (a bare token file, `cut -d= -f2` output); see isOpaqueSetting for other names.
+ * `path` is the file's, when known: a file that holds only secrets (`secrets.yml`) hides a generated token under any name.
+ */
+export declare function findInSecretFile(text: string, { part, path }?: {
+    part?: boolean | undefined;
+    path?: string | undefined;
+}): Finding[];
+/**
+ * The credential in the output of a command that prints one: its first line, whole (a passphrase of words, a PIN, a
+ * password that starts with `[`), unless the line is a message (`Error: …`), a record's field (`ID:  …`), a reference a
+ * variable holds instead (`op://vault/item/field`), a plain URL or address, or a JSON document whole (its pairs'
+ * readers judge it); and later lines that are one token, as `pass show` prints its metadata under the password.
+ * `--output text` rows of AWS's.
+ */
+export declare function findPrintedSecret(text: string): Finding[];
+/**
+ * Where the first prompt line inside a command's output starts (a later command), or its length if none. (Not a `!`
+ * line: in a command's output that is a value or an option file's directive, not a notebook's command.)
+ */
+export declare function firstPrompt(text: string): number;
+/**
+ * Command transcripts inside a text (a notebook's `!cat .env`, a pasted terminal session's
+ * `$ cat ~/.aws/credentials`): the output under a prompt that prints a secret file, up to the next
+ * prompt or a closing fence, is treated as that file.
+ */
+export declare function findEmbeddedOutputs(text: string): Finding[];
+export interface FileSection {
+    start: number;
+    end: number;
+    /** Lines added to a file (`>>`, `tee -a`, one of a group's writes) are part of it, not all of it. */
+    part: boolean;
+}
+/**
+ * Where a secret file's content sits inside a text, as [start, end) spans, in order and apart:
+ * - a diff of one (`git diff` of `.env.production`): its hunks, once normalization has removed their markers;
+ * - what a command line writes into one (commands.ts commandsRun reads it, through `sudo sh -c`, `ssh`, a prompt): a
+ *   heredoc's body (`cat > .env <<'EOF'`, `sudo tee .env > /dev/null <<EOF`, `cat <<EOF | tee .env`), a here-string
+ *   (`tee .env <<< "…"`), or `echo`'s and `printf`'s operands (`echo 'Harbor#2291x' > ~/.vault_pass`,
+ *   `{ echo A=1; echo B=2; } > .env`).
+ * Not a SOPS file, encrypted by design. A command line inside an earlier heredoc's body is that body's text.
+ */
+export declare function secretFileSections(text: string): FileSection[];
+/**
+ * The scopes still open at the end of a text, as the lines that open them, in order: a secret file's diff that runs to
+ * the end (its `+++` line and a hunk header), heredocs that write one and haven't ended (their command lines, made minimal, with any
+ * heredoc before them that is open too), or else the last prompt's command line, while no code fence has closed its
+ * output. Text that continues this one (a stream's next chunk) is read as it would be with these lines in front of it.
+ */
+export declare function openScopes(text: string): string[];