redac---n 0.0.0-stage → 0.0.2

package/dist/spans.d.ts

@@ -0,0 +1,61 @@
+import type { Finding } from "./types.ts";
+/**
+ * Whether a value is never a secret by its shape alone: an ARN, a host name, a file path, a URL with no password (an
+ * http(s) one only when it has no query and no segment long enough for a webhook's token). Learning one would hide it
+ * everywhere after, and a text that says `API_TOKEN=<an ARN>` can be written by anyone the tool output comes from.
+ * (Hex stays learnable: real keys are hex, and a commit SHA taught as one is over-redaction only.)
+ */
+export declare function isNeverSecretShape(value: string): boolean;
+/**
+ * The password of a finding that is a login: a username and a password in one value, `user:password` or
+ * `user/password`, under a name that says it holds one (names.ts namesLogin: `NEO4J_AUTH=neo4j/…`, Bundler's
+ * `BUNDLE_<HOST>: "user:…"`). Learned and repeated on its own too, as a client takes it (`cypher-shell -u neo4j -p
+ * <it>`). Not a `:` or `/` inside another secret (a signature's `abc:<id>`), nor a URL's (its password is read in the
+ * URL); before a `/`, only a user's lower-case name (`neo4j/…`, not a base64 key's `wJalrXUtnFEMI/K7MDENG…`).
+ */
+export declare function loginPassword(f: Finding, value: string): string | undefined;
+/**
+ * Whether a finding is sure enough, and long enough, to be learned as a known value and hidden wherever it appears: one
+ * a rule read by a name, a format, a structure or a file, not the unnamed layer's guess about one place; and whose
+ * `value` (secretValue) isn't of a shape no secret has (isNeverSecretShape).
+ */
+export declare function isLearnable(f: Finding, value: string, minConfidence?: number): boolean;
+/**
+ * The secret a finding covers, as the program reads it, so one secret written two ways gets one tag and is learned
+ * once: its recorded value (a known value whichever way it was written, a literal split in pieces joined, a JSON
+ * escape decoded); or the text it spans, read through a URL's percent-encoding (a userinfo's `Harbor2291%21x`, a
+ * query's `%26`, `+` a blank), a double-quoted string's escapes (`"Ab\"cd9"`, `\u0041`), or a shell's backslashes
+ * outside quotes (`Harbor2291\!x`). (Not a single-quoted string's: a shell's has no escapes, a SQL one doubles its
+ * quote, and `'a''b'` is `ab` to a shell.)
+ */
+export declare function secretValue(text: string, f: {
+    start: number;
+    end: number;
+    value?: string;
+}): string;
+/**
+ * A finding that may have been cut by a read bound (MAY_BE_CUT) and stops inside a word runs to the word's end, a blank
+ * or a quote: a longer secret must not leave its tail. A shorter one ends where its reader meant it to (a URL's
+ * password at its `@`). (A finding whose recorded value differs from its span, decoded or joined, is left as it is.)
+ */
+export declare function toWordEnd(text: string, f: Finding): Finding;
+/**
+ * A finding trimmed to its value, of what is syntax beyond doubt: blanks, an `=` or `:` with a blank or a quote after it,
+ * and a pair of quotes around the value (`= "CANYxh7686!"`); a comma or a semicolon before the next `name=` of a list
+ * (`passwort=COBAlyrxgo%, knownHosts=…`). A password's own first or last character (`:Kx9…`, `Kx9…)`, `Kx9…;`) stays.
+ * A placeholder then replaces the value only, so the syntax around it stays; and the value is what's fingerprinted and
+ * learned. (A finding that records its value, decoded or joined, is that value already.)
+ */
+export declare function trimmedToValue(text: string, f: Finding): Finding;
+/** Findings merged into one span each, sorted; with `text`, kept off escape sequences. (The inputs aren't changed.) */
+export declare function merge(findings: Finding[], text?: string): Finding[];
+/** Whether [start, end) lies inside one of `sorted` (merged, sorted by start). */
+export declare function covered(sorted: Finding[], start: number, end: number): boolean;
+/** Whether [start, end) overlaps one of `sorted` (merged, sorted by start). */
+export declare function overlaps(sorted: Finding[], start: number, end: number): boolean;
+/**
+ * A token cut by a terminal or log viewer wrapping at a fixed width (`Authorization: Bearer eyJ…` at
+ * column 80): a finding that ends exactly at the end of a line of a common wrap width continues on
+ * the following lines while they are bare token characters, up to the first shorter line.
+ */
+export declare function wrapped(text: string, found: Finding[]): Finding[];

package/dist/spans.js

@@ -0,0 +1,287 @@
+/**
+ * Spans of findings: merging overlapping ones, keeping them off escape sequences, extending a token a terminal
+ * wrapped, and what a finding's value is.
+ */
+import { lastText, lineEnd, lineStart, lowerBound } from "./lines.js";
+import { naturalness } from "./charlm.js";
+import { backslashesBefore, escapeLength, UNQUOTED_READ } from "./syntax.js";
+import { leadingUrl } from "./urls.js";
+import { isPath } from "./values.js";
+import { isReplaceable } from "./known.js";
+import { namesLogin } from "./names.js";
+import { continuesAcrossWraps, isGuess, isWholeLine, layerOf, PLACEHOLDER_RULE } from "./rules.js";
+/** An Amazon Resource Name: `arn:aws:iam::483920176654:role/deploy`, in any partition. */
+const ARN = /^arn:[a-z][a-z0-9-]*:[a-z0-9-]+:[a-z0-9-]*:\d*:[\w:/.@+=,*-]+$/;
+/** A host name: dotted lowercase labels ending in a TLD, with a port maybe (`db-main.cluster-cx8k2.us-west-2.rds.amazonaws.com:5432`). */
+const HOST_NAME = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)+[a-z]{2,24}(?::\d{1,5})?$/;
+/** A URL path segment this long may be a generated token (a webhook's), not a name. */
+const TOKEN_SEGMENT = 20;
+/**
+ * Whether a value is never a secret by its shape alone: an ARN, a host name, a file path, a URL with no password (an
+ * http(s) one only when it has no query and no segment long enough for a webhook's token). Learning one would hide it
+ * everywhere after, and a text that says `API_TOKEN=<an ARN>` can be written by anyone the tool output comes from.
+ * (Hex stays learnable: real keys are hex, and a commit SHA taught as one is over-redaction only.)
+ */
+export function isNeverSecretShape(value) {
+    if (ARN.test(value) || HOST_NAME.test(value) || isPath(value))
+        return true;
+    const url = leadingUrl(value);
+    if (!url || url.end !== value.length || url.password)
+        return false;
+    return (!/^https?$/i.test(url.scheme) || (!/[?#]/.test(value) && url.segments.every((s) => s.end - s.start < TOKEN_SEGMENT)));
+}
+/**
+ * The password of a finding that is a login: a username and a password in one value, `user:password` or
+ * `user/password`, under a name that says it holds one (names.ts namesLogin: `NEO4J_AUTH=neo4j/…`, Bundler's
+ * `BUNDLE_<HOST>: "user:…"`). Learned and repeated on its own too, as a client takes it (`cypher-shell -u neo4j -p
+ * <it>`). Not a `:` or `/` inside another secret (a signature's `abc:<id>`), nor a URL's (its password is read in the
+ * URL); before a `/`, only a user's lower-case name (`neo4j/…`, not a base64 key's `wJalrXUtnFEMI/K7MDENG…`).
+ */
+export function loginPassword(f, value) {
+    if (!f.name || !namesLogin(f.name) || value.includes("://"))
+        return undefined;
+    const pair = /^(?:[\w.@+-]{1,64}:|[a-z][a-z0-9._-]{0,31}\/)([^\s/:]\S*)$/.exec(value);
+    return pair && isReplaceable(pair[1]) ? pair[1] : undefined;
+}
+/**
+ * Whether a finding is sure enough, and long enough, to be learned as a known value and hidden wherever it appears: one
+ * a rule read by a name, a format, a structure or a file, not the unnamed layer's guess about one place; and whose
+ * `value` (secretValue) isn't of a shape no secret has (isNeverSecretShape).
+ */
+export function isLearnable(f, value, minConfidence = 0.9) {
+    return (f.confidence >= minConfidence &&
+        f.end - f.start >= 8 &&
+        !isGuess(f.rule) &&
+        f.rule !== PLACEHOLDER_RULE &&
+        !isNeverSecretShape(value));
+}
+/**
+ * The secret a finding covers, as the program reads it, so one secret written two ways gets one tag and is learned
+ * once: its recorded value (a known value whichever way it was written, a literal split in pieces joined, a JSON
+ * escape decoded); or the text it spans, read through a URL's percent-encoding (a userinfo's `Harbor2291%21x`, a
+ * query's `%26`, `+` a blank), a double-quoted string's escapes (`"Ab\"cd9"`, `\u0041`), or a shell's backslashes
+ * outside quotes (`Harbor2291\!x`). (Not a single-quoted string's: a shell's has no escapes, a SQL one doubles its
+ * quote, and `'a''b'` is `ab` to a shell.)
+ */
+export function secretValue(text, f) {
+    if (f.value !== undefined)
+        return f.value;
+    const raw = text.slice(f.start, f.end);
+    const before = text.slice(Math.max(0, f.start - 200), f.start);
+    if (/%[0-9A-Fa-f]{2}/.test(raw)) {
+        const query = /[?&][\w.[\]-]{1,60}=$/.test(before);
+        if (query || /:\/\/[^\s/@]*$/.test(before)) {
+            try {
+                return decodeURIComponent(query ? raw.replace(/\+/g, " ") : raw);
+            }
+            catch { } // an escape that isn't UTF-8: the text as it is
+        }
+    }
+    if (raw.includes("\\") && /"$/.test(before)) {
+        try {
+            return JSON.parse(`"${raw}"`);
+        }
+        catch { } // not JSON's escapes: the text as it is
+    }
+    if (/\\[^\w\s]/.test(raw) && !/["'`]$/.test(before))
+        return raw.replace(/\\([^\w\s])/g, "$1");
+    return raw;
+}
+/** The escape sequence covering position `i` (after its backslash), as [backslash, end), if `i` is inside one. */
+function escapeAt(text, i) {
+    for (let b = i - 1; b >= Math.max(0, i - 6); b--) {
+        if (text[b] !== "\\" || backslashesBefore(text, b) % 2)
+            continue;
+        const end = b + escapeLength(text, b);
+        return end > i ? [b, end] : undefined;
+    }
+    return undefined;
+}
+/** An escape spelling a letter or a digit: `\x63`, `\u0063`. */
+const SPELLED_CHAR = /\\(?:x|u00)(?:3[0-9]|[46][1-9a-fA-F]|[57][0-9aA])/y;
+/** How many of them a token goes on through. */
+const MAX_SPELLED = 64;
+/**
+ * A span kept off escape sequences, so replacing it never breaks an escape or, inside JSON, the document. One that
+ * starts or ends inside an escape (`\n<token>`, `<token>\`, `\u0020`) was read through it by mistake, and leaves it
+ * out. In text encoded twice, `\\n` is the next level's escape, or an escaped backslash and a letter of the
+ * secret (`C:\\nKx82…`): there is no telling, so a span right after an escaped backslash takes the pair in, and an
+ * error over-redacts rather than leaks.
+ */
+function onEscapes(text, f) {
+    let { start, end } = f;
+    const s = escapeAt(text, start);
+    if (s)
+        start = Math.min(end, s[1]);
+    else if (text[start - 1] === "\\" &&
+        text[start - 2] === "\\" &&
+        !escapeAt(text, start - 2) &&
+        /[nrtbfu"/]/.test(text[start] ?? ""))
+        start -= 2;
+    const e = escapeAt(text, end);
+    if (e)
+        end = Math.max(start, e[0]);
+    // A token goes on through an escape that spells one of its characters (`…zdp7d\x63` in a shell's `$'…'`).
+    for (let n = 0; n < MAX_SPELLED && end > start && /[\w-]/.test(text[end - 1]); n++) {
+        SPELLED_CHAR.lastIndex = end;
+        if (backslashesBefore(text, end) % 2 || !SPELLED_CHAR.test(text))
+            break;
+        end = SPELLED_CHAR.lastIndex;
+    }
+    return start === f.start && end === f.end ? f : { ...f, start, end, value: undefined };
+}
+/**
+ * A whole line taken as a secret (a one-line secret file, a command's printed line) that holds a confident
+ * finding of another rule (`export TOKEN="ghp_…"`, a URL with a token) gives way to it: the name and URL stay, and the
+ * placeholder's tag is the secret's own, the same as wherever else it appears.
+ */
+function preferNarrower(findings) {
+    if (!findings.some((f) => isWholeLine(f.rule)))
+        return findings;
+    const narrow = findings
+        .filter((f) => !isWholeLine(f.rule) && f.confidence >= 0.9 && !isGuess(f.rule) && layerOf(f.rule) !== "repeat")
+        .sort((a, b) => a.start - b.start);
+    const starts = narrow.map((f) => f.start);
+    return findings.filter((w) => {
+        if (!isWholeLine(w.rule))
+            return true;
+        // (The narrow findings starting inside the line, by binary search; one that ends inside it too. One that is the
+        // whole line names it better: `cat .vault_pass` printing a GitHub token.)
+        for (let i = lowerBound(starts, w.start); i < narrow.length && narrow[i].start < w.end; i++) {
+            if (narrow[i].end <= w.end)
+                return false;
+        }
+        return true;
+    });
+}
+/** Overlapping findings as one span each, every field but the span the most confident one's. */
+function union(findings) {
+    const sorted = [...findings].sort((a, b) => a.start - b.start || b.end - a.end);
+    const out = [];
+    for (const f of sorted) {
+        if (f.end <= f.start)
+            continue;
+        const last = out.at(-1);
+        if (!last || f.start >= last.end) {
+            out.push({ ...f });
+            continue;
+        }
+        const end = Math.max(last.end, f.end);
+        const winner = f.confidence > last.confidence ? f : last;
+        // (A value goes with its own span only: `last` may start earlier or `f` end later than the winner.)
+        const widened = last.start !== winner.start || end !== winner.end;
+        out[out.length - 1] = { ...winner, start: last.start, end, value: widened ? undefined : winner.value };
+    }
+    return out;
+}
+/** A character that may go on a value cut short: not a blank or a quote. */
+const VALUE_CHAR = /[^\s"'`]/;
+/**
+ * How long a finding is that a read bound may have cut: half of UNQUOTED_READ, as a reader's bound counts from where it
+ * started reading (a command's first word, a line's start), not from the value.
+ */
+const MAY_BE_CUT = UNQUOTED_READ / 2;
+/**
+ * A finding that may have been cut by a read bound (MAY_BE_CUT) and stops inside a word runs to the word's end, a blank
+ * or a quote: a longer secret must not leave its tail. A shorter one ends where its reader meant it to (a URL's
+ * password at its `@`). (A finding whose recorded value differs from its span, decoded or joined, is left as it is.)
+ */
+export function toWordEnd(text, f) {
+    if (f.end - f.start < MAY_BE_CUT || f.value !== undefined || !VALUE_CHAR.test(text[f.end - 1]))
+        return f;
+    const end = wordEnds(text)[f.end];
+    return end === f.end ? f : { ...f, end };
+}
+/**
+ * For each position of a text, where the word (of VALUE_CHAR) it is in ends, an escape (`\n`, `\"`) ending one: built
+ * once a text, so many findings in one long word cost one scan.
+ */
+const wordEnds = lastText((text) => {
+    const ends = new Int32Array(text.length + 1);
+    ends[text.length] = text.length;
+    for (let i = text.length - 1; i >= 0; i--)
+        ends[i] = VALUE_CHAR.test(text[i]) && !(text[i] === "\\" && /[nrt"]/.test(text[i + 1] ?? "")) ? ends[i + 1] : i;
+    return ends;
+});
+/**
+ * A finding trimmed to its value, of what is syntax beyond doubt: blanks, an `=` or `:` with a blank or a quote after it,
+ * and a pair of quotes around the value (`= "CANYxh7686!"`); a comma or a semicolon before the next `name=` of a list
+ * (`passwort=COBAlyrxgo%, knownHosts=…`). A password's own first or last character (`:Kx9…`, `Kx9…)`, `Kx9…;`) stays.
+ * A placeholder then replaces the value only, so the syntax around it stays; and the value is what's fingerprinted and
+ * learned. (A finding that records its value, decoded or joined, is that value already.)
+ */
+export function trimmedToValue(text, f) {
+    if (f.value !== undefined)
+        return f;
+    let { start, end } = f;
+    const lead = /^[ \t]*(?:[=:](?:[ \t]+|(?=["'`])))?/.exec(text.slice(start, Math.min(end, start + 40)))[0].length;
+    start += lead;
+    const q = text[start];
+    if (/["'`]/.test(q ?? "") && end - start > 2 && text[end - 1] === q && !text.slice(start + 1, end - 1).includes(q)) {
+        start++;
+        end--;
+    }
+    if (end - start > 1 && /[,;]/.test(text[end - 1]) && /^[ \t]+[A-Za-z_][\w.-]*=/.test(text.slice(end, end + 80)))
+        end--;
+    return start === f.start && end === f.end ? f : { ...f, start, end };
+}
+/** Findings merged into one span each, sorted; with `text`, kept off escape sequences. (The inputs aren't changed.) */
+export function merge(findings, text) {
+    return union(preferNarrower(text ? findings.map((f) => onEscapes(text, f)) : findings));
+}
+/** The first of `sorted` (merged, sorted by start) that ends after `start`, by binary search. */
+function firstEndingAfter(sorted, start) {
+    let lo = 0;
+    let hi = sorted.length;
+    while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (sorted[mid].end <= start)
+            lo = mid + 1;
+        else
+            hi = mid;
+    }
+    return lo;
+}
+/** Whether [start, end) lies inside one of `sorted` (merged, sorted by start). */
+export function covered(sorted, start, end) {
+    const f = sorted[firstEndingAfter(sorted, start)];
+    return !!f && f.start <= start && f.end >= end;
+}
+/** Whether [start, end) overlaps one of `sorted` (merged, sorted by start). */
+export function overlaps(sorted, start, end) {
+    const f = sorted[firstEndingAfter(sorted, start)];
+    return !!f && f.start < end;
+}
+/** Terminal widths output is commonly wrapped at. */
+const WRAP_WIDTHS = new Set([64, 76, 79, 80, 100, 120, 132]);
+/**
+ * A token cut by a terminal or log viewer wrapping at a fixed width (`Authorization: Bearer eyJ…` at
+ * column 80): a finding that ends exactly at the end of a line of a common wrap width continues on
+ * the following lines while they are bare token characters, up to the first shorter line.
+ */
+export function wrapped(text, found) {
+    return found.map((f) => {
+        if (!continuesAcrossWraps(f.rule))
+            return f;
+        let end = f.end;
+        const width = end - lineStart(text, end, 400);
+        if (!WRAP_WIDTHS.has(width))
+            return f;
+        for (let n = 0; n < 100 && (text[end] === "\n" || (text[end] === "\r" && text[end + 1] === "\n")); n++) {
+            const from = end + (text[end] === "\r" ? 2 : 1);
+            const to = lineEnd(text, from, width + 1);
+            const line = text.slice(from, to).replace(/\r$/, "");
+            if (line.length < 8 ||
+                line.length > width ||
+                !/^[A-Za-z0-9._~+/-]+={0,2}$/.test(line) ||
+                !/\d/.test(line) ||
+                !/[A-Za-z]/.test(line) ||
+                naturalness(line) > 0.5)
+                break;
+            end = from + line.length;
+            if (line.length < width)
+                break;
+        }
+        return end === f.end ? f : { ...f, end };
+    });
+}

package/dist/sql.d.ts

@@ -0,0 +1,20 @@
+/**
+ * SQL `INSERT … VALUES` statements read into cells: each row's values under their table and column. One reader for
+ * every rule that asks what a value in a row is: the name layer hides a credential column's values (named.ts sqlRows),
+ * the random layer keeps the other columns' IDs and data (context.ts, `sqlCell`). A statement without a column list
+ * (`INSERT INTO t VALUES …`, as mysqldump writes by default) names no column, so it isn't read.
+ */
+import { type Argument } from "./calls.ts";
+export interface SqlCell {
+    /** The table, without its schema or quotes: `users` in `` `app`.`users` ``. */
+    table: string;
+    /** The column, without quotes: `password` in `"password"`, `` `password` ``, `[password]`. */
+    column: string;
+    /** The value as an argument of the row: a string literal's content is its `literal`. */
+    value: Argument;
+}
+/**
+ * The cells of every `INSERT` in `text`, row by row (up to 10,000 rows a statement); with `wants`, only of the
+ * statements whose table and columns it accepts (so a dump's other tables cost no reading).
+ */
+export declare function insertCells(text: string, wants?: (table: string, columns: string[]) => boolean): Generator<SqlCell>;

package/dist/sql.js

@@ -0,0 +1,66 @@
+/**
+ * SQL `INSERT … VALUES` statements read into cells: each row's values under their table and column. One reader for
+ * every rule that asks what a value in a row is: the name layer hides a credential column's values (named.ts sqlRows),
+ * the random layer keeps the other columns' IDs and data (context.ts, `sqlCell`). A statement without a column list
+ * (`INSERT INTO t VALUES …`, as mysqldump writes by default) names no column, so it isn't read.
+ */
+import { callArguments, closingBracket } from "./calls.js";
+import { delimitedCells } from "./syntax.js";
+/** An identifier as SQL dialects quote it: `"name"`, `` `name` ``, `[name]`. */
+const unquote = (name) => name.trim().replace(/^["`[]|["`\]]$/g, "");
+/**
+ * `INSERT [INTO] <table> (<columns>) VALUES`, and the dialects' other forms (`INSERT IGNORE`, `INSERT OR REPLACE`,
+ * `REPLACE INTO`), the table quoted as any dialect quotes it (`"my table"`, `` `app`.`users` ``, `[dbo].[users]`).
+ */
+const INSERT = /\b(?:INSERT(?:\s+IGNORE|\s+OR\s+(?:REPLACE|IGNORE|ABORT|FAIL|ROLLBACK))?|REPLACE)\s+(?:INTO\s+)?((?:"[^"\n]{1,100}"|`[^`\n]{1,100}`|\[[^\]\n]{1,100}\]|[\w.]){1,200})\s*\(([^()]{1,2000})\)\s*VALUES\s*/gi;
+/** What may stand between two rows: blanks, comments and one comma. */
+const BETWEEN_ROWS = /(?:\s|--[^\n]*|\/\*[\s\S]{0,2000}?\*\/)*,?(?:\s|--[^\n]*|\/\*[\s\S]{0,2000}?\*\/)*/y;
+/**
+ * The cells of every `INSERT` in `text`, row by row (up to 10,000 rows a statement); with `wants`, only of the
+ * statements whose table and columns it accepts (so a dump's other tables cost no reading).
+ */
+export function* insertCells(text, wants) {
+    if (!/\b(?:INSERT|REPLACE)\b/i.test(text))
+        return;
+    for (const m of text.matchAll(INSERT)) {
+        const table = unquote(m[1].split(".").at(-1));
+        const columns = delimitedCells(m[2], ",").map((c) => unquote(c.text));
+        if (wants && !wants(table, columns))
+            continue;
+        const at = m.index + m[0].length;
+        // (A statement a program holds in a string, `cur.execute("INSERT … VALUES ('…')")`: the string's brackets aren't
+        // read as brackets, so its line is read as the statement on its own.)
+        if (text[at] === "(" && closingBracket(text, at + 1) < 0) {
+            const end = text.indexOf("\n", at);
+            const line = text.slice(m.index, end < 0 ? undefined : end);
+            for (const cell of rowCells(line, at - m.index, table, columns))
+                yield { ...cell, value: shifted(cell.value, m.index) };
+        }
+        else
+            yield* rowCells(text, at, table, columns);
+    }
+}
+/** The cells of the rows that start at `at` (`(…), (…)`), each value under its column. */
+function* rowCells(text, at, table, columns) {
+    for (let row = 0; row < 10_000 && text[at] === "("; row++) {
+        // (A row cut short, a dump's `head`, ends the rows. callArguments reads to the bracket closingBracket finds.)
+        const values = callArguments(text, at + 1);
+        if (!values)
+            break;
+        const close = closingBracket(text, at + 1);
+        for (const [k, value] of values.entries())
+            if (columns[k])
+                yield { table, column: columns[k], value };
+        BETWEEN_ROWS.lastIndex = close + 1;
+        at = close + 1 + BETWEEN_ROWS.exec(text)[0].length;
+    }
+}
+/** An argument read from a piece of a text, placed where the piece starts in it. */
+function shifted(a, by) {
+    return {
+        ...a,
+        start: a.start + by,
+        end: a.end + by,
+        ...(a.literal ? { literal: { ...a.literal, start: a.literal.start + by, end: a.literal.end + by } } : {}),
+    };
+}

package/dist/stream.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Redaction of text that arrives in chunks (a tool's output, a model's response): each chunk is held until it is
+ * far enough behind the newest one that no secret can still be forming across it, then released redacted.
+ */
+import type { Redactor } from "./index.ts";
+import type { DetectOptions } from "./types.ts";
+export declare class RedactionStream {
+    /** How far behind the newest text a release point must be: 8 KB covers the longest single secrets (a private key). */
+    private readonly holdback;
+    /** Text not released yet. */
+    private buffer;
+    /** Text released already, its last CONTEXT characters. */
+    private context;
+    /**
+     * The lines that open the scopes still open in the released text (secretfile.ts openScopes, structure.ts
+     * openManifestData): kept in front of the context however far back they are, so the text after them is read as
+     * it would be in the whole text (that command's output, that heredoc's body, that Secret's data).
+     */
+    private scopes;
+    /** While text is held (MAX_HELD), how long the buffer grows before it's looked at again. */
+    private heldUntil;
+    /** Positions in the whole stream: where `buffer` starts, and where the last value hidden ends. */
+    private released;
+    private hiddenUntil;
+    /** Values hidden earlier in the stream, looked for again however far back they were named, as `redact()` sees the whole text. */
+    private remembered;
+    private readonly redactor;
+    private readonly options;
+    constructor(redactor: Redactor, options: DetectOptions & {
+        holdback?: number;
+    });
+    write(chunk: string): string;
+    end(): string;
+    /** What scanning sees in front of the buffer: the open scopes' lines that have scrolled out of it, and the released context. */
+    private head;
+    /**
+     * Where to release up to: a line break at least `holdback` from the end, so a line is judged whole (a JSONL record,
+     * a base64 blob); 0 to hold an unfinished line. Past MAX_HELD, the last blank or delimiter, or the holdback itself;
+     * never inside a surrogate pair.
+     */
+    private releasePoint;
+    private flush;
+    private remember;
+    /** Moves the first `cut` characters of the buffer into the released context. */
+    private advance;
+}

package/dist/stream.js

@@ -0,0 +1,150 @@
+import { occurrences, repeatable } from "./repeats.js";
+import { openScopes } from "./secretfile.js";
+import { openManifestData } from "./structure.js";
+import { needsNormalizing, normalize } from "./normalize.js";
+import { merge, secretValue } from "./spans.js";
+/** Text already released kept as left context (the line naming a value, the `$ cat .env` above it), scanned again but never released again. */
+const CONTEXT = 8_192;
+/** Values hidden earlier in the stream that are looked for again, the most recent first. */
+const REMEMBERED = 2_000;
+/** How much text is held back while what it is waits on a line still to come (a manifest's `kind` after its `data:`). */
+const MAX_HELD = 1_000_000;
+export class RedactionStream {
+    /** How far behind the newest text a release point must be: 8 KB covers the longest single secrets (a private key). */
+    holdback;
+    /** Text not released yet. */
+    buffer = "";
+    /** Text released already, its last CONTEXT characters. */
+    context = "";
+    /**
+     * The lines that open the scopes still open in the released text (secretfile.ts openScopes, structure.ts
+     * openManifestData): kept in front of the context however far back they are, so the text after them is read as
+     * it would be in the whole text (that command's output, that heredoc's body, that Secret's data).
+     */
+    scopes = [];
+    /** While text is held (MAX_HELD), how long the buffer grows before it's looked at again. */
+    heldUntil = 0;
+    /** Positions in the whole stream: where `buffer` starts, and where the last value hidden ends. */
+    released = 0;
+    hiddenUntil = 0;
+    /** Values hidden earlier in the stream, looked for again however far back they were named, as `redact()` sees the whole text. */
+    remembered = new Map();
+    redactor;
+    options;
+    constructor(redactor, options) {
+        this.redactor = redactor;
+        this.options = { hint: options.hint, guessing: options.guessing };
+        this.holdback = Math.max(8_192, options.holdback ?? 32_768);
+    }
+    write(chunk) {
+        this.buffer += chunk;
+        // Scanned once the buffer has twice the holdback, so each character is scanned about twice.
+        return this.buffer.length < Math.max(this.holdback * 2, this.heldUntil) ? "" : this.flush(false);
+    }
+    end() {
+        return this.flush(true);
+    }
+    /** What scanning sees in front of the buffer: the open scopes' lines that have scrolled out of it, and the released context. */
+    head() {
+        // (Each scope line ends in its line break, so it matches only a whole line of the context, never the start of a longer one.)
+        return this.scopes.filter((line) => !`\n${this.context}`.includes(`\n${line}`)).join("") + this.context;
+    }
+    /**
+     * Where to release up to: a line break at least `holdback` from the end, so a line is judged whole (a JSONL record,
+     * a base64 blob); 0 to hold an unfinished line. Past MAX_HELD, the last blank or delimiter, or the holdback itself;
+     * never inside a surrogate pair.
+     */
+    releasePoint(all) {
+        const buffer = this.buffer;
+        if (all)
+            return buffer.length;
+        let cut = buffer.lastIndexOf("\n", buffer.length - this.holdback) + 1;
+        if (cut <= 0 && buffer.length < MAX_HELD)
+            return 0;
+        if (cut <= 0) {
+            const limit = buffer.length - this.holdback;
+            cut =
+                Math.max(buffer.lastIndexOf(" ", limit), buffer.lastIndexOf(",", limit), buffer.lastIndexOf(";", limit)) + 1;
+            if (cut <= 0)
+                cut = limit;
+        }
+        if (cut > 0 && isHighSurrogate(buffer.charCodeAt(cut - 1)))
+            cut--;
+        return cut;
+    }
+    flush(all) {
+        const head = this.head();
+        const left = head.length;
+        // (Up to the buffer's last line break, but at the end: no detector sees a line cut short that the whole text wouldn't.)
+        const lineEnd = all ? -1 : this.buffer.lastIndexOf("\n");
+        const whole = head + (lineEnd < 0 ? this.buffer : this.buffer.slice(0, lineEnd + 1));
+        // An unfinished line, and a Secret's data whose `kind` hasn't come yet, are held until they are whole.
+        let cut = this.releasePoint(all);
+        if (cut <= 0 || (!all && this.buffer.length < MAX_HELD && openManifestData(whole)?.pending)) {
+            this.heldUntil = this.buffer.length + this.holdback;
+            return "";
+        }
+        this.heldUntil = 0;
+        const detected = this.redactor.detect(whole, this.options);
+        const findings = (this.remembered.size ? merge([...detected, ...occurrences(whole, this.remembered, detected)], whole) : detected)
+            .filter((f) => f.end > left)
+            .map((f) => ({ ...f, start: f.start - left, end: f.end - left, value: secretValue(whole, f) }));
+        // A finding across the cut is released whole later, unless it started before this buffer (a blob longer than
+        // the holdback): that one is hidden now, and its continuation dropped as it arrives, while its start is still in
+        // the context read before the buffer (CONTEXT; a single secret longer than that, which no key is, shows its tail).
+        for (const f of findings)
+            if (f.start < cut && f.end > cut && f.start > 0)
+                cut = f.start;
+        let out = "";
+        let at = 0;
+        for (const f of findings) {
+            if (f.start >= cut)
+                break;
+            // (What replaces it may be wider than the finding: a JSON number's whole token, quoted.)
+            const r = this.redactor.replacement(whole, { ...f, start: f.start + left, end: f.end + left }, f.value);
+            const end = Math.max(f.end, r.end - left);
+            out += this.buffer.slice(at, Math.max(at, r.start - left));
+            // The rest of a value already replaced by a placeholder adds nothing; a value that began in text already
+            // released, found only now, is hidden from here on. (Judged in the whole scanned text, so a JSON key
+            // released before still counts.)
+            if (this.released + f.start >= this.hiddenUntil)
+                out += r.text;
+            this.hiddenUntil = Math.max(this.hiddenUntil, this.released + end);
+            at = Math.min(end, cut);
+        }
+        out += this.buffer.slice(at, cut);
+        this.remember(repeatable(whole, detected.filter((f) => f.start >= left && f.start - left < cut)));
+        this.advance(cut);
+        return out;
+    }
+    remember(values) {
+        for (const [form, r] of values) {
+            this.remembered.delete(form);
+            this.remembered.set(form, r);
+            if (this.remembered.size > REMEMBERED)
+                this.remembered.delete(this.remembered.keys().next().value);
+        }
+    }
+    /** Moves the first `cut` characters of the buffer into the released context. */
+    advance(cut) {
+        const out = this.buffer.slice(0, cut);
+        // (Read as detection reads it: normalized, colours and line continuations gone, a diff's markers too.)
+        const raw = this.head() + out;
+        const seen = needsNormalizing(raw) ? normalize(raw).text : raw;
+        this.scopes = [...openScopes(seen), ...(openManifestData(seen)?.lines ?? [])];
+        // The context starts at a line's start, as the whole text's lines do (a piece of a line would read as one).
+        const joined = this.context + out;
+        this.context = joined.slice(-CONTEXT);
+        if (joined.length > CONTEXT && joined[joined.length - CONTEXT - 1] !== "\n") {
+            const nl = this.context.indexOf("\n");
+            if (nl >= 0)
+                this.context = this.context.slice(nl + 1);
+        }
+        if (isLowSurrogate(this.context.charCodeAt(0)))
+            this.context = this.context.slice(1);
+        this.buffer = this.buffer.slice(cut);
+        this.released += cut;
+    }
+}
+const isHighSurrogate = (c) => c >= 0xd800 && c <= 0xdbff;
+const isLowSurrogate = (c) => c >= 0xdc00 && c <= 0xdfff;

package/dist/structure.d.ts

@@ -0,0 +1,24 @@
+import type { Finding } from "./types.ts";
+/**
+ * Whether a URL's password field (or a command line's password) holds a password rather than a variable, format
+ * specifier or placeholder. `quoted`: the value sat in single quotes, where a shell expands nothing (`-p'$mIWy…'`
+ * is a literal).
+ */
+export declare function isLiteralPassword(value: string, quoted?: boolean): boolean;
+/** HTTP Basic credentials whose password is a docs placeholder (`user:password`, `username:[password]`). */
+export declare function isPlaceholderBasic(b64: string): boolean;
+/** Whether a cookie holds something worth hiding: not an attribute, a flag, a number, or an analytics or preference cookie. */
+export declare function isSessionCookie(name: string, value: string): boolean;
+/**
+ * A manifest's `data:` or `stringData:` mapping still open at the end of a text (no line has come back to its column),
+ * by the lines that open it: its own and its `kind` (a Secret's, or a ConfigMap's), at its column. `pending`: its
+ * `kind` hasn't come yet (kubectl prints `data:` first), so what follows can't be judged until it does.
+ */
+export declare function openManifestData(text: string): {
+    lines: string[];
+    pending: boolean;
+} | undefined;
+/** Everything in this module. */
+export declare function findStructure(text: string): Finding[];
+/** Whether a text is a published BIP-39 phrase (see isPublishedPhrase): 12 to 24 list words whose checksum verifies. */
+export declare function isPublishedMnemonic(text: string): boolean;