sealkeep 0.5.0

package/dist/src/leakscan.js

@@ -0,0 +1,222 @@
+/**
+ * Seal-time secret detection. Deterministic, offline, dependency-free.
+ *
+ * Agent transcripts are where pasted credentials go to be forgotten: an AWS
+ * key dropped into a prompt in March is still in the archive in November, and
+ * a sealed archive keeps it faithfully forever. Detection has to happen here,
+ * on this machine, at seal time — E2EE means nothing downstream (bucket,
+ * control plane, org dashboard) can ever look at the plaintext, so a scanner
+ * that lives anywhere else would need the very access the product promises
+ * away.
+ *
+ * This module is detection only, in the gitleaks/trufflehog family: fixed
+ * patterns plus a Shannon-entropy fallback. No AI, no network, no config —
+ * the same bytes always produce the same findings, which is what lets the
+ * exit code gate a script.
+ *
+ * Phase 2 — integration points, deliberately not implemented here:
+ *
+ * (a) Worker seal path: worker.ts calls scanFileStreaming(job.sourcePath)
+ *     just before sealing and stores the findings — kind, severity, line,
+ *     masked preview — as metadata beside the archive record, so "which
+ *     archives hold live credentials?" is answerable without opening any
+ *     of them.
+ * (b) Org rollup: the control plane receives kind/severity/counts only —
+ *     never content, never lines, never previews. Because only this machine
+ *     ever holds plaintext, an org can learn *that* a credential was sealed
+ *     and *what sort*, but the credential itself cannot travel.
+ */
+import { createReadStream } from "node:fs";
+import { readdir, stat } from "node:fs/promises";
+import { join } from "node:path";
+import { createInterface } from "node:readline";
+import { fail } from "./errors.js";
+/**
+ * First four characters, then only the length. A leak report that repeats the
+ * leak is a second leak: findings travel further than the transcript did —
+ * terminals, CI logs, screenshots, phase-2 metadata — so the preview carries
+ * just enough to locate the match and never enough to use it.
+ */
+export function maskSecret(secret) {
+    return `${secret.slice(0, 4)}…(${secret.length} chars)`;
+}
+/** Character-frequency Shannon entropy, in bits per character. */
+export function shannonEntropy(value) {
+    const chars = [...value];
+    if (chars.length === 0)
+        return 0;
+    const counts = new Map();
+    for (const char of chars)
+        counts.set(char, (counts.get(char) ?? 0) + 1);
+    let entropy = 0;
+    for (const count of counts.values()) {
+        const p = count / chars.length;
+        entropy -= p * Math.log2(p);
+    }
+    return entropy;
+}
+/** A JWT is three base64url segments, but so is a lot of noise. The header must decode to JSON naming an alg. */
+function plausibleJwtHeader(token) {
+    try {
+        const header = JSON.parse(Buffer.from(token.slice(0, token.indexOf(".")), "base64url").toString("utf8"));
+        return typeof header === "object" && header !== null && typeof header.alg === "string";
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Pattern detectors: shapes only a real credential takes. Every one is
+ * anchored on both sides (a 17th character disqualifies an AWS key id rather
+ * than truncating it) because precision is the whole game — false positives
+ * are how people come to disable scanners, and a disabled scanner catches
+ * nothing. A hit here is near-certain, so these report "high".
+ *
+ * Transcripts are jsonl, so quotes often arrive escaped (`key=\"value\"`);
+ * the assignment-context patterns accept the backslash.
+ */
+const PATTERN_DETECTORS = [
+    // AKIA (long-lived) / ASIA (temporary) + exactly 16 more upper-alnum.
+    { kind: "aws-access-key-id", pattern: /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/dg },
+    // The 40-char base64-ish value alone is far too common a shape, so it only
+    // counts inside an assignment whose name says "secret".
+    {
+        kind: "aws-secret-access-key",
+        pattern: /[A-Za-z0-9_.-]*secret[A-Za-z0-9_.-]*(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?([A-Za-z0-9/+]{40})(?![A-Za-z0-9/+=])/dgi,
+        group: 1
+    },
+    { kind: "github-token", pattern: /\b(?:(?:ghp|gho|ghs)_[A-Za-z0-9]{36,251}|github_pat_[A-Za-z0-9_]{22,255})\b/dg },
+    { kind: "google-api-key", pattern: /(?<![0-9A-Za-z_-])AIza[0-9A-Za-z_-]{35}(?![0-9A-Za-z_-])/dg },
+    { kind: "slack-token", pattern: /\bxox[bpars]-[A-Za-z0-9][A-Za-z0-9-]{9,}(?![A-Za-z0-9-])/dg },
+    // The armor header is proof enough; the body may be split across lines, but
+    // no innocent text starts a line of itself this way.
+    { kind: "private-key", pattern: /-----BEGIN (?:(?:RSA|EC|OPENSSH|PGP|ENCRYPTED) )?PRIVATE KEY(?: BLOCK)?-----/dg },
+    {
+        kind: "jwt",
+        pattern: /\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{10,}(?![A-Za-z0-9_.-])/dg,
+        validate: plausibleJwtHeader
+    }
+];
+/** Tests derive their vectors from these rather than restating the numbers. */
+export const GENERIC_MIN_LENGTH = 24;
+export const GENERIC_ENTROPY_THRESHOLD = 4.2;
+/**
+ * The generic fallback: something named like a credential, assigned a value
+ * long and dense enough that prose and identifiers cannot reach it. This is
+ * where scanners lose their users, so it is gated three ways — assignment
+ * context, length + entropy, and an allowlist of known-benign shapes — and
+ * even then it reports only "medium": pattern hits are certainties, entropy
+ * hits are suspicions, and the exit code (phase 1) and seal metadata
+ * (phase 2) treat them differently.
+ */
+const GENERIC_ASSIGNMENT = new RegExp(String.raw `[A-Za-z0-9_.-]*(?:key|token|secret|password|passwd|pwd)[A-Za-z0-9_.-]*(?:\\?["'])?\s*[:=]\s*(?:\\?["'])?([A-Za-z0-9+/=_-]{${GENERIC_MIN_LENGTH},})(?![A-Za-z0-9+/=_-])`, "dgi");
+/**
+ * Benign shapes that clear the length bar and would otherwise page someone.
+ * Pure hex (and UUIDs) cannot actually exceed 4.2 bits/char, but relying on
+ * that emergent arithmetic would make the next threshold change a regression;
+ * the allowlist states the intent.
+ */
+const BENIGN_VALUE_SHAPES = [
+    /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i, // UUIDs: session ids, request ids, vault ids
+    /^[0-9a-f]{32,}$/i, // hex digests (sha256 and friends): integrity data, not credentials
+    /^AAAA/, // base64 of leading zero bytes: well-known placeholders and ssh key headers
+    /^(?:vlt|vault)_/i // vaultline's own refs (vlt_…, vault_ref ids), which are public by design
+];
+function scanLine(line, lineNumber) {
+    const findings = [];
+    const claimed = [];
+    for (const detector of PATTERN_DETECTORS) {
+        for (const match of line.matchAll(detector.pattern)) {
+            const span = match.indices?.[detector.group ?? 0];
+            if (!span)
+                continue;
+            const secret = line.slice(span[0], span[1]);
+            if (detector.validate && !detector.validate(secret))
+                continue;
+            claimed.push(span);
+            findings.push({ kind: detector.kind, line: lineNumber, column: span[0] + 1, preview: maskSecret(secret), severity: "high" });
+        }
+    }
+    for (const match of line.matchAll(GENERIC_ASSIGNMENT)) {
+        const span = match.indices?.[1];
+        if (!span)
+            continue;
+        // A secret a precise detector already named is one finding, not two.
+        if (claimed.some(([start, end]) => span[0] < end && start < span[1]))
+            continue;
+        const value = line.slice(span[0], span[1]);
+        if (BENIGN_VALUE_SHAPES.some((shape) => shape.test(value)))
+            continue;
+        if (shannonEntropy(value) <= GENERIC_ENTROPY_THRESHOLD)
+            continue;
+        findings.push({ kind: "high-entropy-value", line: lineNumber, column: span[0] + 1, preview: maskSecret(value), severity: "medium" });
+    }
+    return findings.sort((a, b) => a.column - b.column || a.kind.localeCompare(b.kind));
+}
+export function scanText(text) {
+    return text.split(/\r?\n/).flatMap((line, index) => scanLine(line, index + 1));
+}
+/**
+ * Line-by-line over a read stream: memory is bounded by the longest single
+ * line, never by the file, so a multi-GB jsonl scans in the same footprint as
+ * a small one. An unreadable input rejects — a scanner that swallows a read
+ * error would be reporting "clean" about bytes it never saw.
+ */
+export async function scanFileStreaming(path, onFinding) {
+    const input = createReadStream(path, { encoding: "utf8" });
+    const lines = createInterface({ input, crlfDelay: Infinity });
+    const findings = [];
+    let lineNumber = 0;
+    try {
+        for await (const line of lines) {
+            lineNumber += 1;
+            for (const finding of scanLine(line, lineNumber)) {
+                findings.push(finding);
+                onFinding?.(finding);
+            }
+        }
+    }
+    finally {
+        lines.close();
+        input.close();
+    }
+    return findings;
+}
+/**
+ * Every *.jsonl under a directory, depth-first, sorted at each level so a
+ * report is comparable run to run. Symlinks are left alone: transcript trees
+ * do not need them and following them invites cycles.
+ */
+async function jsonlFiles(dir) {
+    const entries = await readdir(dir, { withFileTypes: true });
+    const found = [];
+    for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+        const path = join(dir, entry.name);
+        if (entry.isDirectory())
+            found.push(...(await jsonlFiles(path)));
+        else if (entry.isFile() && entry.name.endsWith(".jsonl"))
+            found.push(path);
+    }
+    return found;
+}
+/**
+ * A file is scanned as pointed at, whatever its name; a directory means every
+ * *.jsonl beneath it. Only high findings set the exit code — failing a script
+ * over a "medium" suspicion is exactly the cry-wolf behaviour the severity
+ * split exists to avoid.
+ */
+export async function scanPath(target, onFinding) {
+    const info = await stat(target).catch(() => fail("source_unreadable", `Cannot read ${target}`));
+    const files = info.isDirectory() ? await jsonlFiles(target) : [target];
+    const findings = [];
+    for (const file of files) {
+        await scanFileStreaming(file, (finding) => {
+            const placed = { ...finding, file };
+            findings.push(placed);
+            onFinding?.(placed);
+        });
+    }
+    const high = findings.filter((finding) => finding.severity === "high").length;
+    const medium = findings.length - high;
+    return { files: files.length, findings, high, medium, exitCode: high > 0 ? 1 : 0 };
+}

package/dist/src/local-api.d.ts

@@ -0,0 +1,132 @@
+import { type Server } from "node:http";
+import { z } from "zod";
+import { type AgentId } from "./adapters.js";
+import { ArchiveQueue } from "./queue.js";
+import { localApiTokenPath } from "./paths.js";
+import { type ServiceOptions } from "./service.js";
+declare const setupStorageSchema: z.ZodObject<{
+    provider: z.ZodEnum<{
+        s3: "s3";
+        r2: "r2";
+        b2: "b2";
+        gcs: "gcs";
+        gdrive: "gdrive";
+    }>;
+    bucket: z.ZodString;
+    prefix: z.ZodDefault<z.ZodString>;
+    region: z.ZodOptional<z.ZodString>;
+    endpoint: z.ZodOptional<z.ZodString>;
+    credentials: z.ZodUnion<readonly [z.ZodObject<{
+        accessKeyId: z.ZodString;
+        secretAccessKey: z.ZodString;
+        sessionToken: z.ZodOptional<z.ZodString>;
+    }, z.core.$strict>, z.ZodObject<{
+        clientEmail: z.ZodString;
+        privateKey: z.ZodString;
+    }, z.core.$strict>]>;
+    confirm: z.ZodDefault<z.ZodBoolean>;
+}, z.core.$strict>;
+/**
+ * Proves a storage target works, before anyone trusts it with an archive.
+ *
+ * Credentials are the step of setup most likely to be wrong, and every way of
+ * being wrong used to surface at the same moment: hours later, in a background
+ * upload, as a failure nobody was watching. The bucket name has a typo, the key
+ * lacks PutObject, the region is the console's display name rather than its id,
+ * the B2 key is a native one rather than S3-compatible. All of that is knowable
+ * in a second while the person is still looking at the form.
+ *
+ * It writes a small object, reads it back, and compares the bytes, because
+ * anything less proves less: a successful PUT says nothing about whether GET is
+ * permitted, and this product's whole promise is that the bytes come back.
+ *
+ * Nothing is stored. The credential is used to build a client, held for the
+ * round trip, and dropped — the caller has not committed to this target yet,
+ * and a test that saved a broken configuration would be worse than no test.
+ */
+export declare function testStorageTarget(input: z.infer<typeof setupStorageSchema>): Promise<{
+    ok: boolean;
+    detail: string;
+    probeKey?: string;
+}>;
+export type SessionRow = {
+    agent: AgentId;
+    path: string;
+    bytes: number;
+    modifiedAt: string;
+    /** True when an archive already covers this file at exactly this length. */
+    archived: boolean;
+    /** True when that archive also has a verified copy in storage — off this machine, not just sealed on it. */
+    stored: boolean;
+    /** True while a queued or in-flight job exists for this file: it is being sealed and must not be selected again. */
+    queued: boolean;
+    /** Why a queued job is waiting instead of running — today, the disk guard's message. Null when it is genuinely being worked. A held row saying "sealing…" was a lie the queue knew better than. */
+    held: string | null;
+    /** Whole percentages of the source read so far while sealing, else null. The number a person actually watches on a 1.6 GB row. */
+    sealingPct: number | null;
+    /** The project this session ran in, when its metadata says — the Sessions page filters and labels on it. Null when nothing says. */
+    project: string | null;
+    /** The working directory behind `project`, best-effort decoded; a tooltip, never a filter. */
+    projectPath: string | null;
+};
+/**
+ * Every session on this machine, with the two facts a person needs to choose:
+ * how big it is, and when they last touched it.
+ *
+ * This exists because archiving everything automatically is the wrong default
+ * for the situation people are actually in — a nearly full disk, a handful of
+ * enormous transcripts, and no appetite for a background process deciding which
+ * ones to work through. Choosing is cheap when you can see sizes and dates;
+ * guessing is not.
+ *
+ * Already-archived sessions are marked rather than hidden, because "this one is
+ * done" is as useful as "this one is not", and hiding them would make the list
+ * shrink as you work and lose your place.
+ */
+export declare function listSessions(dataDir: string, queue?: ArchiveQueue): Promise<{
+    sessions: SessionRow[];
+    totalBytes: number;
+    unarchivedBytes: number;
+}>;
+export declare function ensureDraining(dataDir: string, queue: ArchiveQueue): Promise<void>;
+export declare function archiveSelected(dataDir: string, paths: string[], queue: ArchiveQueue): Promise<{
+    requested: number;
+    queued: number;
+    bytes: number;
+    background: boolean;
+    results: {
+        path: string;
+        ok: boolean;
+        bytes?: number;
+        error?: string;
+    }[];
+}>;
+export { localApiTokenPath };
+/** Mints the loopback token once and keeps it owner-readable. It is never logged or returned by an endpoint. */
+export declare function localApiToken(dataDir: string): Promise<string>;
+/** One row of GET /v1/cloud/copies: a copy held off this machine, wherever it is held. */
+export type StoredCopy = {
+    ref: string;
+    bytes: number;
+    addedAt: string | null;
+    state: string;
+    /** True when this machine no longer holds the archive's bytes, so this stored copy is the archive. */
+    lastCopy: boolean;
+};
+/** Test seam: the setup endpoints act on the real machine, so tests need a home and a service manager of their own. */
+export type SetupEnvironment = {
+    home?: string;
+    platform?: NodeJS.Platform;
+    exec?: ServiceOptions["exec"];
+};
+/**
+ * The loopback API shared by the CLI, dashboard, and MCP surfaces. It is the only
+ * place local surfaces read state from, so business logic is never duplicated.
+ *
+ * It deliberately cannot decrypt: no endpoint accepts a recovery phrase, and
+ * archive listings omit wrapped key material.
+ */
+export declare function createLocalApiServer(dataDir: string, token: string, options?: {
+    queue?: ArchiveQueue;
+    setup?: SetupEnvironment;
+}): Server;