pi-harness-runtime 1.1.394 → 1.1.396

package/packages/logging/src/redact.ts

@@ -0,0 +1,271 @@
+/**
+ * Secret redaction for log records.
+ *
+ * Two independent layers, because either one alone leaks:
+ *   1. Key based - a field named `apiKey` is replaced wholesale.
+ *   2. Value based - a secret embedded in free text (`Bearer abc...`) is
+ *      replaced even when the key name is innocuous.
+ *
+ * The rules are intentionally conservative: over-redacting a log line is
+ * cheap, leaking a credential is not.
+ */
+
+/**
+ * The redacted shape of a log value. Everything that leaves the process is
+ * JSON, so the redactor is typed against JSON instead of `unknown`.
+ */
+export type JsonValue =
+	| string
+	| number
+	| boolean
+	| null
+	| JsonValue[]
+	| { [key: string]: JsonValue };
+
+export interface RedactionOptions {
+	/** Longest string kept verbatim before truncation. Default 2000. */
+	maxStringLength?: number;
+	/** Maximum nesting depth kept. Default 5. */
+	maxDepth?: number;
+	/** Maximum number of keys kept per object. Default 50. */
+	maxKeys?: number;
+	/** Maximum number of items kept per array. Default 20. */
+	maxArrayItems?: number;
+	/** Placeholder written instead of a redacted value. */
+	placeholder?: string;
+}
+
+const DEFAULT_PLACEHOLDER = "[REDACTED]";
+
+/**
+ * Key names that always hold a secret. Matched as a whole token after the
+ * key is split on non-alphanumerics and camelCase boundaries.
+ */
+const SECRET_KEY_TOKENS = new Set([
+	"password",
+	"passwd",
+	"passphrase",
+	"secret",
+	"secrets",
+	"token",
+	"tokens",
+	"apikey",
+	"apisecret",
+	"authorization",
+	"auth",
+	"credential",
+	"credentials",
+	"privatekey",
+	"cookie",
+	"setcookie",
+	"sessionkey",
+	"accesskey",
+	"secretaccesskey",
+	"refreshtoken",
+	"bearertoken",
+	"signature",
+	"otp",
+	"pin",
+	"mfa",
+	"totp",
+]);
+
+/**
+ * Key names whose VALUE is sensitive even though the key itself is not a
+ * credential: prompts, raw model output and embeddings must never leave the
+ * machine in a log line.
+ */
+const SENSITIVE_VALUE_TOKENS = new Set([
+	"content",
+	"raw",
+	"prompt",
+	"prompts",
+	"messages",
+	"embedding",
+	"embeddings",
+	"vector",
+	"vectors",
+	"body",
+	"chatid",
+	"payload",
+	"transcript",
+]);
+
+/**
+ * Value patterns that look like credentials wherever they appear.
+ * Order matters: the more specific patterns run first.
+ */
+const SECRET_VALUE_PATTERNS: RegExp[] = [
+	// Authorization headers and bearer tokens.
+	/\b(?:bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi,
+	// JWT.
+	/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{4,}/g,
+	// OpenAI / Anthropic style keys.
+	/\bsk-[A-Za-z0-9_-]{12,}/g,
+	// AWS access key ids.
+	/\bAKIA[0-9A-Z]{12,}/g,
+	// GitHub / Slack / Stripe style tokens.
+	/\bgh[pousr]_[A-Za-z0-9]{16,}/g,
+	/\bxox[baprs]-[A-Za-z0-9-]{10,}/g,
+	// Cloudflare R2 style secret keys (long base64-ish runs).
+	/\b[A-Za-z0-9_-]{40,}\b/g,
+];
+
+/**
+ * `key = value` assignments of a credential-looking key. The key prefix is
+ * preserved, only the value is replaced.
+ */
+const SECRET_ASSIGNMENT_PATTERN =
+	/\b((?:api[_-]?key|apikey|access[_-]?key|secret[_-]?key|secret|token|password|passwd|authorization)\s*[:=]\s*)("?)([^\s"',;}]{4,})\2/gi;
+
+/** Split a key into lowercase tokens (`apiKey` -> `api`, `key`). */
+export function splitKeyTokens(key: string): string[] {
+	return key
+		.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
+		.split(/[^A-Za-z0-9]+/)
+		.filter(Boolean)
+		.map((part) => part.toLowerCase());
+}
+
+/** True when the key itself identifies a secret. */
+export function isSecretKey(key: string): boolean {
+	const tokens = splitKeyTokens(key);
+	if (tokens.some((token) => SECRET_KEY_TOKENS.has(token))) return true;
+	// Compound names such as `openaiApiKey` or `r2_secret_access_key`.
+	const joined = tokens.join("");
+	return (
+		joined.includes("apikey") ||
+		joined.includes("secretaccesskey") ||
+		joined.includes("privatekey") ||
+		joined.includes("accesstoken") ||
+		joined.includes("authtoken")
+	);
+}
+
+/** True when the key holds content that must not be logged verbatim. */
+export function isSensitiveValueKey(key: string): boolean {
+	const tokens = splitKeyTokens(key);
+	return tokens.some((token) => SENSITIVE_VALUE_TOKENS.has(token));
+}
+
+/** Replace credential-looking substrings inside a free-text value. */
+export function redactText(text: string, placeholder = DEFAULT_PLACEHOLDER): string {
+	let result = text;
+
+	for (const pattern of SECRET_VALUE_PATTERNS) {
+		// The regexes are global, so `lastIndex` must be reset per use.
+		pattern.lastIndex = 0;
+		result = result.replace(pattern, placeholder);
+	}
+
+	// `apiKey=abc` keeps the key name, loses the value. This runs last so a
+	// credential that was already replaced is not matched a second time.
+	SECRET_ASSIGNMENT_PATTERN.lastIndex = 0;
+	result = result.replace(SECRET_ASSIGNMENT_PATTERN, `$1${placeholder}`);
+
+	return result;
+}
+
+function truncate(text: string, maxLength: number): string {
+	if (text.length <= maxLength) return text;
+	return `${text.slice(0, maxLength)}...[truncated:${text.length - maxLength}]`;
+}
+
+/** Redact an arbitrary value (strings, numbers, objects, arrays). */
+export function redactValue(
+	value: unknown,
+	options: RedactionOptions = {},
+	depth = 0,
+	keyHint?: string,
+): JsonValue {
+	const maxStringLength = options.maxStringLength ?? 2000;
+	const maxDepth = options.maxDepth ?? 5;
+	const maxKeys = options.maxKeys ?? 50;
+	const maxArrayItems = options.maxArrayItems ?? 20;
+	const placeholder = options.placeholder ?? DEFAULT_PLACEHOLDER;
+
+	if (keyHint && isSecretKey(keyHint)) {
+		return placeholder;
+	}
+
+	if (value === null || value === undefined) return null;
+
+	switch (typeof value) {
+		case "string":
+			return truncate(redactText(value, placeholder), maxStringLength);
+		case "number":
+		case "boolean":
+			return value;
+		case "bigint":
+			return value.toString();
+		case "function":
+		case "symbol":
+			return `[${typeof value}]`;
+		default:
+			break;
+	}
+
+	if (value instanceof Date) return value.toISOString();
+	if (value instanceof Error) {
+		const info: { name: string; message: string; stack?: string } = {
+			name: value.name,
+			message: redactText(value.message, placeholder),
+		};
+		if (value.stack) info.stack = redactText(value.stack, placeholder);
+		return info;
+	}
+
+	if (depth >= maxDepth) return "[MAX_DEPTH]";
+
+	if (Array.isArray(value)) {
+		const items = value.slice(0, maxArrayItems).map((item) => redactValue(item, options, depth + 1));
+		if (value.length > maxArrayItems) {
+			items.push(`[+${value.length - maxArrayItems} more]`);
+		}
+		return items;
+	}
+
+	if (value instanceof Map) {
+		const out: Record<string, JsonValue> = {};
+		let count = 0;
+		for (const [key, item] of value) {
+			if (count >= maxKeys) break;
+			out[String(key)] = redactValue(item, options, depth + 1, String(key));
+			count++;
+		}
+		return out;
+	}
+
+	if (value instanceof Set) {
+		return redactValue([...value], options, depth + 1);
+	}
+
+	const record = value as Record<string, unknown>;
+	const out: Record<string, JsonValue> = {};
+	const keys = Object.keys(record);
+	for (const key of keys.slice(0, maxKeys)) {
+		const item = record[key];
+		if (isSecretKey(key)) {
+			out[key] = placeholder;
+			continue;
+		}
+		if (isSensitiveValueKey(key)) {
+			out[key] = placeholder;
+			continue;
+		}
+		out[key] = redactValue(item, options, depth + 1, key);
+	}
+	if (keys.length > maxKeys) {
+		out["[truncatedKeys]"] = keys.length - maxKeys;
+	}
+	return out;
+}
+
+/** Redact a flat field bag (the `fields` member of a record). */
+export function redactFields(
+	fields: Record<string, unknown> | undefined,
+	options: RedactionOptions = {},
+): Record<string, JsonValue> | undefined {
+	if (!fields) return undefined;
+	return redactValue(fields, options) as Record<string, JsonValue>;
+}

package/packages/logging/src/sampler.ts

@@ -0,0 +1,97 @@
+/**
+ * Sampling for the centralized log pipeline.
+ *
+ * Sampling protects the transport budget, not the producer: a chatty
+ * component must not be able to flood R2 (or the webhook server) with
+ * debug-level noise.
+ *
+ * Three independent gates:
+ *   1. minimum level  - everything below the configured level is dropped
+ *   2. sample rate    - debug records are kept deterministically per event
+ *   3. rate limit     - a cap per source+event per minute
+ *
+ * Errors bypass the sample rate and the rate limit. Losing an error to
+ * save a request is the wrong trade.
+ */
+
+import { LOG_LEVEL_SEVERITY, type LogLevel, type LogRecord } from "./types.js";
+
+export interface SamplingPolicy {
+	/** Records below this level are dropped. Default "info". */
+	minLevel: LogLevel;
+	/** Fraction of debug records kept, 0..1. Default 0.1. */
+	sampleRate: number;
+	/** Max records per source+event per minute (errors exempt). Default 60. */
+	rateLimitPerMinute: number;
+}
+
+export const DEFAULT_SAMPLING_POLICY: SamplingPolicy = {
+	minLevel: "info",
+	sampleRate: 0.1,
+	rateLimitPerMinute: 60,
+};
+
+/**
+ * Stable hash so the same event keeps the same decision across restarts -
+ * a debug event is either always sampled or never, never randomly.
+ */
+function stableHash(input: string): number {
+	let hash = 2166136261;
+	for (let i = 0; i < input.length; i++) {
+		hash ^= input.charCodeAt(i);
+		hash = Math.imul(hash, 16777619);
+	}
+	return (hash >>> 0) / 0xffffffff;
+}
+
+export class LogSampler {
+	private readonly policy: SamplingPolicy;
+	private readonly counters = new Map<string, { windowStart: number; count: number }>();
+
+	constructor(policy: Partial<SamplingPolicy> = {}) {
+		this.policy = { ...DEFAULT_SAMPLING_POLICY, ...policy };
+	}
+
+	getPolicy(): SamplingPolicy {
+		return { ...this.policy };
+	}
+
+	/** True when the record should be kept. */
+	shouldLog(record: LogRecord, now: number = Date.now()): boolean {
+		if (LOG_LEVEL_SEVERITY[record.level] < LOG_LEVEL_SEVERITY[this.policy.minLevel]) {
+			return false;
+		}
+
+		if (record.level === "error") return true;
+
+		if (record.level === "debug") {
+			const bucket = stableHash(`${record.source}:${record.event}`);
+			if (bucket >= this.policy.sampleRate) return false;
+		}
+
+		return this.allowByRateLimit(record, now);
+	}
+
+	private allowByRateLimit(record: LogRecord, now: number): boolean {
+		const limit = this.policy.rateLimitPerMinute;
+		if (limit <= 0) return true;
+
+		const key = `${record.source}:${record.event}`;
+		const windowStart = Math.floor(now / 60_000) * 60_000;
+		const entry = this.counters.get(key);
+
+		if (!entry || entry.windowStart !== windowStart) {
+			this.counters.set(key, { windowStart, count: 1 });
+			return true;
+		}
+
+		if (entry.count >= limit) return false;
+		entry.count++;
+		return true;
+	}
+
+	/** Forget the rate-limit counters (used by tests and by long sessions). */
+	reset(): void {
+		this.counters.clear();
+	}
+}

package/packages/logging/src/shipper.ts

@@ -0,0 +1,281 @@
+/**
+ * The shipper owns the delivery decision.
+ *
+ * Producers call `log()`. The shipper redacts, samples, buffers, and then
+ * delivers a batch through the configured transports in priority order:
+ *
+ *   R2 (durable archive) -> webhook (live) -> local spool (last resort)
+ *
+ * If every transport fails the batch is spooled to disk and retried on the
+ * next flush, so a temporary outage costs latency, not data.
+ */
+
+import { hostname } from "node:os";
+import { LogBuffer, type LogBufferOptions } from "./buffer.js";
+import { redactFields, redactText, type RedactionOptions } from "./redact.js";
+import { LogSampler, type SamplingPolicy } from "./sampler.js";
+import { LogSpool } from "./spool.js";
+import type { FlushStats, LogBatch, LogErrorInfo, LogInput, LogLevel, LogRecord, LogTransport } from "./types.js";
+
+export interface LogShipperOptions {
+	/** Application name reported with every batch. */
+	app?: string;
+	/** Host name reported with every batch. */
+	host?: string;
+	/** Transports in priority order. */
+	transports?: LogTransport[];
+	/** Local spool used when every transport fails. */
+	spool?: LogSpool;
+	/** Buffer bounds. */
+	buffer?: LogBufferOptions;
+	/** Sampling policy. */
+	sampling?: Partial<SamplingPolicy>;
+	/** Redaction limits. */
+	redaction?: RedactionOptions;
+	/** Interval for the automatic flush timer. Default 15000 ms. */
+	flushIntervalMs?: number;
+	/** Clock injection (tests). */
+	now?: () => number;
+	/** Batch id factory (tests). */
+	createBatchId?: () => string;
+}
+
+function defaultBatchId(): string {
+	const stamp = new Date().toISOString().replace(/[-:.]/g, "").slice(0, 15);
+	return `${stamp}-${Math.random().toString(36).slice(2, 8)}`;
+}
+
+function toErrorInfo(error: unknown): LogErrorInfo {
+	if (error instanceof Error) {
+		const info: LogErrorInfo = { name: error.name, message: error.message };
+		if (error.stack) info.stack = error.stack;
+		return info;
+	}
+	return { name: "Error", message: String(error) };
+}
+
+export class LogShipper {
+	private readonly app: string;
+	private readonly host: string;
+	private readonly transports: LogTransport[];
+	private readonly spool: LogSpool | undefined;
+	private readonly buffer: LogBuffer;
+	private readonly sampler: LogSampler;
+	private readonly redaction: RedactionOptions;
+	private readonly flushIntervalMs: number;
+	private readonly now: () => number;
+	private readonly createBatchId: () => string;
+	private timer: ReturnType<typeof setInterval> | undefined;
+	private flushing: Promise<void> | undefined;
+	private stats: FlushStats = {
+		shipped: 0,
+		spooled: 0,
+		dropped: 0,
+		sampled: 0,
+		errors: {},
+	};
+
+	constructor(options: LogShipperOptions = {}) {
+		this.app = options.app ?? "pi-harness";
+		this.host = options.host ?? hostname();
+		this.transports = options.transports ?? [];
+		this.spool = options.spool;
+		this.buffer = new LogBuffer(options.buffer);
+		this.sampler = new LogSampler(options.sampling);
+		this.redaction = options.redaction ?? {};
+		this.flushIntervalMs = options.flushIntervalMs ?? 15_000;
+		this.now = options.now ?? (() => Date.now());
+		this.createBatchId = options.createBatchId ?? defaultBatchId;
+	}
+
+	/** Record a log event. Never throws. */
+	log(input: LogInput): void {
+		try {
+			const record = this.buildRecord(input);
+			if (!this.sampler.shouldLog(record, this.now())) {
+				this.stats.sampled++;
+				return;
+			}
+			this.buffer.push(record);
+			if (this.buffer.size >= this.buffer.batchSize) {
+				void this.flush();
+			}
+		} catch {
+			// A logging failure must never break the caller.
+		}
+	}
+
+	/** Convenience wrappers around `log()`. */
+	debug(source: string, event: string, message?: string, fields?: Record<string, unknown>): void {
+		this.log({ level: "debug", source, event, message, fields });
+	}
+
+	info(source: string, event: string, message?: string, fields?: Record<string, unknown>): void {
+		this.log({ level: "info", source, event, message, fields });
+	}
+
+	warn(source: string, event: string, message?: string, fields?: Record<string, unknown>): void {
+		this.log({ level: "warn", source, event, message, fields });
+	}
+
+	error(
+		source: string,
+		event: string,
+		message?: string,
+		fields?: Record<string, unknown>,
+		error?: unknown,
+	): void {
+		this.log({ level: "error", source, event, message, fields, error });
+	}
+
+	/** Deliver buffered records (and drain the spool). Never throws. */
+	async flush(): Promise<void> {
+		if (this.flushing) return this.flushing;
+		this.flushing = this.doFlush().finally(() => {
+			this.flushing = undefined;
+		});
+		return this.flushing;
+	}
+
+	/**
+	 * Synchronous last resort used on process exit: whatever is still in
+	 * memory is written to the local spool so it is not lost.
+	 */
+	flushSync(): void {
+		try {
+			const records = this.buffer.peekAll();
+			if (records.length === 0) return;
+			this.buffer.clear();
+			if (!this.spool) return;
+			this.spool.append(this.buildBatch(records));
+			this.stats.spooled += records.length;
+		} catch {
+			// Nothing else we can do at exit time.
+		}
+	}
+
+	/** Start the periodic flush timer. */
+	start(): void {
+		if (this.timer) return;
+		this.timer = setInterval(() => {
+			void this.flush();
+		}, this.flushIntervalMs);
+		// Never keep the process alive just for log delivery.
+		this.timer.unref?.();
+	}
+
+	/** Stop the timer and flush once more. */
+	async stop(): Promise<void> {
+		if (this.timer) {
+			clearInterval(this.timer);
+			this.timer = undefined;
+		}
+		await this.flush();
+	}
+
+	getStats(): FlushStats {
+		return {
+			...this.stats,
+			dropped: this.stats.dropped + this.buffer.dropped,
+			errors: { ...this.stats.errors },
+		};
+	}
+
+	private buildRecord(input: LogInput): LogRecord {
+		const record: LogRecord = {
+			ts: new Date(this.now()).toISOString(),
+			level: input.level,
+			source: input.source,
+			event: input.event,
+		};
+
+		if (input.message !== undefined) {
+			record.message = redactText(input.message, this.redaction.placeholder);
+		}
+		if (input.sessionId) record.sessionId = input.sessionId;
+		if (typeof input.durationMs === "number") record.durationMs = input.durationMs;
+
+		const fields = redactFields(input.fields, this.redaction);
+		if (fields && Object.keys(fields).length > 0) record.fields = fields;
+
+		if (input.error !== undefined) {
+			const info = toErrorInfo(input.error);
+			record.error = {
+				name: info.name,
+				message: redactText(info.message, this.redaction.placeholder),
+				...(info.stack ? { stack: redactText(info.stack, this.redaction.placeholder) } : {}),
+			};
+		}
+
+		return record;
+	}
+
+	private buildBatch(records: LogRecord[]): LogBatch {
+		return {
+			batchId: this.createBatchId(),
+			app: this.app,
+			host: this.host,
+			sentAt: new Date(this.now()).toISOString(),
+			records,
+		};
+	}
+
+	private async doFlush(): Promise<void> {
+		try {
+			await this.drainSpool();
+
+			const records = this.buffer.takeBatch();
+			if (records.length === 0) return;
+
+			const batch = this.buildBatch(records);
+			const delivered = await this.deliver(batch);
+			if (delivered) {
+				this.stats.shipped += records.length;
+				return;
+			}
+
+			if (this.spool) {
+				this.spool.append(batch);
+				this.stats.spooled += records.length;
+			} else {
+				this.stats.dropped += records.length;
+			}
+		} catch {
+			// Never let a flush failure escape.
+		}
+	}
+
+	/** Try every transport in order. Returns true when one accepted the batch. */
+	private async deliver(batch: LogBatch): Promise<boolean> {
+		for (const transport of this.transports) {
+			try {
+				await transport.send(batch);
+				return true;
+			} catch (error) {
+				this.stats.errors[transport.name] =
+					error instanceof Error ? error.message : String(error);
+			}
+		}
+		return false;
+	}
+
+	/** Re-send spooled files. Stops at the first failure. */
+	private async drainSpool(): Promise<void> {
+		if (!this.spool) return;
+		const files = this.spool.list();
+		for (const file of files) {
+			const records = this.spool.read(file);
+			if (records.length === 0) {
+				this.spool.remove(file);
+				continue;
+			}
+			const batch = this.buildBatch(records);
+			const delivered = await this.deliver(batch);
+			if (!delivered) return;
+			this.spool.remove(file);
+			this.stats.shipped += records.length;
+		}
+	}
+}
+
+export type { LogLevel };