pi-harness-runtime 1.1.396 → 1.1.398

package/packages/logging/src/classification.ts

@@ -0,0 +1,175 @@
+/**
+ * What may leave the machine.
+ *
+ * The local log file keeps everything. The centralized archive keeps what an
+ * operator actually needs: service state, event codes, sanitized errors,
+ * durations and opaque identifiers.
+ *
+ * Two rules decide:
+ *
+ *   1. The event decides. A catch-all mirror of a local log line
+ *      (`harness.log`) or anything that smells like a prompt, a response or
+ *      command output never leaves the machine.
+ *   2. The payload decides. A message only travels when it looks like a
+ *      sanitized summary: short, one line, no path and no URL with a query
+ *      string. Fields with a private key name are dropped.
+ *
+ * The diagnostic mode (`PI_LOG_DIAGNOSTIC=1`) relaxes rule 2 for a user who
+ * explicitly asked for more, and nothing else.
+ */
+
+import { isLocalOnlyFieldKey } from "./sensitive-keys.js";
+import type { LogRecord } from "./types.js";
+
+export interface ClassificationPolicy {
+	/** Events that may go remote. */
+	remoteEvents: RegExp[];
+	/** Events that never go remote, whatever the level is. */
+	localEvents: RegExp[];
+	/** Keep a message even when it does not look sanitized. */
+	allowRawMessages: boolean;
+	/** Maximum length of a message that is kept. */
+	maxMessageLength: number;
+}
+
+/**
+ * Events that stay local.
+ *
+ * `harness.log` is the mirror of the catch-all local log line - it carries
+ * whatever the harness printed, so it is never remote.
+ */
+export const DEFAULT_LOCAL_EVENTS: RegExp[] = [
+	/^harness\.log$/,
+	/prompt/i,
+	/response/i,
+	/transcript/i,
+	/output/i,
+	/command/i,
+	/content/i,
+	/message\.text/i,
+	/notification\.text/i,
+];
+
+/**
+ * Events that may go remote.
+ *
+ * They are structured: a service state change, a session lifecycle event, a
+ * validation tier, a provider or quota event.
+ */
+export const DEFAULT_REMOTE_EVENTS: RegExp[] = [
+	/^harness\.service-diagnostics\./,
+	/^harness\.session\./,
+	/^harness\.startup\./,
+	/^harness\.shutdown\./,
+	/^harness\.planner\.decision$/,
+	/^harness\.glm-quota\.quota$/,
+	/^validation\./,
+	/^tier\./,
+	/^service\./,
+	/^session\./,
+	/^startup\./,
+	/^config\./,
+	/^provider\./,
+	/^quota\./,
+	/^logging\./,
+	/^jest\./,
+];
+
+export const DEFAULT_CLASSIFICATION_POLICY: ClassificationPolicy = {
+	remoteEvents: DEFAULT_REMOTE_EVENTS,
+	localEvents: DEFAULT_LOCAL_EVENTS,
+	allowRawMessages: false,
+	maxMessageLength: 200,
+};
+
+const ABSOLUTE_PATH = /(?:^|\s)\/(?:[\w.-]+\/)+[\w.-]+/;
+const URL_WITH_QUERY = /https?:\/\/\S+\?\S+/;
+
+/**
+ * True when a message looks like a sanitized summary and may travel.
+ */
+export function isSanitizedMessage(message: string, policy: ClassificationPolicy): boolean {
+	if (policy.allowRawMessages) return true;
+	if (message.length > policy.maxMessageLength) return false;
+	if (message.includes("\n")) return false;
+	if (ABSOLUTE_PATH.test(message)) return false;
+	if (URL_WITH_QUERY.test(message)) return false;
+	return true;
+}
+
+/**
+ * Classify one record.
+ *
+ * "remote" means the record may be shipped; "local" means it must stay on
+ * the machine.
+ */
+export function classifyRecord(
+	record: LogRecord,
+	policy: ClassificationPolicy = DEFAULT_CLASSIFICATION_POLICY,
+): "remote" | "local" {
+	for (const pattern of policy.localEvents) {
+		if (pattern.test(record.event)) return "local";
+	}
+	for (const pattern of policy.remoteEvents) {
+		if (pattern.test(record.event)) return "remote";
+	}
+	// Errors stay valuable even when the event is unknown: the message and
+	// the fields are sanitized anyway.
+	if (record.level === "warn" || record.level === "error") return "remote";
+	// Unknown events are local by default: an allowlist is safer than a
+	// denylist.
+	return "local";
+}
+
+/**
+ * Build the version of a record that may leave the machine.
+ *
+ * Returns null when the record must stay local.
+ */
+export function prepareRemoteRecord(
+	record: LogRecord,
+	policy: ClassificationPolicy = DEFAULT_CLASSIFICATION_POLICY,
+): LogRecord | null {
+	if (classifyRecord(record, policy) === "local") return null;
+
+	const remote: LogRecord = {
+		ts: record.ts,
+		level: record.level,
+		source: record.source,
+		event: record.event,
+	};
+
+	if (record.message !== undefined && isSanitizedMessage(record.message, policy)) {
+		remote.message = record.message;
+	}
+	if (record.sessionId) remote.sessionId = record.sessionId;
+	if (typeof record.durationMs === "number") remote.durationMs = record.durationMs;
+
+	if (record.fields) {
+		const fields: Record<string, unknown> = {};
+		for (const [key, value] of Object.entries(record.fields)) {
+			if (isLocalOnlyFieldKey(key)) continue;
+			fields[key] = value;
+		}
+		if (Object.keys(fields).length > 0) remote.fields = fields;
+	}
+
+	if (record.error) {
+		remote.error = { ...record.error };
+	}
+
+	return remote;
+}
+
+/** Keep only the records that may leave the machine. */
+export function filterRemoteRecords(
+	records: LogRecord[],
+	policy: ClassificationPolicy = DEFAULT_CLASSIFICATION_POLICY,
+): LogRecord[] {
+	const remote: LogRecord[] = [];
+	for (const record of records) {
+		const prepared = prepareRemoteRecord(record, policy);
+		if (prepared) remote.push(prepared);
+	}
+	return remote;
+}

package/packages/logging/src/config.ts

@@ -13,6 +13,11 @@ import { homedir } from "node:os";
 import { join } from "node:path";
 import { createR2Transport } from "./transports/r2.js";
 import { createWebhookTransport } from "./transports/webhook.js";
+import { resolveInstallationId } from "./installation.js";
+import {
+	DEFAULT_CLASSIFICATION_POLICY,
+	type ClassificationPolicy,
+} from "./classification.js";
 import type { LogBufferOptions } from "./buffer.js";
 import type { SamplingPolicy } from "./sampler.js";
 import type { LogTransport } from "./types.js";
@@ -24,6 +29,10 @@ export interface LogSinkConfig {
 	enabled: boolean;
 	/** Application name reported with every batch. */
 	app: string;
+	/** Version of the emitting application. */
+	appVersion?: string;
+	/** Stable id of this installation. */
+	installationId: string;
 	/** Transports in delivery priority order. */
 	transports: LogTransport[];
 	/** Directory of the local spool (undefined when logging is off). */
@@ -31,6 +40,8 @@ export interface LogSinkConfig {
 	sampling: Partial<SamplingPolicy>;
 	buffer: LogBufferOptions;
 	flushIntervalMs: number;
+	/** What may leave the machine. */
+	classification: ClassificationPolicy;
 	/** Human readable notes about what was enabled and why. */
 	notes: string[];
 }
@@ -39,6 +50,8 @@ export interface ResolveConfigOptions {
 	env?: Record<string, string | undefined>;
 	/** Directory holding the credential key files. */
 	keysDir?: string;
+	/** Overrides the installation id (tests, containers). */
+	installationId?: string;
 }
 
 function readKeyFile(keysDir: string, name: string): string | undefined {
@@ -67,6 +80,21 @@ export function defaultSpoolDir(): string {
 	return join(homedir(), ".pi-harness-runtime", "logs", "spool");
 }
 
+/**
+ * Build the classification policy.
+ *
+ * `PI_LOG_DIAGNOSTIC=1` is the explicit, user controlled diagnostic mode: it
+ * lets messages travel that the default policy keeps local. It never widens
+ * the event allowlist - only the message rule.
+ */
+function classificationFor(
+	env: Record<string, string | undefined>,
+): ClassificationPolicy {
+	const diagnostic = env.PI_LOG_DIAGNOSTIC === "1" || env.PI_LOG_DIAGNOSTIC === "true";
+	if (!diagnostic) return DEFAULT_CLASSIFICATION_POLICY;
+	return { ...DEFAULT_CLASSIFICATION_POLICY, allowRawMessages: true };
+}
+
 /**
  * Build the log sink configuration from the environment and the key files.
  */
@@ -76,14 +104,19 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 	const mode = (env.PI_LOG_SINK ?? "auto") as LogSinkMode;
 	const notes: string[] = [];
 
+	const installationId =
+		options.installationId ?? env.PI_LOG_INSTALLATION_ID ?? resolveInstallationId();
+
 	if (mode === "off") {
 		return {
 			enabled: false,
 			app: env.PI_LOG_APP ?? "pi-harness",
+			installationId,
 			transports: [],
 			sampling: {},
 			buffer: {},
 			flushIntervalMs: 0,
+			classification: classificationFor(env),
 			notes: ["logging disabled by PI_LOG_SINK=off"],
 		};
 	}
@@ -102,7 +135,21 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 		(accountId ? `https://${accountId}.r2.cloudflarestorage.com` : undefined);
 
 	const r2Ready = Boolean(bucket && accessKeyId && secretAccessKey && endpoint);
-	const wantR2 = mode === "r2" || mode === "r2+webhook" || (mode === "auto" && r2Ready);
+	const webhookUrl = env.PI_LOG_WEBHOOK_URL;
+
+	/*
+	 * Transport selection.
+	 *
+	 * The transports form a FALLBACK chain, not a broadcast: the first
+	 * transport that accepts the batch wins and the later ones are not
+	 * called. `auto` therefore prefers the webhook ingress when one is
+	 * configured: the server is the single writer to R2, so client machines
+	 * never need R2 credentials at all.
+	 */
+	const wantR2 =
+		mode === "r2" ||
+		mode === "r2+webhook" ||
+		(mode === "auto" && r2Ready && !webhookUrl);
 
 	if (wantR2) {
 		if (r2Ready) {
@@ -121,7 +168,6 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 		}
 	}
 
-	const webhookUrl = env.PI_LOG_WEBHOOK_URL;
 	const wantWebhook =
 		mode === "webhook" || mode === "r2+webhook" || (mode === "auto" && Boolean(webhookUrl));
 
@@ -134,6 +180,11 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 				}),
 			);
 			notes.push(`webhook enabled (${webhookUrl})`);
+			if (mode === "auto") {
+				notes.push(
+					"webhook is the client ingress point - the server writes to R2 (direct r2 transport skipped)",
+				);
+			}
 		} else {
 			notes.push("webhook requested but PI_LOG_WEBHOOK_URL is not set - transport skipped");
 		}
@@ -143,9 +194,10 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 		notes.push("no transport configured - logs stay local");
 	}
 
-	return {
+	const config: LogSinkConfig = {
 		enabled: true,
 		app: env.PI_LOG_APP ?? "pi-harness",
+		installationId,
 		transports,
 		spoolDir: env.PI_LOG_SPOOL_DIR ?? defaultSpoolDir(),
 		sampling: {
@@ -158,6 +210,11 @@ export function resolveLogSinkConfig(options: ResolveConfigOptions = {}): LogSin
 			maxBatchSize: readNumber(env, "PI_LOG_BATCH_SIZE", 100),
 		},
 		flushIntervalMs: readNumber(env, "PI_LOG_FLUSH_INTERVAL_MS", 15_000),
+		classification: classificationFor(env),
 		notes,
 	};
+
+	if (env.PI_LOG_APP_VERSION) config.appVersion = env.PI_LOG_APP_VERSION;
+
+	return config;
 }

package/packages/logging/src/index.ts

@@ -12,7 +12,7 @@
 
 // --- Types -----------------------------------------------------------------
 
-export { LOG_LEVEL_SEVERITY } from "./types.js";
+export { LOG_LEVEL_SEVERITY, LOG_SCHEMA_VERSION } from "./types.js";
 export type {
 	FlushStats,
 	LogBatch,
@@ -77,6 +77,50 @@ export {
 } from "./singleton.js";
 export type { InitCentralLoggingOptions } from "./singleton.js";
 
+// --- Installation identity --------------------------------------------------
+
+export {
+	INSTALLATION_ID_FILE,
+	defaultInstallationIdPath,
+	isValidInstallationId,
+	resolveInstallationId,
+} from "./installation.js";
+export type { ResolveInstallationIdOptions } from "./installation.js";
+
+// --- Classification (what may leave the machine) ----------------------------
+
+export {
+	DEFAULT_CLASSIFICATION_POLICY,
+	DEFAULT_LOCAL_EVENTS,
+	DEFAULT_REMOTE_EVENTS,
+	classifyRecord,
+	filterRemoteRecords,
+	isSanitizedMessage,
+	prepareRemoteRecord,
+} from "./classification.js";
+export type { ClassificationPolicy } from "./classification.js";
+
+export { LOCAL_ONLY_FIELD_KEYS, isLocalOnlyFieldKey } from "./sensitive-keys.js";
+
+// --- Ingest contract (shared by client and server) -------------------------
+
+export {
+	DEFAULT_BATCH_LIMITS,
+	LOG_LEVELS,
+	sanitizeRecord,
+	validateBatch,
+} from "./ingest/schema.js";
+export type {
+	BatchValidationError,
+	BatchValidationLimits,
+	BatchValidationResult,
+	SanitizeRecordResult,
+} from "./ingest/schema.js";
+
+// --- Server side ------------------------------------------------------------
+
+export * from "./server/index.js";
+
 // --- Configuration ---------------------------------------------------------
 
 export {

package/packages/logging/src/ingest/schema.ts

@@ -0,0 +1,295 @@
+/**
+ * Ingest contract: strict validation and allowlisted redaction.
+ *
+ * The client and the server share this module, so both sides agree on what a
+ * batch is. The server never trusts the body:
+ *
+ *   - the tenant comes from the authenticated credential, never from the JSON,
+ *   - the batch is validated against a strict, versioned schema,
+ *   - the records are rebuilt from an allowlist, so an unknown property can
+ *     never smuggle data into the archive,
+ *   - every string is redacted again on the server (defence in depth).
+ */
+
+import { redactText, redactValue } from "../redact.js";
+import { isLocalOnlyFieldKey } from "../sensitive-keys.js";
+import { LOG_SCHEMA_VERSION, type LogBatch, type LogErrorInfo, type LogLevel, type LogRecord } from "../types.js";
+
+export const LOG_LEVELS: readonly LogLevel[] = ["debug", "info", "warn", "error"];
+
+export interface BatchValidationLimits {
+	/** Maximum number of records in one batch. */
+	maxRecords: number;
+	/** Maximum length of a single string value. */
+	maxStringLength: number;
+	/** Maximum number of structured fields per record. */
+	maxFieldKeys: number;
+}
+
+export const DEFAULT_BATCH_LIMITS: BatchValidationLimits = {
+	maxRecords: 1000,
+	maxStringLength: 8192,
+	maxFieldKeys: 64,
+};
+
+export interface BatchValidationError {
+	code: string;
+	message: string;
+}
+
+export type BatchValidationResult =
+	| { ok: true; batch: LogBatch; warnings: string[] }
+	| { ok: false; error: BatchValidationError };
+
+function isPlainObject(value: unknown): value is Record<string, unknown> {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function isIsoTimestamp(value: string): boolean {
+	return Number.isFinite(Date.parse(value));
+}
+
+/** Batch ids are opaque but must stay safe to use in an object key. */
+const BATCH_ID_PATTERN = /^[A-Za-z0-9._:-]{1,128}$/;
+const INSTALLATION_ID_PATTERN = /^[A-Za-z0-9._:-]{8,64}$/;
+
+/**
+ * Validate an inbound batch.
+ *
+ * Unknown properties are NOT an error here - the sanitizer drops them. Only a
+ * structurally broken batch is rejected.
+ */
+export function validateBatch(
+	input: unknown,
+	options: { limits?: Partial<BatchValidationLimits> } = {},
+): BatchValidationResult {
+	const limits = { ...DEFAULT_BATCH_LIMITS, ...options.limits };
+	const warnings: string[] = [];
+
+	if (!isPlainObject(input)) {
+		return { ok: false, error: { code: "invalid_batch", message: "batch must be an object" } };
+	}
+
+	let schemaVersion = input.schemaVersion;
+	if (schemaVersion === undefined) {
+		// A client that predates the version marker sends v1.
+		schemaVersion = 1;
+		warnings.push("batch without a schema version, treated as legacy v1");
+	}
+	if (typeof schemaVersion !== "number" || !Number.isFinite(schemaVersion)) {
+		return {
+			ok: false,
+			error: { code: "invalid_schema_version", message: "schemaVersion must be a number" },
+		};
+	}
+	if (schemaVersion < 1) {
+		return {
+			ok: false,
+			error: { code: "invalid_schema_version", message: "schemaVersion must be >= 1" },
+		};
+	}
+	if (schemaVersion > LOG_SCHEMA_VERSION) {
+		return {
+			ok: false,
+			error: {
+				code: "unsupported_schema_version",
+				message: `schemaVersion ${schemaVersion} is newer than the supported version ${LOG_SCHEMA_VERSION}`,
+			},
+		};
+	}
+	if (schemaVersion < LOG_SCHEMA_VERSION) {
+		warnings.push(`legacy schemaVersion ${schemaVersion}`);
+	}
+
+	const batchId = input.batchId;
+	if (typeof batchId !== "string" || !BATCH_ID_PATTERN.test(batchId)) {
+		return {
+			ok: false,
+			error: { code: "invalid_batch_id", message: "batchId is missing or malformed" },
+		};
+	}
+
+	const app = input.app;
+	if (typeof app !== "string" || app.length === 0 || app.length > 64) {
+		return { ok: false, error: { code: "invalid_app", message: "app is missing or malformed" } };
+	}
+
+	const installationId = input.installationId;
+	if (installationId !== undefined && typeof installationId !== "string") {
+		return {
+			ok: false,
+			error: { code: "invalid_installation_id", message: "installationId must be a string" },
+		};
+	}
+	if (schemaVersion >= 2) {
+		if (typeof installationId !== "string" || !INSTALLATION_ID_PATTERN.test(installationId)) {
+			return {
+				ok: false,
+				error: { code: "invalid_installation_id", message: "installationId is missing or malformed" },
+			};
+		}
+	} else if (!installationId) {
+		warnings.push("batch without an installation id");
+	}
+
+	const sentAt = input.sentAt;
+	if (typeof sentAt !== "string" || !isIsoTimestamp(sentAt)) {
+		return { ok: false, error: { code: "invalid_sent_at", message: "sentAt is not a timestamp" } };
+	}
+
+	const records = input.records;
+	if (!Array.isArray(records)) {
+		return { ok: false, error: { code: "invalid_records", message: "records must be an array" } };
+	}
+	if (records.length === 0) {
+		return { ok: false, error: { code: "empty_batch", message: "records must not be empty" } };
+	}
+	if (records.length > limits.maxRecords) {
+		return {
+			ok: false,
+			error: {
+				code: "too_many_records",
+				message: `records exceed the limit of ${limits.maxRecords}`,
+			},
+		};
+	}
+
+	const sanitized: LogRecord[] = [];
+	for (const record of records) {
+		const result = sanitizeRecord(record, limits);
+		if (!result.ok) {
+			return { ok: false, error: result.error };
+		}
+		sanitized.push(result.record);
+	}
+
+	const batch: LogBatch = {
+		schemaVersion,
+		batchId,
+		app,
+		installationId: typeof installationId === "string" ? installationId : "legacy",
+		sentAt,
+		records: sanitized,
+	};
+
+	if (typeof input.appVersion === "string") batch.appVersion = input.appVersion.slice(0, 32);
+	if (typeof input.hostLabel === "string") batch.hostLabel = input.hostLabel.slice(0, 128);
+	if (typeof input.sessionId === "string") batch.sessionId = input.sessionId.slice(0, 128);
+
+	return { ok: true, batch, warnings };
+}
+
+/** Record keys that are accepted. Everything else is dropped. */
+const ALLOWED_RECORD_KEYS = new Set([
+	"ts",
+	"level",
+	"source",
+	"event",
+	"message",
+	"sessionId",
+	"durationMs",
+	"fields",
+	"error",
+]);
+
+/**
+ * Copy only the allowed properties of a record.
+ *
+ * This is the allowlist in action: a property the contract does not know is
+ * dropped before anything else happens to the record.
+ */
+function pickAllowedProperties(input: Record<string, unknown>): Record<string, unknown> {
+	const picked: Record<string, unknown> = {};
+	for (const key of ALLOWED_RECORD_KEYS) {
+		if (key in input) picked[key] = input[key];
+	}
+	return picked;
+}
+
+export type SanitizeRecordResult =
+	| { ok: true; record: LogRecord }
+	| { ok: false; error: BatchValidationError };
+
+/**
+ * Rebuild one record from the allowlist and redact every string.
+ *
+ * Unknown properties are dropped, `error.name`, `error.message` and
+ * `error.stack` are redacted, and structured fields lose the keys that can
+ * carry private content.
+ */
+export function sanitizeRecord(
+	input: unknown,
+	limits: Partial<BatchValidationLimits> = {},
+): SanitizeRecordResult {
+	const merged = { ...DEFAULT_BATCH_LIMITS, ...limits };
+	if (!isPlainObject(input)) {
+		return { ok: false, error: { code: "invalid_record", message: "record must be an object" } };
+	}
+
+	const allowed = pickAllowedProperties(input);
+
+	const ts = allowed.ts;
+	if (typeof ts !== "string" || !isIsoTimestamp(ts)) {
+		return { ok: false, error: { code: "invalid_ts", message: "ts is not a timestamp" } };
+	}
+
+	const level = allowed.level;
+	if (typeof level !== "string" || !LOG_LEVELS.includes(level as LogLevel)) {
+		return { ok: false, error: { code: "invalid_level", message: "level is not known" } };
+	}
+
+	const source = allowed.source;
+	if (typeof source !== "string" || source.length === 0 || source.length > 128) {
+		return { ok: false, error: { code: "invalid_source", message: "source is missing or malformed" } };
+	}
+
+	const event = allowed.event;
+	if (typeof event !== "string" || event.length === 0 || event.length > 128) {
+		return { ok: false, error: { code: "invalid_event", message: "event is missing or malformed" } };
+	}
+
+	const record: LogRecord = {
+		ts,
+		level: level as LogLevel,
+		source,
+		event,
+	};
+
+	if (typeof allowed.message === "string") {
+		record.message = redactText(allowed.message).slice(0, merged.maxStringLength);
+	}
+	if (typeof allowed.sessionId === "string") {
+		record.sessionId = allowed.sessionId.slice(0, 128);
+	}
+	if (typeof allowed.durationMs === "number" && Number.isFinite(allowed.durationMs)) {
+		record.durationMs = allowed.durationMs;
+	}
+
+	if (isPlainObject(allowed.fields)) {
+		const fields: Record<string, unknown> = {};
+		let kept = 0;
+		for (const [key, value] of Object.entries(allowed.fields)) {
+			if (kept >= merged.maxFieldKeys) break;
+			if (isLocalOnlyFieldKey(key)) continue;
+			fields[key] = redactValue(value);
+			kept++;
+		}
+		if (Object.keys(fields).length > 0) record.fields = fields;
+	}
+
+	if (isPlainObject(allowed.error)) {
+		const error: LogErrorInfo = { name: "Error", message: "" };
+		if (typeof allowed.error.name === "string") {
+			error.name = redactText(allowed.error.name).slice(0, 256);
+		}
+		if (typeof allowed.error.message === "string") {
+			error.message = redactText(allowed.error.message).slice(0, merged.maxStringLength);
+		}
+		if (typeof allowed.error.stack === "string") {
+			error.stack = redactText(allowed.error.stack).slice(0, merged.maxStringLength);
+		}
+		record.error = error;
+	}
+
+	return { ok: true, record };
+}