pi-harness-runtime 1.1.396 → 1.1.397

package/packages/logging/src/server/outbox.ts

@@ -0,0 +1,316 @@
+/**
+ * Durable server-side outbox.
+ *
+ * The ingest endpoint must not tell a client "accepted" before the batch
+ * survived a crash. The outbox is the durable hand-off point between the
+ * HTTP request and the R2 upload:
+ *
+ *   request -> validate -> enqueue (durable) -> 202
+ *                                         |
+ *                                         v
+ *                               worker -> upload -> done
+ *
+ * The queue is a directory of JSON files, one per accepted batch:
+ *
+ *   <dir>/pending/<key>.json        the batch payload
+ *   <dir>/pending/<key>.meta.json   attempts, next attempt, last error
+ *   <dir>/done/<key>.json           uploaded batches (pruned after a while)
+ *
+ * A plain file queue is used on purpose: the server runs under Bun, which
+ * has no `node:sqlite`, and the queue stays inspectable with `ls`/`jq`.
+ *
+ * The key is derived from (tenant, installation, batch), so the same batch
+ * is enqueued only once - the ingest endpoint is idempotent.
+ */
+
+import { createHash } from "node:crypto";
+import {
+	existsSync,
+	mkdirSync,
+	readdirSync,
+	readFileSync,
+	renameSync,
+	rmSync,
+	statSync,
+	writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+import type { LogRecord } from "../types.js";
+
+export interface OutboxBatch {
+	tenantId: string;
+	installationId: string;
+	batchId: string;
+	app: string;
+	appVersion?: string;
+	hostLabel?: string;
+	sessionId?: string;
+	sentAt: string;
+	records: LogRecord[];
+}
+
+export interface OutboxEntry {
+	/** File name key of the batch. */
+	key: string;
+	/** Absolute path of the payload file. */
+	path: string;
+	batch: OutboxBatch;
+	attempts: number;
+	nextAttemptAt: number;
+	lastError?: string;
+	createdAt: number;
+}
+
+export interface OutboxStats {
+	/** Batches waiting for an upload. */
+	pending: number;
+	/** Age of the oldest pending batch in milliseconds. */
+	oldestPendingAgeMs: number;
+	/** Batches that failed at least once. */
+	failures: number;
+	/** Last upload error (message only, never a payload). */
+	lastError?: string;
+}
+
+export interface EnqueueResult {
+	accepted: boolean;
+	/** True when the same batch was already queued (idempotent replay). */
+	duplicate: boolean;
+}
+
+export interface LogOutboxOptions {
+	/** Directory of the queue. */
+	dir: string;
+	/** How long uploaded batches are kept, in milliseconds. Default 7 days. */
+	retentionMs?: number;
+}
+
+/** Make a value safe to use inside a file name. */
+export function safeSegment(value: string, maxLength = 48): string {
+	const cleaned = value
+		.replace(/\.\./g, "_")
+		.replace(/[^A-Za-z0-9._-]/g, "_")
+		.slice(0, maxLength)
+		.replace(/^\.+/, "_");
+	return cleaned.length > 0 ? cleaned : "unknown";
+}
+
+/**
+ * Build the queue key of a batch.
+ *
+ * The key is unique per (tenant, installation, batch) and contains no
+ * user controlled path segment.
+ */
+export function outboxKey(batch: {
+	tenantId: string;
+	installationId: string;
+	batchId: string;
+}): string {
+	const digest = createHash("sha256")
+		.update(`${batch.tenantId}|${batch.installationId}|${batch.batchId}`, "utf8")
+		.digest("hex")
+		.slice(0, 12);
+	return `${safeSegment(batch.tenantId)}--${safeSegment(batch.batchId)}--${digest}`;
+}
+
+export class LogOutbox {
+	private readonly dir: string;
+	private readonly pendingDir: string;
+	private readonly doneDir: string;
+	private readonly retentionMs: number;
+
+	constructor(options: LogOutboxOptions) {
+		this.dir = options.dir;
+		this.pendingDir = join(options.dir, "pending");
+		this.doneDir = join(options.dir, "done");
+		this.retentionMs = options.retentionMs ?? 7 * 24 * 60 * 60 * 1000;
+		try {
+			mkdirSync(this.pendingDir, { recursive: true });
+			mkdirSync(this.doneDir, { recursive: true });
+		} catch {
+			// The enqueue call reports the failure with a retryable error.
+		}
+	}
+
+	/** Directory of the queue. */
+	get directory(): string {
+		return this.dir;
+	}
+
+	/**
+	 * Store a batch durably.
+	 *
+	 * The write is atomic (write to a temp file, then rename), so a crash
+	 * can never leave a half written batch behind.
+	 */
+	enqueue(batch: OutboxBatch): EnqueueResult {
+		const key = outboxKey(batch);
+		const payloadPath = join(this.pendingDir, `${key}.json`);
+
+		if (existsSync(payloadPath) || existsSync(join(this.doneDir, `${key}.json`))) {
+			return { accepted: true, duplicate: true };
+		}
+
+		const tempPath = `${payloadPath}.tmp`;
+		try {
+			writeFileSync(tempPath, `${JSON.stringify(batch)}\n`, { mode: 0o600 });
+			renameSync(tempPath, payloadPath);
+		} catch (error) {
+			try {
+				rmSync(tempPath, { force: true });
+			} catch {
+				// Nothing else to do.
+			}
+			throw error;
+		}
+
+		this.writeMeta(key, {
+			attempts: 0,
+			nextAttemptAt: Date.now(),
+			createdAt: Date.now(),
+		});
+
+		return { accepted: true, duplicate: false };
+	}
+
+	/** The batches waiting for an upload, oldest first. */
+	listPending(limit = 100): OutboxEntry[] {
+		const entries: OutboxEntry[] = [];
+		let names: string[];
+		try {
+			names = readdirSync(this.pendingDir);
+		} catch {
+			return entries;
+		}
+
+		for (const name of names) {
+			if (!name.endsWith(".json") || name.endsWith(".meta.json")) continue;
+			const entry = this.readEntry(join(this.pendingDir, name));
+			if (entry) entries.push(entry);
+		}
+
+		entries.sort((a, b) => a.createdAt - b.createdAt);
+		return entries.slice(0, limit);
+	}
+
+	/** Mark a batch as uploaded and move it out of the pending directory. */
+	markUploaded(entry: OutboxEntry): void {
+		try {
+			renameSync(entry.path, join(this.doneDir, `${entry.key}.json`));
+		} catch {
+			// The file may have been moved already; nothing to do.
+		}
+		try {
+			rmSync(join(this.pendingDir, `${entry.key}.meta.json`), { force: true });
+		} catch {
+			// Ignore.
+		}
+	}
+
+	/**
+	 * Record a failed upload and schedule the next attempt.
+	 */
+	markFailed(entry: OutboxEntry, error: string, nextAttemptAt: number): void {
+		this.writeMeta(entry.key, {
+			attempts: entry.attempts + 1,
+			nextAttemptAt,
+			lastError: error,
+			createdAt: entry.createdAt,
+		});
+	}
+
+	/** Queue depth, oldest pending age and failure information. */
+	stats(now = Date.now()): OutboxStats {
+		const entries = this.listPending(Number.MAX_SAFE_INTEGER);
+		let oldestPendingAgeMs = 0;
+		let failures = 0;
+		let lastError: string | undefined;
+
+		for (const entry of entries) {
+			const age = now - entry.createdAt;
+			if (oldestPendingAgeMs === 0 || age > oldestPendingAgeMs) oldestPendingAgeMs = age;
+			if (entry.attempts > 0) failures++;
+			if (entry.lastError) lastError = entry.lastError;
+		}
+
+		return { pending: entries.length, oldestPendingAgeMs, failures, lastError };
+	}
+
+	/** Delete uploaded batches that are older than the retention period. */
+	prune(now = Date.now()): number {
+		let removed = 0;
+		let names: string[];
+		try {
+			names = readdirSync(this.doneDir);
+		} catch {
+			return 0;
+		}
+		for (const name of names) {
+			const path = join(this.doneDir, name);
+			try {
+				const age = now - statSync(path).mtimeMs;
+				if (age > this.retentionMs) {
+					rmSync(path, { force: true });
+					removed++;
+				}
+			} catch {
+				// Ignore a file that vanished.
+			}
+		}
+		return removed;
+	}
+
+	private writeMeta(
+		key: string,
+		meta: { attempts: number; nextAttemptAt: number; createdAt: number; lastError?: string },
+	): void {
+		const path = join(this.pendingDir, `${key}.meta.json`);
+		const tempPath = `${path}.tmp`;
+		try {
+			writeFileSync(tempPath, `${JSON.stringify(meta)}\n`, { mode: 0o600 });
+			renameSync(tempPath, path);
+		} catch {
+			// The meta file is an optimisation: without it the batch is still
+			// retried, only without the backoff hint.
+		}
+	}
+
+	private readEntry(path: string): OutboxEntry | null {
+		const key = path.slice(path.lastIndexOf("/") + 1).replace(/\.json$/, "");
+		try {
+			const batch = JSON.parse(readFileSync(path, "utf8")) as OutboxBatch;
+			let meta: { attempts: number; nextAttemptAt: number; createdAt: number; lastError?: string } = {
+				attempts: 0,
+				nextAttemptAt: 0,
+				createdAt: 0,
+			};
+			try {
+				const metaPath = join(this.pendingDir, `${key}.meta.json`);
+				if (existsSync(metaPath)) {
+					meta = { ...meta, ...(JSON.parse(readFileSync(metaPath, "utf8")) as typeof meta) };
+				}
+			} catch {
+				// Fall back to the defaults.
+			}
+			let createdAt = meta.createdAt;
+			if (!createdAt) {
+				try {
+					createdAt = statSync(path).mtimeMs;
+				} catch {
+					createdAt = Date.now();
+				}
+			}
+			return {
+				key,
+				path,
+				batch,
+				attempts: meta.attempts ?? 0,
+				nextAttemptAt: meta.nextAttemptAt ?? 0,
+				lastError: meta.lastError,
+				createdAt,
+			};
+		} catch {
+			return null;
+		}
+	}
+}

package/packages/logging/src/server/service.ts

@@ -0,0 +1,196 @@
+/**
+ * The log ingest service.
+ *
+ * The HTTP handler is a thin adapter around this class, so the whole ingest
+ * and query policy is testable without a socket:
+ *
+ *   - authentication happens BEFORE the body is parsed,
+ *   - the tenant comes from the credential, never from the request body,
+ *   - a batch is only acknowledged after it is durably queued,
+ *   - a query is always scoped to the tenant of the caller.
+ */
+
+import { validateBatch, type BatchValidationLimits } from "../ingest/schema.js";
+import {
+	LOG_READ_SCOPE,
+	LOG_WRITE_SCOPE,
+	type AuthenticatedCredential,
+	type LogCredentialStore,
+} from "./credentials.js";
+import { serverObjectKey } from "./keys.js";
+import type { LogIndexStore, LogQueryParams, LogQueryResult } from "./index-store.js";
+import type { LogOutbox, OutboxBatch } from "./outbox.js";
+
+export interface IngestSuccess {
+	accepted: number;
+	batchId: string;
+	duplicate: boolean;
+	tenantId: string;
+	queued: true;
+}
+
+export interface IngestFailure {
+	error: string;
+	code: string;
+}
+
+export interface IngestResult {
+	status: number;
+	body: IngestSuccess | IngestFailure;
+}
+
+export interface QuerySuccess {
+	entries: LogQueryResult["entries"];
+	nextCursor?: string;
+	hasMore: boolean;
+	scannedFiles: number;
+	truncated: boolean;
+	tenantId: string;
+}
+
+export interface LogIngestServiceOptions {
+	credentials: LogCredentialStore;
+	outbox: LogOutbox;
+	index: LogIndexStore;
+	/** Validation limits for an inbound batch. */
+	limits?: Partial<BatchValidationLimits>;
+}
+
+export class LogIngestService {
+	private readonly credentials: LogCredentialStore;
+	private readonly outbox: LogOutbox;
+	private readonly index: LogIndexStore;
+	private readonly limits: Partial<BatchValidationLimits>;
+
+	constructor(options: LogIngestServiceOptions) {
+		this.credentials = options.credentials;
+		this.outbox = options.outbox;
+		this.index = options.index;
+		this.limits = options.limits ?? {};
+	}
+
+	/** True when the ingest endpoint has at least one usable credential. */
+	get configured(): boolean {
+		return this.credentials.configured;
+	}
+
+	/**
+	 * Authenticate a presented bearer token.
+	 *
+	 * The HTTP layer uses it to reject a request before the body is read.
+	 */
+	authenticate(token: string | undefined): AuthenticatedCredential | null {
+		return this.credentials.authenticate(token);
+	}
+
+	/**
+	 * Accept a batch.
+	 *
+	 * The status codes matter:
+	 *   503 - no credential is configured (fail closed) or the queue is
+	 *         unavailable. The client keeps the batch in its spool.
+	 *   401 - the credential is missing or unknown.
+	 *   403 - the credential may not write.
+	 *   400 - the batch does not match the schema.
+	 *   202 - the batch is durably queued.
+	 */
+	ingest(body: unknown, bearerToken: string | undefined): IngestResult {
+		if (!this.credentials.configured) {
+			return {
+				status: 503,
+				body: { error: "log ingest is not configured", code: "ingest_unconfigured" },
+			};
+		}
+
+		const auth = this.credentials.authenticate(bearerToken);
+		if (!auth) {
+			return { status: 401, body: { error: "unauthorized", code: "unauthorized" } };
+		}
+		if (!auth.scopes.includes(LOG_WRITE_SCOPE)) {
+			return { status: 403, body: { error: "forbidden", code: "forbidden" } };
+		}
+
+		const validation = validateBatch(body, { limits: this.limits });
+		if (!validation.ok) {
+			return {
+				status: 400,
+				body: { error: validation.error.message, code: validation.error.code },
+			};
+		}
+
+		const incoming = validation.batch;
+		const batch: OutboxBatch = {
+			tenantId: auth.tenantId,
+			installationId: incoming.installationId,
+			batchId: incoming.batchId,
+			app: incoming.app,
+			sentAt: incoming.sentAt,
+			records: incoming.records,
+		};
+		if (incoming.appVersion) batch.appVersion = incoming.appVersion;
+		if (incoming.hostLabel) batch.hostLabel = incoming.hostLabel;
+		if (incoming.sessionId) batch.sessionId = incoming.sessionId;
+
+		try {
+			const result = this.outbox.enqueue(batch);
+			// The metadata index is best effort: a failing index never loses
+			// the batch itself.
+			this.index.append(batch, serverObjectKey(batch));
+			return {
+				status: 202,
+				body: {
+					accepted: batch.records.length,
+					batchId: batch.batchId,
+					duplicate: result.duplicate,
+					tenantId: batch.tenantId,
+					queued: true,
+				},
+			};
+		} catch (error) {
+			return {
+				status: 503,
+				body: {
+					error: `log queue unavailable: ${error instanceof Error ? error.message : String(error)}`,
+					code: "queue_unavailable",
+				},
+			};
+		}
+	}
+
+	/**
+	 * Query the metadata index for the tenant of the caller.
+	 *
+	 * The `tenantId` of the request is ignored on purpose - only the
+	 * credential decides whose records are visible.
+	 */
+	query(params: Omit<LogQueryParams, "tenantId">, bearerToken: string | undefined): {
+		status: number;
+		body: QuerySuccess | IngestFailure;
+	} {
+		if (!this.credentials.configured) {
+			return {
+				status: 503,
+				body: { error: "log ingest is not configured", code: "ingest_unconfigured" },
+			};
+		}
+
+		const auth = this.credentials.authenticate(bearerToken);
+		if (!auth) {
+			return { status: 401, body: { error: "unauthorized", code: "unauthorized" } };
+		}
+		if (!auth.scopes.includes(LOG_READ_SCOPE)) {
+			return { status: 403, body: { error: "forbidden", code: "forbidden" } };
+		}
+
+		const result = this.index.query({ ...params, tenantId: auth.tenantId });
+		const body: QuerySuccess = {
+			entries: result.entries,
+			hasMore: result.hasMore,
+			scannedFiles: result.scannedFiles,
+			truncated: result.truncated,
+			tenantId: auth.tenantId,
+		};
+		if (result.nextCursor) body.nextCursor = result.nextCursor;
+		return { status: 200, body };
+	}
+}

package/packages/logging/src/shipper.ts

@@ -12,16 +12,39 @@
 
 import { hostname } from "node:os";
 import { LogBuffer, type LogBufferOptions } from "./buffer.js";
+import {
+	DEFAULT_CLASSIFICATION_POLICY,
+	prepareRemoteRecord,
+	type ClassificationPolicy,
+} from "./classification.js";
 import { redactFields, redactText, type RedactionOptions } from "./redact.js";
 import { LogSampler, type SamplingPolicy } from "./sampler.js";
 import { LogSpool } from "./spool.js";
-import type { FlushStats, LogBatch, LogErrorInfo, LogInput, LogLevel, LogRecord, LogTransport } from "./types.js";
+import {
+	LOG_SCHEMA_VERSION,
+	type FlushStats,
+	type LogBatch,
+	type LogErrorInfo,
+	type LogInput,
+	type LogLevel,
+	type LogRecord,
+	type LogTransport,
+} from "./types.js";
 
 export interface LogShipperOptions {
 	/** Application name reported with every batch. */
 	app?: string;
-	/** Host name reported with every batch. */
-	host?: string;
+	/** Version of the emitting application. */
+	appVersion?: string;
+	/** Stable id of this installation. */
+	installationId?: string;
+	/**
+	 * Optional display label for the machine (never an identity).
+	 * Defaults to the hostname, which is display metadata only.
+	 */
+	hostLabel?: string;
+	/** Opaque session id attached to the batch when the records agree. */
+	sessionId?: string;
 	/** Transports in priority order. */
 	transports?: LogTransport[];
 	/** Local spool used when every transport fails. */
@@ -32,6 +55,8 @@ export interface LogShipperOptions {
 	sampling?: Partial<SamplingPolicy>;
 	/** Redaction limits. */
 	redaction?: RedactionOptions;
+	/** What may leave the machine. */
+	classification?: ClassificationPolicy;
 	/** Interval for the automatic flush timer. Default 15000 ms. */
 	flushIntervalMs?: number;
 	/** Clock injection (tests). */
@@ -56,12 +81,16 @@ function toErrorInfo(error: unknown): LogErrorInfo {
 
 export class LogShipper {
 	private readonly app: string;
-	private readonly host: string;
+	private readonly appVersion: string | undefined;
+	private readonly installationId: string;
+	private readonly hostLabel: string | undefined;
+	private readonly sessionId: string | undefined;
 	private readonly transports: LogTransport[];
 	private readonly spool: LogSpool | undefined;
 	private readonly buffer: LogBuffer;
 	private readonly sampler: LogSampler;
 	private readonly redaction: RedactionOptions;
+	private readonly classification: ClassificationPolicy;
 	private readonly flushIntervalMs: number;
 	private readonly now: () => number;
 	private readonly createBatchId: () => string;
@@ -72,17 +101,22 @@ export class LogShipper {
 		spooled: 0,
 		dropped: 0,
 		sampled: 0,
+		localOnly: 0,
 		errors: {},
 	};
 
 	constructor(options: LogShipperOptions = {}) {
 		this.app = options.app ?? "pi-harness";
-		this.host = options.host ?? hostname();
+		this.appVersion = options.appVersion;
+		this.installationId = options.installationId ?? "unknown";
+		this.hostLabel = options.hostLabel ?? hostname();
+		this.sessionId = options.sessionId;
 		this.transports = options.transports ?? [];
 		this.spool = options.spool;
 		this.buffer = new LogBuffer(options.buffer);
 		this.sampler = new LogSampler(options.sampling);
 		this.redaction = options.redaction ?? {};
+		this.classification = options.classification ?? DEFAULT_CLASSIFICATION_POLICY;
 		this.flushIntervalMs = options.flushIntervalMs ?? 15_000;
 		this.now = options.now ?? (() => Date.now());
 		this.createBatchId = options.createBatchId ?? defaultBatchId;
@@ -96,7 +130,13 @@ export class LogShipper {
 				this.stats.sampled++;
 				return;
 			}
-			this.buffer.push(record);
+			// What must stay on the machine never enters the shipping path.
+			const remote = prepareRemoteRecord(record, this.classification);
+			if (!remote) {
+				this.stats.localOnly++;
+				return;
+			}
+			this.buffer.push(remote);
 			if (this.buffer.size >= this.buffer.batchSize) {
 				void this.flush();
 			}
@@ -211,13 +251,19 @@ export class LogShipper {
 	}
 
 	private buildBatch(records: LogRecord[]): LogBatch {
-		return {
+		const sessionId = this.sessionId ?? records.find((record) => record.sessionId)?.sessionId;
+		const batch: LogBatch = {
+			schemaVersion: LOG_SCHEMA_VERSION,
 			batchId: this.createBatchId(),
 			app: this.app,
-			host: this.host,
+			installationId: this.installationId,
 			sentAt: new Date(this.now()).toISOString(),
 			records,
 		};
+		if (this.appVersion) batch.appVersion = this.appVersion;
+		if (this.hostLabel) batch.hostLabel = this.hostLabel;
+		if (sessionId) batch.sessionId = sessionId;
+		return batch;
 	}
 
 	private async doFlush(): Promise<void> {

package/packages/logging/src/singleton.ts

@@ -46,10 +46,13 @@ export function initCentralLogging(
 		const spool = resolved.spoolDir ? new LogSpool({ dir: resolved.spoolDir }) : undefined;
 		shipper = new LogShipper({
 			app: resolved.app,
+			appVersion: resolved.appVersion,
+			installationId: resolved.installationId,
 			transports: resolved.transports,
 			spool,
 			buffer: resolved.buffer,
 			sampling: resolved.sampling,
+			classification: resolved.classification,
 			flushIntervalMs: resolved.flushIntervalMs,
 		});
 		shipper.start();

package/packages/logging/src/types.ts

@@ -12,6 +12,15 @@
 
 export type LogLevel = "debug" | "info" | "warn" | "error";
 
+/**
+ * Schema version of the batch/record contract.
+ *
+ * v2 adds the installation identity (`installationId`), the app version and
+ * a `schemaVersion` marker, so a server can tell an old client from a new
+ * one instead of guessing.
+ */
+export const LOG_SCHEMA_VERSION = 2;
+
 /** Severity order used by the sampler and the buffer drop policy. */
 export const LOG_LEVEL_SEVERITY: Record<LogLevel, number> = {
 	debug: 10,
@@ -58,9 +67,26 @@ export interface LogInput {
 
 /** The transport unit: a batch of records shipped together. */
 export interface LogBatch {
+	/** Contract version, see `LOG_SCHEMA_VERSION`. */
+	schemaVersion: number;
 	batchId: string;
+	/** Application name, e.g. "pi-harness". */
 	app: string;
-	host: string;
+	/** Version of the emitting application. */
+	appVersion?: string;
+	/**
+	 * Stable random id of this installation, created once per local install.
+	 *
+	 * It is the only machine identity in the protocol. It is never derived
+	 * from the hostname and it is not an identity on the server either - the
+	 * tenant comes from the authenticated credential.
+	 */
+	installationId: string;
+	/** Optional user-editable display label (never an identity). */
+	hostLabel?: string;
+	/** Opaque session id, when the producer knows one. */
+	sessionId?: string;
+	/** ISO-8601 timestamp of the send. */
 	sentAt: string;
 	records: LogRecord[];
 }
@@ -80,6 +106,8 @@ export interface FlushStats {
 	dropped: number;
 	/** Records dropped by the sampler (level, sample rate, rate limit). */
 	sampled: number;
+	/** Records that stayed on the machine (classification: local only). */
+	localOnly: number;
 	/** Transport errors, keyed by transport name. */
 	errors: Record<string, string>;
 }