pi-harness-runtime 1.1.396 → 1.1.398

package/packages/logging/src/installation.ts

@@ -0,0 +1,76 @@
+/**
+ * Stable installation identity.
+ *
+ * One random id per local install. It is created on first use, persisted
+ * under `~/.pi-harness-runtime/installation-id.txt` and sent with every
+ * batch so the server can tell two machines of the same user apart.
+ *
+ * The id is deliberately opaque:
+ *   - it never contains the hostname, the user name or a workspace path,
+ *   - it is not derived from anything that could identify a person,
+ *   - it is never used as an identity on the server (the tenant comes from
+ *     the authenticated credential).
+ */
+
+import { randomUUID } from "node:crypto";
+import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+
+export const INSTALLATION_ID_FILE = "installation-id.txt";
+
+/** Default location of the installation id file. */
+export function defaultInstallationIdPath(): string {
+	return join(homedir(), ".pi-harness-runtime", INSTALLATION_ID_FILE);
+}
+
+const UUID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+/** True when the value looks like a uuid (the only shape we write). */
+export function isValidInstallationId(value: string): boolean {
+	return UUID_PATTERN.test(value.trim());
+}
+
+export interface ResolveInstallationIdOptions {
+	/** File the id is read from and written to. */
+	path?: string;
+	/** Id factory (tests). */
+	generate?: () => string;
+}
+
+/**
+ * Resolve the installation id: read it when it exists, create it otherwise.
+ *
+ * A failed read or write never throws - the id is best effort metadata and
+ * must not break logging.
+ */
+export function resolveInstallationId(options: ResolveInstallationIdOptions = {}): string {
+	const path = options.path ?? defaultInstallationIdPath();
+	const generate = options.generate ?? randomUUID;
+
+	try {
+		if (existsSync(path)) {
+			const existing = readFileSync(path, "utf8").trim();
+			if (isValidInstallationId(existing)) return existing;
+		}
+	} catch {
+		// Fall through and create a new id.
+	}
+
+	let id: string;
+	try {
+		id = generate();
+	} catch {
+		id = randomUUID();
+	}
+
+	try {
+		mkdirSync(dirname(path), { recursive: true });
+		writeFileSync(path, `${id}\n`, { mode: 0o600 });
+		chmodSync(path, 0o600);
+	} catch {
+		// The id still works for this process even when it cannot be persisted.
+	}
+
+	return id;
+}

package/packages/logging/src/sensitive-keys.ts

@@ -0,0 +1,40 @@
+/**
+ * Keys that never leave the machine.
+ *
+ * A key name is a strong hint about its content: "prompt", "output" or
+ * "transcript" carry private text, "path" and "url" carry the shape of the
+ * user's workspace. The list is shared by the client side classifier and the
+ * server side sanitizer, so both sides agree on what is private.
+ */
+
+export const LOCAL_ONLY_FIELD_KEYS = [
+	"prompt",
+	"response",
+	"message",
+	"content",
+	"transcript",
+	"output",
+	"stdout",
+	"stderr",
+	"command",
+	"text",
+	"body",
+	"payload",
+	"code",
+	"diff",
+	"file",
+	"path",
+	"url",
+	"uri",
+	"email",
+	"user",
+	"chat",
+	"question",
+	"answer",
+];
+
+/** True when the key is one that must not be shipped. */
+export function isLocalOnlyFieldKey(key: string): boolean {
+	const lower = key.toLowerCase();
+	return LOCAL_ONLY_FIELD_KEYS.some((needle) => lower.includes(needle));
+}

package/packages/logging/src/server/credentials.ts

@@ -0,0 +1,198 @@
+/**
+ * Scoped ingest credentials.
+ *
+ * The tenant is derived from the credential, never from the request body:
+ * a client cannot claim a tenant it does not own.
+ *
+ * Two credential shapes are supported:
+ *
+ *   1. A credential store (`log-ingest-credentials.json`) with one entry per
+ *      installation. Tokens are stored as SHA-256 hashes, so the file is safe
+ *      to keep in the key directory and a token can be revoked by removing
+ *      or revoking its entry.
+ *   2. A single shared token (`LOG_INGEST_TOKEN` or the
+ *      `log-ingest-token.txt` key file) for a single-tenant deployment. It
+ *      maps to the tenant `LOG_INGEST_TENANT` (default `default`).
+ *
+ * When neither is configured the store is NOT configured: the ingest
+ * endpoint then refuses every write (fail closed) instead of accepting
+ * anonymous batches.
+ */
+
+import { createHash, timingSafeEqual } from "node:crypto";
+import { existsSync, readFileSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+export interface LogCredential {
+	/** Stable id of the credential (safe to log). */
+	id: string;
+	/** The tenant this credential may write for. */
+	tenantId: string;
+	/** Human readable label, e.g. "laptop". */
+	label?: string;
+	/** Allowed scopes, e.g. ["logs:write", "logs:read"]. */
+	scopes: string[];
+	/** SHA-256 hash of the token, prefixed with the algorithm. */
+	tokenHash: string;
+	createdAt: string;
+	/** Set when the credential was revoked. */
+	revokedAt?: string | null;
+}
+
+export interface CredentialFile {
+	credentials: LogCredential[];
+}
+
+export interface AuthenticatedCredential {
+	credentialId: string;
+	tenantId: string;
+	scopes: string[];
+	/** True when the credential came from the shared token. */
+	shared: boolean;
+}
+
+export interface LogCredentialStoreOptions {
+	/** Path of the credential store file. */
+	path?: string;
+	/** Environment used for the shared token. */
+	env?: Record<string, string | undefined>;
+	/** Directory holding the key files. */
+	keysDir?: string;
+}
+
+export const DEFAULT_TENANT_ID = "default";
+export const LOG_WRITE_SCOPE = "logs:write";
+export const LOG_READ_SCOPE = "logs:read";
+
+/** Hash a token the way the store does. */
+export function hashToken(token: string): string {
+	return `sha256:${createHash("sha256").update(token, "utf8").digest("hex")}`;
+}
+
+function safeEqual(a: string, b: string): boolean {
+	const left = Buffer.from(a, "utf8");
+	const right = Buffer.from(b, "utf8");
+	if (left.length !== right.length) return false;
+	return timingSafeEqual(left, right);
+}
+
+function readFileTrimmed(path: string): string | undefined {
+	try {
+		if (!existsSync(path)) return undefined;
+		const value = readFileSync(path, "utf8").trim();
+		return value.length > 0 ? value : undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+export class LogCredentialStore {
+	private readonly credentials: LogCredential[];
+	private readonly shared: { token: string; tenantId: string } | undefined;
+
+	private constructor(credentials: LogCredential[], shared?: { token: string; tenantId: string }) {
+		this.credentials = credentials;
+		this.shared = shared;
+	}
+
+	/**
+	 * Load the store from the credential file and the shared token.
+	 *
+	 * A malformed credential file is ignored (and reported through
+	 * `loadErrors`), never fatal: the endpoint then fails closed.
+	 */
+	static load(options: LogCredentialStoreOptions = {}): LogCredentialStore {
+		const env = options.env ?? process.env;
+		const keysDir = options.keysDir ?? join(homedir(), ".pi-harness-runtime", "keys");
+		const path = options.path ?? join(keysDir, "log-ingest-credentials.json");
+
+		const credentials: LogCredential[] = [];
+		try {
+			if (existsSync(path)) {
+				const parsed = JSON.parse(readFileSync(path, "utf8")) as CredentialFile;
+				if (Array.isArray(parsed?.credentials)) {
+					for (const entry of parsed.credentials) {
+						if (
+							entry &&
+							typeof entry.id === "string" &&
+							typeof entry.tenantId === "string" &&
+							typeof entry.tokenHash === "string"
+						) {
+							credentials.push({
+								...entry,
+								scopes: Array.isArray(entry.scopes) ? entry.scopes : [LOG_WRITE_SCOPE],
+								createdAt: entry.createdAt ?? new Date(0).toISOString(),
+								revokedAt: entry.revokedAt ?? null,
+							});
+						}
+					}
+				}
+			}
+		} catch {
+			// A broken file must not open the door: the store stays unconfigured.
+		}
+
+		const sharedToken =
+			env.LOG_INGEST_TOKEN ?? readFileTrimmed(join(keysDir, "log-ingest-token.txt"));
+		const shared = sharedToken
+			? { token: sharedToken, tenantId: env.LOG_INGEST_TENANT ?? DEFAULT_TENANT_ID }
+			: undefined;
+
+		return new LogCredentialStore(credentials, shared);
+	}
+
+	/** True when at least one usable credential exists. */
+	get configured(): boolean {
+		return (
+			this.shared !== undefined ||
+			this.credentials.some((credential) => !credential.revokedAt)
+		);
+	}
+
+	/** The credentials without their token hashes. */
+	list(): Omit<LogCredential, "tokenHash">[] {
+		return this.credentials.map(({ tokenHash: _tokenHash, ...rest }) => rest);
+	}
+
+	/**
+	 * Authenticate a presented token.
+	 *
+	 * Returns null when the token is missing, unknown or revoked.
+	 */
+	authenticate(token: string | undefined): AuthenticatedCredential | null {
+		if (!token) return null;
+
+		if (this.shared && safeEqual(hashToken(token), hashToken(this.shared.token))) {
+			return {
+				credentialId: "shared",
+				tenantId: this.shared.tenantId,
+				scopes: [LOG_WRITE_SCOPE, LOG_READ_SCOPE],
+				shared: true,
+			};
+		}
+
+		const presented = hashToken(token);
+		for (const credential of this.credentials) {
+			if (credential.revokedAt) continue;
+			if (!safeEqual(presented, credential.tokenHash)) continue;
+			return {
+				credentialId: credential.id,
+				tenantId: credential.tenantId,
+				scopes: credential.scopes,
+				shared: false,
+			};
+		}
+
+		return null;
+	}
+}
+
+/**
+ * Extract the bearer token from an Authorization header.
+ */
+export function readBearerToken(header: string | undefined): string | undefined {
+	if (!header) return undefined;
+	const match = /^Bearer\s+(.+)$/i.exec(header.trim());
+	return match ? match[1].trim() : undefined;
+}

package/packages/logging/src/server/http.ts

@@ -0,0 +1,170 @@
+/**
+ * HTTP surface of the log service.
+ *
+ * The handlers live here (not in the server script) so the whole request
+ * path - authentication, size limits, validation, durable acceptance - can
+ * be tested over a real socket.
+ *
+ * Routes:
+ *   POST /api/logs         accept a batch (202 after the durable write)
+ *   POST /api/logs/query   read-only, tenant scoped metadata query
+ *   GET  /api/logs/health  queue depth, oldest pending age, failure count
+ */
+
+import { appendFileSync, existsSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import type { IncomingMessage, ServerResponse } from "node:http";
+import { readBearerToken } from "./credentials.js";
+import type { LogIndexStore, LogQueryParams } from "./index-store.js";
+import type { LogOutbox } from "./outbox.js";
+import type { LogIngestService } from "./service.js";
+
+export interface LogHttpOptions {
+	service: LogIngestService;
+	outbox: LogOutbox;
+	index: LogIndexStore;
+	/** Largest accepted ingest body. Default 2 MB. */
+	maxBodyBytes?: number;
+	/** Largest accepted query body. Default 64 KB. */
+	maxQueryBytes?: number;
+	/**
+	 * Directory for a human readable copy of the accepted batches
+	 * (`logs-<day>.ndjson`). Optional.
+	 */
+	localCopyDir?: string;
+}
+
+export type LogHttpHandler = (req: IncomingMessage, res: ServerResponse) => Promise<void>;
+
+function sendJson(res: ServerResponse, status: number, body: unknown): void {
+	res.writeHead(status, { "Content-Type": "application/json" });
+	res.end(JSON.stringify(body));
+}
+
+async function readBody(req: IncomingMessage, maxBytes: number): Promise<Buffer | null> {
+	const chunks: Buffer[] = [];
+	let size = 0;
+	for await (const chunk of req) {
+		size += (chunk as Buffer).length;
+		if (size > maxBytes) return null;
+		chunks.push(chunk as Buffer);
+	}
+	return Buffer.concat(chunks);
+}
+
+/**
+ * Write a human readable copy of the accepted records next to the other
+ * server logs. Best effort: a failing copy never affects the request.
+ */
+function writeLocalCopy(dir: string | undefined, payload: unknown): void {
+	if (!dir) return;
+	const records = (payload as { records?: unknown[] }).records;
+	if (!Array.isArray(records) || records.length === 0) return;
+	try {
+		if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+		const day = new Date().toISOString().slice(0, 10);
+		const file = join(dir, `logs-${day}.ndjson`);
+		appendFileSync(file, `${records.map((record) => JSON.stringify(record)).join("\n")}\n`);
+	} catch {
+		// The copy is a convenience, not a requirement.
+	}
+}
+
+/**
+ * Build the HTTP handler for the log routes.
+ */
+export function createLogHttpHandler(options: LogHttpOptions): LogHttpHandler {
+	const maxBodyBytes = options.maxBodyBytes ?? 2 * 1024 * 1024;
+	const maxQueryBytes = options.maxQueryBytes ?? 64 * 1024;
+
+	return async (req: IncomingMessage, res: ServerResponse): Promise<void> => {
+		const url = new URL(req.url ?? "/", "http://localhost");
+		const token = readBearerToken(req.headers.authorization);
+
+		if (url.pathname === "/api/logs" && req.method === "POST") {
+			// Fail closed: without a credential the endpoint does not exist.
+			if (!options.service.configured) {
+				sendJson(res, 503, { error: "log ingest is not configured", code: "ingest_unconfigured" });
+				return;
+			}
+			if (!options.service.authenticate(token)) {
+				sendJson(res, 401, { error: "unauthorized", code: "unauthorized" });
+				return;
+			}
+
+			const raw = await readBody(req, maxBodyBytes);
+			if (!raw) {
+				sendJson(res, 413, { error: "payload too large", code: "payload_too_large" });
+				return;
+			}
+
+			let payload: unknown;
+			try {
+				payload = JSON.parse(raw.toString());
+			} catch {
+				sendJson(res, 400, { error: "invalid json", code: "invalid_json" });
+				return;
+			}
+
+			const result = options.service.ingest(payload, token);
+			sendJson(res, result.status, result.body);
+			if (result.status === 202) writeLocalCopy(options.localCopyDir, payload);
+			return;
+		}
+
+		if (url.pathname === "/api/logs/query" && req.method === "POST") {
+			if (!options.service.configured) {
+				sendJson(res, 503, { error: "log ingest is not configured", code: "ingest_unconfigured" });
+				return;
+			}
+
+			const raw = await readBody(req, maxQueryBytes);
+			if (!raw) {
+				sendJson(res, 413, { error: "payload too large", code: "payload_too_large" });
+				return;
+			}
+
+			let params: Partial<LogQueryParams> = {};
+			if (raw.length > 0) {
+				try {
+					params = JSON.parse(raw.toString()) as Partial<LogQueryParams>;
+				} catch {
+					sendJson(res, 400, { error: "invalid json", code: "invalid_json" });
+					return;
+				}
+			}
+
+			const result = options.service.query(
+				{
+					installationId: params.installationId,
+					sessionId: params.sessionId,
+					since: params.since,
+					until: params.until,
+					levels: params.levels,
+					source: params.source,
+					event: params.event,
+					limit: params.limit,
+					cursor: params.cursor,
+				},
+				token,
+			);
+			sendJson(res, result.status, result.body);
+			return;
+		}
+
+		if (url.pathname === "/api/logs/health" && req.method === "GET") {
+			if (!options.service.configured) {
+				sendJson(res, 503, { configured: false });
+				return;
+			}
+			if (!options.service.authenticate(token)) {
+				sendJson(res, 401, { error: "unauthorized", code: "unauthorized" });
+				return;
+			}
+			sendJson(res, 200, { configured: true, outbox: options.outbox.stats() });
+			return;
+		}
+
+		sendJson(res, 404, { error: "not found" });
+	};
+}