pi-harness-runtime 1.1.397 → 1.1.402

package/packages/logging/src/server/incident-store.ts

@@ -0,0 +1,192 @@
+/**
+ * Durable incident storage.
+ *
+ * The log index answers "which records exist"; the incident store answers
+ * "what is actually broken". It keeps the derived incidents of a tenant in
+ * plain NDJSON files, one per tenant and day:
+ *
+ *   <dir>/<tenant>/incidents-<day>.ndjson
+ *
+ * The store is tenant scoped by construction: the tenant is a parameter of
+ * every method and the server fills it from the authenticated credential.
+ * A caller can therefore never read or write another tenant's incidents.
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync, readdirSync } from "node:fs";
+import { join } from "node:path";
+import {
+	INCIDENT_SCHEMA_VERSION,
+	mergeIncident,
+	type LogIncident,
+} from "../incidents/schema.js";
+import { safeSegment } from "./outbox.js";
+
+export interface IncidentQueryParams {
+	/** Tenant of the authenticated caller. Required. */
+	tenantId: string;
+	/** Only incidents of this severity. */
+	severity?: "warn" | "error";
+	/** Only incidents of this component. */
+	source?: string;
+	/** Only incidents of this event. */
+	event?: string;
+	/** ISO timestamp, inclusive lower bound on `lastSeen`. */
+	since?: string;
+	/** ISO timestamp, exclusive upper bound on `lastSeen`. */
+	until?: string;
+	/** Maximum number of incidents. Default 50, hard maximum 200. */
+	limit?: number;
+}
+
+export interface IncidentQueryResult {
+	incidents: LogIncident[];
+	/** True when the scan hit its day budget. */
+	truncated: boolean;
+	/** Number of day files that were read. */
+	scannedFiles: number;
+}
+
+export interface IncidentStoreOptions {
+	dir: string;
+	/** Days a query may scan back. Default 7, hard maximum 31. */
+	maxScanDays?: number;
+	/** Incidents per page by default. */
+	defaultLimit?: number;
+	/** Hard maximum of incidents per page. */
+	maxLimit?: number;
+}
+
+function toIsoDay(value: Date): string {
+	return value.toISOString().slice(0, 10);
+}
+
+export class IncidentStore {
+	private readonly dir: string;
+	private readonly maxScanDays: number;
+	private readonly defaultLimit: number;
+	private readonly maxLimit: number;
+
+	constructor(options: IncidentStoreOptions) {
+		this.dir = options.dir;
+		this.maxScanDays = Math.min(options.maxScanDays ?? 7, 31);
+		this.defaultLimit = options.defaultLimit ?? 50;
+		this.maxLimit = options.maxLimit ?? 200;
+	}
+
+	/**
+	 * Persist the incidents of one batch.
+	 *
+	 * The write is best effort: a failing store must never lose the batch
+	 * itself. The returned number is the number of incidents that were
+	 * written.
+	 */
+	append(incidents: LogIncident[]): number {
+		if (!Array.isArray(incidents) || incidents.length === 0) return 0;
+		const lines = incidents
+			.filter((incident) => incident && incident.fingerprint)
+			.map((incident) => JSON.stringify({ ...incident, schemaVersion: INCIDENT_SCHEMA_VERSION }));
+		if (lines.length === 0) return 0;
+
+		const day = toIsoDay(new Date());
+		try {
+			const dir = join(this.dir, safeSegment(incidents[0].tenantId));
+			mkdirSync(dir, { recursive: true });
+			appendFileSync(join(dir, `incidents-${day}.ndjson`), `${lines.join("\n")}\n`, {
+				mode: 0o600,
+			});
+			return lines.length;
+		} catch {
+			return 0;
+		}
+	}
+
+	/**
+	 * Query the incidents of one tenant.
+	 *
+	 * The incidents of the scanned files are merged by fingerprint, so a
+	 * failure that was seen in many batches is one incident with a summed
+	 * count, not many incidents.
+	 */
+	query(params: IncidentQueryParams): IncidentQueryResult {
+		const limit = Math.min(params.limit ?? this.defaultLimit, this.maxLimit);
+		const dir = join(this.dir, safeSegment(params.tenantId));
+		const merged = new Map<string, LogIncident>();
+		let scannedFiles = 0;
+		let truncated = false;
+
+		for (const day of this.dayRange(params.since)) {
+			const path = join(dir, `incidents-${day}.ndjson`);
+			if (!existsSync(path)) continue;
+			scannedFiles++;
+			let content: string;
+			try {
+				content = readFileSync(path, "utf8");
+			} catch {
+				continue;
+			}
+			for (const line of content.split("\n")) {
+				if (!line.trim()) continue;
+				let incident: LogIncident;
+				try {
+					incident = JSON.parse(line) as LogIncident;
+				} catch {
+					continue;
+				}
+				if (!incident.fingerprint) continue;
+				if (!this.matches(incident, params)) continue;
+				const existing = merged.get(incident.fingerprint);
+				merged.set(
+					incident.fingerprint,
+					existing ? mergeIncident(existing, incident) : incident,
+				);
+			}
+		}
+
+		const incidents = [...merged.values()]
+			.sort((a, b) => {
+				if (a.severity !== b.severity) return a.severity === "error" ? -1 : 1;
+				return b.lastSeen.localeCompare(a.lastSeen);
+			})
+			.slice(0, limit);
+
+		return { incidents, truncated, scannedFiles };
+	}
+
+	/** The days a query may read, oldest first. */
+	private dayRange(since?: string): string[] {
+		const days: string[] = [];
+		const end = new Date();
+		const start = new Date(end.getTime() - (this.maxScanDays - 1) * 86_400_000);
+		if (since) {
+			const parsed = new Date(since);
+			if (!Number.isNaN(parsed.getTime()) && parsed > start) start.setTime(parsed.getTime());
+		}
+		for (let day = new Date(start); day <= end; day.setTime(day.getTime() + 86_400_000)) {
+			days.push(toIsoDay(day));
+		}
+		return days;
+	}
+
+	/** Does the incident match the filters of the query? */
+	private matches(incident: LogIncident, params: IncidentQueryParams): boolean {
+		if (incident.tenantId !== params.tenantId) return false;
+		if (params.severity && incident.severity !== params.severity) return false;
+		if (params.source && incident.source !== params.source) return false;
+		if (params.event && incident.event !== params.event) return false;
+		if (params.since && incident.lastSeen < params.since) return false;
+		if (params.until && incident.lastSeen >= params.until) return false;
+		return true;
+	}
+
+	/** The tenant directories of the store. Used by the maintenance tests. */
+	tenants(): string[] {
+		if (!existsSync(this.dir)) return [];
+		try {
+			return readdirSync(this.dir, { withFileTypes: true })
+				.filter((entry) => entry.isDirectory())
+				.map((entry) => entry.name);
+		} catch {
+			return [];
+		}
+	}
+}

package/packages/logging/src/server/index.ts

@@ -54,3 +54,24 @@ export type {
 
 export { createLogHttpHandler } from "./http.js";
 export type { LogHttpHandler, LogHttpOptions } from "./http.js";
+
+export { IncidentStore } from "./incident-store.js";
+export type {
+	IncidentQueryParams,
+	IncidentQueryResult,
+	IncidentStoreOptions,
+} from "./incident-store.js";
+
+export {
+	INCIDENT_COLLECTION_NAME,
+	IncidentQdrantWriter,
+	incidentPointId,
+	incidentVector,
+	tenantFilter,
+	toIncidentPayload,
+} from "./incident-qdrant.js";
+export type {
+	IncidentPayload,
+	IncidentPoint,
+	QdrantLike,
+} from "./incident-qdrant.js";

package/packages/logging/src/server/service.ts

@@ -11,6 +11,7 @@
  */
 
 import { validateBatch, type BatchValidationLimits } from "../ingest/schema.js";
+import { extractIncidents, type LogIncident } from "../incidents/schema.js";
 import {
 	LOG_READ_SCOPE,
 	LOG_WRITE_SCOPE,
@@ -19,6 +20,8 @@ import {
 } from "./credentials.js";
 import { serverObjectKey } from "./keys.js";
 import type { LogIndexStore, LogQueryParams, LogQueryResult } from "./index-store.js";
+import type { IncidentQueryParams, IncidentStore } from "./incident-store.js";
+import type { IncidentQdrantWriter } from "./incident-qdrant.js";
 import type { LogOutbox, OutboxBatch } from "./outbox.js";
 
 export interface IngestSuccess {
@@ -52,20 +55,35 @@ export interface LogIngestServiceOptions {
 	credentials: LogCredentialStore;
 	outbox: LogOutbox;
 	index: LogIndexStore;
+	/** Durable storage of the derived incidents. Optional. */
+	incidents?: IncidentStore;
+	/** Searchable view of the incidents. Optional. */
+	incidentIndex?: IncidentQdrantWriter;
 	/** Validation limits for an inbound batch. */
 	limits?: Partial<BatchValidationLimits>;
 }
 
+export interface IncidentQuerySuccess {
+	incidents: LogIncident[];
+	truncated: boolean;
+	scannedFiles: number;
+	tenantId: string;
+}
+
 export class LogIngestService {
 	private readonly credentials: LogCredentialStore;
 	private readonly outbox: LogOutbox;
 	private readonly index: LogIndexStore;
+	private readonly incidents: IncidentStore | undefined;
+	private readonly incidentIndex: IncidentQdrantWriter | undefined;
 	private readonly limits: Partial<BatchValidationLimits>;
 
 	constructor(options: LogIngestServiceOptions) {
 		this.credentials = options.credentials;
 		this.outbox = options.outbox;
 		this.index = options.index;
+		this.incidents = options.incidents;
+		this.incidentIndex = options.incidentIndex;
 		this.limits = options.limits ?? {};
 	}
 
@@ -136,6 +154,9 @@ export class LogIngestService {
 			// The metadata index is best effort: a failing index never loses
 			// the batch itself.
 			this.index.append(batch, serverObjectKey(batch));
+			// The same holds for the derived incidents: they are a view of
+			// the batch, never a precondition for accepting it.
+			this.recordIncidents(batch);
 			return {
 				status: 202,
 				body: {
@@ -193,4 +214,62 @@ export class LogIngestService {
 		if (result.nextCursor) body.nextCursor = result.nextCursor;
 		return { status: 200, body };
 	}
+
+	/**
+	 * Derive the incidents of a batch and store them.
+	 *
+	 * The tenant comes from the credential that was authenticated, never
+	 * from the batch itself. A failing store or a failing Qdrant never
+	 * affects the acceptance of the batch.
+	 */
+	private recordIncidents(batch: OutboxBatch): void {
+		if (!this.incidents && !this.incidentIndex) return;
+		const derived = extractIncidents(batch.records, batch.tenantId, batch.installationId);
+		if (derived.length === 0) return;
+		if (this.incidents) this.incidents.append(derived);
+		if (this.incidentIndex) void this.incidentIndex.upsert(derived);
+	}
+
+	/**
+	 * Query the incidents of the tenant of the caller.
+	 *
+	 * The `tenantId` of the request is ignored on purpose - only the
+	 * credential decides whose incidents are visible.
+	 */
+	queryIncidents(
+		params: Omit<IncidentQueryParams, "tenantId">,
+		bearerToken: string | undefined,
+	): { status: number; body: IncidentQuerySuccess | IngestFailure } {
+		if (!this.credentials.configured) {
+			return {
+				status: 503,
+				body: { error: "log ingest is not configured", code: "ingest_unconfigured" },
+			};
+		}
+
+		const auth = this.credentials.authenticate(bearerToken);
+		if (!auth) {
+			return { status: 401, body: { error: "unauthorized", code: "unauthorized" } };
+		}
+		if (!auth.scopes.includes(LOG_READ_SCOPE)) {
+			return { status: 403, body: { error: "forbidden", code: "forbidden" } };
+		}
+		if (!this.incidents) {
+			return {
+				status: 503,
+				body: { error: "incident storage is not configured", code: "incidents_unconfigured" },
+			};
+		}
+
+		const result = this.incidents.query({ ...params, tenantId: auth.tenantId });
+		return {
+			status: 200,
+			body: {
+				incidents: result.incidents,
+				truncated: result.truncated,
+				scannedFiles: result.scannedFiles,
+				tenantId: auth.tenantId,
+			},
+		};
+	}
 }

package/packages/logging/src/shipper.ts

@@ -2,9 +2,25 @@
  * The shipper owns the delivery decision.
  *
  * Producers call `log()`. The shipper redacts, samples, buffers, and then
- * delivers a batch through the configured transports in priority order:
+ * delivers a batch through the configured transports.
  *
- *   R2 (durable archive) -> webhook (live) -> local spool (last resort)
+ * Two delivery modes exist:
+ *
+ *   fallback (the default) - the transports form a chain. The first
+ *     transport that accepts the batch wins and the later ones are not
+ *     called:
+ *
+ *       R2 (durable archive) -> webhook (live) -> local spool
+ *
+ *   fanout (explicit opt-in) - the transports are independent. Every
+ *     transport receives the batch, each one tracks its own success, and
+ *     the batch is spooled only when EVERY transport failed:
+ *
+ *       R2 (durable archive) + webhook (live) -> local spool
+ *
+ * Fallback is the default because it avoids duplicate records in the
+ * archive. Fanout is opt-in because it deliberately creates a second copy
+ * of every batch.
  *
  * If every transport fails the batch is spooled to disk and retried on the
  * next flush, so a temporary outage costs latency, not data.
@@ -31,6 +47,28 @@ import {
 	type LogTransport,
 } from "./types.js";
 
+/**
+ * How the batch is handed to the transports.
+ *
+ * - `fallback`: the first transport that accepts the batch wins.
+ * - `fanout`: every transport receives the batch independently.
+ */
+export type DeliveryMode = "fallback" | "fanout";
+
+/** The delivery state of one transport. */
+export interface TransportStatus {
+	/** The name of the transport. */
+	name: string;
+	/** How many batches this transport accepted. */
+	delivered: number;
+	/** How many batches this transport rejected. */
+	failed: number;
+	/** The message of the last failure. */
+	lastError?: string;
+	/** ISO timestamp of the last successful delivery. */
+	lastSuccessAt?: string;
+}
+
 export interface LogShipperOptions {
 	/** Application name reported with every batch. */
 	app?: string;
@@ -47,6 +85,12 @@ export interface LogShipperOptions {
 	sessionId?: string;
 	/** Transports in priority order. */
 	transports?: LogTransport[];
+	/**
+	 * How the transports are used. `fallback` (the default) lets the first
+	 * transport that accepts the batch win; `fanout` delivers the batch to
+	 * every transport independently.
+	 */
+	delivery?: DeliveryMode;
 	/** Local spool used when every transport fails. */
 	spool?: LogSpool;
 	/** Buffer bounds. */
@@ -86,6 +130,8 @@ export class LogShipper {
 	private readonly hostLabel: string | undefined;
 	private readonly sessionId: string | undefined;
 	private readonly transports: LogTransport[];
+	private readonly delivery: DeliveryMode;
+	private readonly transportStatus = new Map<string, TransportStatus>();
 	private readonly spool: LogSpool | undefined;
 	private readonly buffer: LogBuffer;
 	private readonly sampler: LogSampler;
@@ -112,6 +158,7 @@ export class LogShipper {
 		this.hostLabel = options.hostLabel ?? hostname();
 		this.sessionId = options.sessionId;
 		this.transports = options.transports ?? [];
+		this.delivery = options.delivery ?? "fallback";
 		this.spool = options.spool;
 		this.buffer = new LogBuffer(options.buffer);
 		this.sampler = new LogSampler(options.sampling);
@@ -221,6 +268,21 @@ export class LogShipper {
 		};
 	}
 
+	/**
+	 * The delivery state of every transport.
+	 *
+	 * In fanout mode this is the only way to see that one transport is
+	 * broken while the others still deliver.
+	 */
+	getTransportStatus(): TransportStatus[] {
+		return [...this.transportStatus.values()].map((status) => ({ ...status }));
+	}
+
+	/** The delivery mode of the shipper. */
+	get deliveryMode(): DeliveryMode {
+		return this.delivery;
+	}
+
 	private buildRecord(input: LogInput): LogRecord {
 		const record: LogRecord = {
 			ts: new Date(this.now()).toISOString(),
@@ -291,20 +353,73 @@ export class LogShipper {
 		}
 	}
 
-	/** Try every transport in order. Returns true when one accepted the batch. */
+	/**
+	 * Deliver the batch to the transports.
+	 *
+	 * In `fallback` mode the transports form a chain: the first one that
+	 * accepts the batch wins and the later ones are not called.
+	 *
+	 * In `fanout` mode every transport receives the batch independently.
+	 * Each transport tracks its own success, so a broken transport does not
+	 * hide the success of the others. The batch is only lost when EVERY
+	 * transport failed.
+	 */
 	private async deliver(batch: LogBatch): Promise<boolean> {
+		if (this.delivery === "fanout") return this.deliverToAll(batch);
 		for (const transport of this.transports) {
 			try {
 				await transport.send(batch);
+				this.noteSuccess(transport.name);
 				return true;
 			} catch (error) {
-				this.stats.errors[transport.name] =
-					error instanceof Error ? error.message : String(error);
+				this.noteFailure(transport.name, error);
 			}
 		}
 		return false;
 	}
 
+	/** Deliver the batch to every transport. Returns true when one accepted it. */
+	private async deliverToAll(batch: LogBatch): Promise<boolean> {
+		let accepted = false;
+		for (const transport of this.transports) {
+			try {
+				await transport.send(batch);
+				this.noteSuccess(transport.name);
+				accepted = true;
+			} catch (error) {
+				this.noteFailure(transport.name, error);
+			}
+		}
+		return accepted;
+	}
+
+	/** Record a successful delivery. */
+	private noteSuccess(name: string): void {
+		const status = this.statusFor(name);
+		status.delivered++;
+		status.lastSuccessAt = new Date(this.now()).toISOString();
+		delete status.lastError;
+	}
+
+	/** Record a failed delivery. */
+	private noteFailure(name: string, error: unknown): void {
+		const message = error instanceof Error ? error.message : String(error);
+		this.stats.errors[name] = message;
+		const status = this.statusFor(name);
+		status.failed++;
+		status.lastError = message;
+	}
+
+	/** The delivery state of one transport. */
+	private statusFor(name: string): TransportStatus {
+		let status = this.transportStatus.get(name);
+		if (!status) {
+			status = { name, delivered: 0, failed: 0 };
+			this.transportStatus.set(name, status);
+		}
+		return status;
+	}
+
 	/** Re-send spooled files. Stops at the first failure. */
 	private async drainSpool(): Promise<void> {
 		if (!this.spool) return;

package/packages/logging/src/singleton.ts

@@ -49,6 +49,7 @@ export function initCentralLogging(
 			appVersion: resolved.appVersion,
 			installationId: resolved.installationId,
 			transports: resolved.transports,
+			delivery: resolved.delivery,
 			spool,
 			buffer: resolved.buffer,
 			sampling: resolved.sampling,