pi-crew 0.9.53 → 0.9.55

package/src/observability/metric-registry.ts

@@ -0,0 +1,100 @@
+import { logInternalError } from "../utils/internal-error.ts";
+import { Counter, Gauge, Histogram, type Metric, type MetricSnapshot } from "./metrics-primitives.ts";
+
+const METRIC_NAME_PATTERN = /^crew\.[a-z]+\.[a-z][a-z_]*$/;
+
+function assertMetricName(name: string): void {
+	if (!METRIC_NAME_PATTERN.test(name)) throw new Error(`Invalid metric name '${name}'. Expected crew.<domain>.<measure>.`);
+}
+
+export class MetricRegistry {
+	private metrics = new Map<string, Metric>();
+
+	registerCounter(name: string, description: string): Counter {
+		assertMetricName(name);
+		if (this.metrics.has(name)) throw new Error(`Metric '${name}' is already registered.`);
+		const metric = new Counter(name, description);
+		this.metrics.set(name, metric);
+		return metric;
+	}
+
+	registerGauge(name: string, description: string): Gauge {
+		assertMetricName(name);
+		if (this.metrics.has(name)) throw new Error(`Metric '${name}' is already registered.`);
+		const metric = new Gauge(name, description);
+		this.metrics.set(name, metric);
+		return metric;
+	}
+
+	registerHistogram(name: string, description: string, buckets?: number[]): Histogram {
+		assertMetricName(name);
+		if (this.metrics.has(name)) throw new Error(`Metric '${name}' is already registered.`);
+		const metric = new Histogram(name, description, buckets);
+		this.metrics.set(name, metric);
+		return metric;
+	}
+
+	counter(name: string, description: string): Counter {
+		const existing = this.metrics.get(name);
+		if (existing instanceof Counter) {
+			if (existing.description !== description) {
+				logInternalError(
+					"metric-registry.counter",
+					new Error("description mismatch"),
+					`name='${name}' original='${existing.description}'`,
+				);
+			}
+			return existing;
+		}
+		if (existing) throw new Error(`Metric '${name}' is not a counter.`);
+		return this.registerCounter(name, description);
+	}
+
+	gauge(name: string, description: string): Gauge {
+		const existing = this.metrics.get(name);
+		if (existing instanceof Gauge) {
+			if (existing.description !== description) {
+				logInternalError(
+					"metric-registry.gauge",
+					new Error("description mismatch"),
+					`name='${name}' original='${existing.description}'`,
+				);
+			}
+			return existing;
+		}
+		if (existing) throw new Error(`Metric '${name}' is not a gauge.`);
+		return this.registerGauge(name, description);
+	}
+
+	histogram(name: string, description: string, buckets?: number[]): Histogram {
+		const existing = this.metrics.get(name);
+		if (existing instanceof Histogram) {
+			if (existing.description !== description) {
+				logInternalError(
+					"metric-registry.histogram",
+					new Error("description mismatch"),
+					`name='${name}' original='${existing.description}'`,
+				);
+			}
+			return existing;
+		}
+		if (existing) throw new Error(`Metric '${name}' is not a histogram.`);
+		return this.registerHistogram(name, description, buckets);
+	}
+
+	get(name: string): Metric | undefined {
+		return this.metrics.get(name);
+	}
+
+	snapshot(): MetricSnapshot[] {
+		return [...this.metrics.values()].map((metric) => metric.snapshot());
+	}
+
+	dispose(): void {
+		this.metrics.clear();
+	}
+}
+
+export function createMetricRegistry(): MetricRegistry {
+	return new MetricRegistry();
+}

package/src/observability/metric-retention.ts

@@ -0,0 +1,64 @@
+import { labelKey, type MetricLabels } from "./metrics-primitives.ts";
+
+interface WindowEvent {
+	timestamp: number;
+	labels: MetricLabels;
+	delta: number;
+}
+
+export class TimeWindowedCounter {
+	private events: WindowEvent[] = [];
+	private readonly windowMs: number;
+	private readonly now: () => number;
+	private static readonly MAX_EVENTS = 100_000;
+
+	constructor(windowMs = 3_600_000, now: () => number = () => Date.now()) {
+		this.windowMs = windowMs;
+		this.now = now;
+	}
+
+	inc(labels: MetricLabels = {}, delta = 1): void {
+		if (!Number.isFinite(delta)) return;
+		// Cap the event array to prevent unbounded memory growth.
+		if (this.events.length >= TimeWindowedCounter.MAX_EVENTS) this.prune();
+		this.events.push({
+			timestamp: this.now(),
+			labels: { ...labels },
+			delta,
+		});
+		this.prune();
+	}
+
+	// PERF (OBS-5): This method is O(N) per call because it linearly scans
+	// all events and calls labelKey() (which uses JSON.stringify) on each.
+	// A more efficient aggregation (e.g., pre-aggregated label buckets or a
+	// sliding-window histogram) is planned to replace this hot path.
+	count(labels: MetricLabels = {}, durationMs = this.windowMs): number {
+		const now = this.now();
+		this.pruneAt(now);
+		const key = labelKey(labels);
+		const cutoff = now - durationMs;
+		return this.events
+			.filter((event) => event.timestamp >= cutoff && labelKey(event.labels) === key)
+			.reduce((sum, event) => sum + event.delta, 0);
+	}
+
+	rate(labels: MetricLabels = {}, durationMs = this.windowMs): number {
+		if (durationMs <= 0) return 0;
+		return this.count(labels, durationMs) / (durationMs / 1000);
+	}
+
+	size(): number {
+		this.prune();
+		return this.events.length;
+	}
+
+	private prune(): void {
+		this.pruneAt(this.now());
+	}
+
+	private pruneAt(now: number): void {
+		const cutoff = now - this.windowMs;
+		this.events = this.events.filter((event) => event.timestamp >= cutoff);
+	}
+}

package/src/observability/metric-sink.ts

@@ -0,0 +1,99 @@
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { logInternalError } from "../utils/internal-error.ts";
+import { redactSecrets } from "../utils/redaction.ts";
+import type { MetricRegistry } from "./metric-registry.ts";
+import type { MetricSnapshot } from "./metrics-primitives.ts";
+
+export interface MetricSink {
+	writeSnapshot(snapshots: MetricSnapshot[]): void | Promise<void>;
+	dispose(): void;
+}
+
+export interface MetricFileSinkOptions {
+	crewRoot: string;
+	registry: MetricRegistry;
+	retentionDays?: number;
+	intervalMs?: number;
+}
+
+function rotateOldFiles(dir: string, retentionDays: number, now = Date.now()): void {
+	if (!fs.existsSync(dir)) return;
+	const maxAge = retentionDays * 24 * 60 * 60 * 1000;
+	for (const file of fs.readdirSync(dir)) {
+		if (!file.endsWith(".jsonl")) continue;
+		const fullPath = path.join(dir, file);
+		try {
+			if (now - fs.statSync(fullPath).mtimeMs > maxAge) fs.unlinkSync(fullPath);
+		} catch (error) {
+			logInternalError("metric-sink.rotate", error, fullPath);
+		}
+	}
+}
+
+export function createMetricFileSink(opts: MetricFileSinkOptions): MetricSink {
+	const dir = path.join(opts.crewRoot, "state", "metrics");
+	const retentionDays = opts.retentionDays ?? 7;
+	// 4.1: hold a single open fd per UTC date instead of `appendFileSync` per
+	// snapshot. Avoids open/close syscalls each tick and keeps the
+	// synchronous write semantics that callers (and tests) expect.
+	let fd: number | undefined;
+	let fdDate: string | undefined;
+	const ensureFd = (date: string): number => {
+		if (fd !== undefined && fdDate === date) return fd;
+		if (fd !== undefined) {
+			try {
+				fs.closeSync(fd);
+			} catch (error) {
+				logInternalError("metric-sink.closeFd", error);
+			}
+		}
+		fs.mkdirSync(dir, { recursive: true });
+		rotateOldFiles(dir, retentionDays);
+		fd = fs.openSync(path.join(dir, `${date}.jsonl`), "a");
+		fdDate = date;
+		return fd;
+	};
+	const writeSnapshot = (snapshots: MetricSnapshot[]): Promise<void> => {
+		try {
+			const now = new Date();
+			const date = now.toISOString().slice(0, 10);
+			const redacted = redactSecrets(snapshots);
+			if (!Array.isArray(redacted)) {
+				logInternalError("metric-sink.type", new Error("redactSecrets did not return an array"), `got=${typeof redacted}`);
+				return Promise.resolve();
+			}
+			const target = ensureFd(date);
+			const line = `${JSON.stringify({ exportedAt: now.toISOString(), snapshots: redacted as MetricSnapshot[] })}\n`;
+			// Async write to avoid blocking the main thread on the 60s tick.
+			// Return a promise that resolves once the write callback fires so
+			// tests (and any callers awaiting) can synchronize on completion.
+			return new Promise<void>((resolve) => {
+				fs.write(target, line, (err) => {
+					if (err) logInternalError("metric-sink.asyncWrite", err);
+					resolve();
+				});
+			});
+		} catch (error) {
+			logInternalError("metric-sink.write", error);
+			return Promise.resolve();
+		}
+	};
+	const timer = setInterval(() => writeSnapshot(opts.registry.snapshot()), opts.intervalMs ?? 60_000);
+	timer.unref();
+	return {
+		writeSnapshot,
+		dispose: () => {
+			clearInterval(timer);
+			if (fd !== undefined) {
+				try {
+					fs.closeSync(fd);
+				} catch (error) {
+					logInternalError("metric-sink.dispose", error);
+				}
+				fd = undefined;
+				fdDate = undefined;
+			}
+		},
+	};
+}

package/src/observability/metrics-primitives.ts

@@ -0,0 +1,219 @@
+export type MetricLabelValue = string | number;
+export type MetricLabels = Record<string, MetricLabelValue>;
+
+export interface MetricPoint {
+	labels: MetricLabels;
+	value: number;
+}
+
+export interface HistogramPoint {
+	labels: MetricLabels;
+	buckets: number[];
+	counts: number[];
+	sum: number;
+	count: number;
+	quantiles: Record<string, number>;
+}
+
+export interface MetricSnapshot {
+	type: "counter" | "gauge" | "histogram";
+	name: string;
+	description: string;
+	values: MetricPoint[] | HistogramPoint[];
+}
+
+interface StoredValue {
+	labels: MetricLabels;
+	value: number;
+}
+
+interface StoredHistogram {
+	labels: MetricLabels;
+	counts: number[];
+	sum: number;
+	count: number;
+}
+
+export const DEFAULT_HISTOGRAM_BUCKETS = [1, 2, 5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000] as const;
+
+/** Maximum number of unique label combinations per metric. */
+const MAX_LABEL_COMBINATIONS = 10_000;
+
+function enforceLabelCap(map: Map<string, unknown>, metricName: string): void {
+	while (map.size > MAX_LABEL_COMBINATIONS) {
+		const firstKey = map.keys().next().value;
+		if (firstKey !== undefined) map.delete(firstKey);
+	}
+}
+
+function normalizeLabels(labels: MetricLabels = {}): MetricLabels {
+	const normalized: MetricLabels = {};
+	for (const [key, value] of Object.entries(labels).sort(([left], [right]) => left.localeCompare(right))) normalized[key] = value;
+	return normalized;
+}
+
+export function labelKey(labels: MetricLabels = {}): string {
+	return JSON.stringify(normalizeLabels(labels));
+}
+
+function cloneLabels(labels: MetricLabels): MetricLabels {
+	return { ...labels };
+}
+
+export abstract class Metric {
+	readonly name: string;
+	readonly description: string;
+
+	constructor(name: string, description: string) {
+		this.name = name;
+		this.description = description;
+	}
+
+	abstract snapshot(): MetricSnapshot;
+}
+
+export class Counter extends Metric {
+	private values = new Map<string, StoredValue>();
+
+	inc(labels: MetricLabels = {}, delta = 1): void {
+		if (!Number.isFinite(delta) || delta < 0) return;
+		const key = labelKey(labels);
+		const current = this.values.get(key) ?? {
+			labels: normalizeLabels(labels),
+			value: 0,
+		};
+		// Delete before set to move key to end of insertion order (MRU).
+		// Without this, enforceLabelCap could evict an actively-used entry.
+		this.values.delete(key);
+		this.values.set(key, {
+			labels: current.labels,
+			value: current.value + delta,
+		});
+		enforceLabelCap(this.values, this.name);
+	}
+
+	value(labels: MetricLabels = {}): number {
+		return this.values.get(labelKey(labels))?.value ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return {
+			type: "counter",
+			name: this.name,
+			description: this.description,
+			values: [...this.values.values()].map((entry) => ({
+				labels: cloneLabels(entry.labels),
+				value: entry.value,
+			})),
+		};
+	}
+}
+
+export class Gauge extends Metric {
+	private values = new Map<string, StoredValue>();
+
+	set(labels: MetricLabels = {}, value: number): void {
+		if (!Number.isFinite(value)) return;
+		const key = labelKey(labels);
+		// Delete before set to move key to end of insertion order (MRU).
+		this.values.delete(key);
+		this.values.set(key, { labels: normalizeLabels(labels), value });
+		enforceLabelCap(this.values, this.name);
+	}
+
+	add(labels: MetricLabels = {}, delta: number): void {
+		if (!Number.isFinite(delta)) return;
+		this.set(labels, this.value(labels) + delta);
+	}
+
+	value(labels: MetricLabels = {}): number {
+		return this.values.get(labelKey(labels))?.value ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return {
+			type: "gauge",
+			name: this.name,
+			description: this.description,
+			values: [...this.values.values()].map((entry) => ({
+				labels: cloneLabels(entry.labels),
+				value: entry.value,
+			})),
+		};
+	}
+}
+
+export class Histogram extends Metric {
+	private readonly buckets: number[];
+	private observations = new Map<string, StoredHistogram>();
+
+	constructor(name: string, description: string, buckets?: number[]) {
+		super(name, description);
+		const source = buckets?.length ? buckets : [...DEFAULT_HISTOGRAM_BUCKETS];
+		this.buckets = [...new Set(source.filter((bucket) => Number.isFinite(bucket)).sort((left, right) => left - right))];
+	}
+
+	observe(labels: MetricLabels = {}, value: number): void {
+		if (!Number.isFinite(value)) return;
+		const key = labelKey(labels);
+		const existing = this.observations.get(key);
+		const current = existing ?? {
+			labels: normalizeLabels(labels),
+			counts: new Array(this.buckets.length + 1).fill(0) as number[],
+			sum: 0,
+			count: 0,
+		};
+		const bucketIndex = this.buckets.findIndex((bucket) => value <= bucket);
+		current.counts[bucketIndex === -1 ? this.buckets.length : bucketIndex] =
+			(current.counts[bucketIndex === -1 ? this.buckets.length : bucketIndex] ?? 0) + 1;
+		current.sum += value;
+		current.count += 1;
+		if (!existing) this.observations.set(key, current);
+		enforceLabelCap(this.observations, this.name);
+	}
+
+	quantile(labels: MetricLabels = {}, q: number): number {
+		const obs = this.observations.get(labelKey(labels));
+		if (!obs || obs.count === 0 || !Number.isFinite(q)) return Number.NaN;
+		const bounded = Math.min(1, Math.max(0, q));
+		const target = Math.max(1, bounded * obs.count);
+		let cumulative = 0;
+		for (let index = 0; index < obs.counts.length; index += 1) {
+			const count = obs.counts[index] ?? 0;
+			cumulative += count;
+			if (cumulative >= target) {
+				const previous = cumulative - count;
+				const lower = index === 0 ? 0 : (this.buckets[index - 1] ?? this.buckets.at(-1) ?? 0);
+				const upper =
+					index < this.buckets.length ? (this.buckets[index] ?? lower) : Math.max(lower, obs.sum / Math.max(1, obs.count));
+				const fraction = count === 0 ? 0 : (target - previous) / Math.max(1, count);
+				return lower + fraction * (upper - lower);
+			}
+		}
+		return this.buckets.at(-1) ?? Number.NaN;
+	}
+
+	count(labels: MetricLabels = {}): number {
+		return this.observations.get(labelKey(labels))?.count ?? 0;
+	}
+
+	snapshot(): MetricSnapshot {
+		return {
+			type: "histogram",
+			name: this.name,
+			description: this.description,
+			values: [...this.observations.values()].map((entry) => ({
+				labels: cloneLabels(entry.labels),
+				buckets: [...this.buckets, Number.POSITIVE_INFINITY],
+				counts: [...entry.counts],
+				sum: entry.sum,
+				count: entry.count,
+				quantiles: {
+					p50: this.quantile(entry.labels, 0.5),
+					p95: this.quantile(entry.labels, 0.95),
+					p99: this.quantile(entry.labels, 0.99),
+				},
+			})),
+		};
+	}
+}

package/src/plugins/plugin-define.ts

@@ -0,0 +1,6 @@
+/**
+ * Simplifies plugin struct definition.
+ */
+export function definePlugin<T extends { name: string; enablers: readonly string[] }>(spec: T): T {
+	return spec;
+}

package/src/plugins/plugin-registry.ts

@@ -0,0 +1,32 @@
+export interface Plugin {
+	readonly name: string;
+	readonly enablers: readonly string[];
+	readonly entryPatterns?: readonly string[];
+	readonly configPatterns?: readonly string[];
+	readonly toolingDependencies?: readonly string[];
+	readonly pathAliases?: readonly string[][];
+	readonly virtualModulePrefixes?: readonly string[];
+}
+
+export class PluginRegistry {
+	private plugins: Plugin[] = [];
+
+	register(plugin: Plugin): void {
+		this.plugins.push(plugin);
+	}
+
+	activePlugins(allDeps: string[]): Plugin[] {
+		return this.plugins.filter((p) =>
+			p.enablers.some((enabler) => {
+				if (enabler.endsWith("/")) {
+					return allDeps.some((d) => d.startsWith(enabler));
+				}
+				return allDeps.includes(enabler);
+			}),
+		);
+	}
+
+	allPlugins(): Plugin[] {
+		return [...this.plugins];
+	}
+}

package/src/plugins/plugins/index.ts

@@ -0,0 +1,3 @@
+export { NextJsPlugin } from "./nextjs.ts";
+export { VitePlugin } from "./vite.ts";
+export { VitestPlugin } from "./vitest.ts";

package/src/plugins/plugins/nextjs.ts

@@ -0,0 +1,19 @@
+import { definePlugin } from "../plugin-define.ts";
+
+export const NextJsPlugin = definePlugin({
+	name: "nextjs",
+	enablers: ["next"],
+	entryPatterns: [
+		"src/app/**/*.{ts,tsx}",
+		"src/pages/**/*.{ts,tsx}",
+		"src/app/**/page.{ts,tsx}",
+		"src/app/**/layout.{ts,tsx}",
+		"src/app/**/route.{ts,tsx}",
+		"middleware.{ts,js}",
+		"next.config.{ts,js,mjs}",
+	],
+	configPatterns: ["next.config.{ts,js,mjs}"],
+	toolingDependencies: ["next", "@next/font", "@next/mdx"],
+	pathAliases: [["~", "src"]],
+	virtualModulePrefixes: ["next:"],
+});

package/src/plugins/plugins/vite.ts

@@ -0,0 +1,10 @@
+import { definePlugin } from "../plugin-define.ts";
+
+export const VitePlugin = definePlugin({
+	name: "vite",
+	enablers: ["vite", "rolldown-vite"],
+	entryPatterns: ["src/main.{ts,tsx,js,jsx}", "src/index.{ts,tsx,js,jsx}", "index.html"],
+	configPatterns: ["vite.config.{ts,js,mts,mjs}"],
+	toolingDependencies: ["vite"],
+	virtualModulePrefixes: ["virtual:"],
+});

package/src/plugins/plugins/vitest.ts

@@ -0,0 +1,9 @@
+import { definePlugin } from "../plugin-define.ts";
+
+export const VitestPlugin = definePlugin({
+	name: "vitest",
+	enablers: ["vitest"],
+	entryPatterns: ["**/*.test.{ts,tsx}", "**/*.spec.{ts,tsx}", "src/**/*.test.{ts,tsx}", "src/**/*.spec.{ts,tsx}"],
+	configPatterns: ["vitest.config.{ts,js,mjs}", "vite.config.ts"],
+	toolingDependencies: ["vitest"],
+});