pi-harness-runtime 0.5.0 → 0.6.3-beta.13

package/packages/model-registry/test/registry.test.ts

@@ -0,0 +1,342 @@
+/**
+ * Model Registry Tests (RFC-0053)
+ */
+
+import { describe, it, expect, beforeEach } from "bun:test";
+import {
+	createModelRegistry,
+	type InMemoryModelRegistry,
+	DEFAULT_MODELS,
+	findModels,
+	getCheapestModel,
+	getLargestContextModel,
+	filterByCostEfficiency,
+	calculateCost,
+	getCostPer1M,
+	calculateSavings,
+} from "../src/index.js";
+import type { ModelInfo } from "../src/types.js";
+
+describe("ModelRegistry", () => {
+	let registry: InMemoryModelRegistry;
+
+	beforeEach(() => {
+		registry = createModelRegistry(true);
+	});
+
+	describe("initialization", () => {
+		it("should load default models", () => {
+			const models = registry.list();
+			expect(models.length).toBeGreaterThan(0);
+		});
+
+		it("should have OpenAI models", () => {
+			const models = registry.list("openai");
+			expect(models.length).toBeGreaterThan(0);
+		});
+
+		it("should have Anthropic models", () => {
+			const models = registry.list("anthropic");
+			expect(models.length).toBeGreaterThan(0);
+		});
+
+		it("should have MiniMax models", () => {
+			const models = registry.list("minimax");
+			expect(models.length).toBeGreaterThan(0);
+		});
+	});
+
+	describe("register and unregister", () => {
+		it("should register a new model", () => {
+			const newModel: ModelInfo = {
+				id: "test-model",
+				providerId: "test-provider",
+				name: "Test Model",
+				contextWindow: 100000,
+				maxOutputTokens: 4096,
+				pricing: {
+					currency: "USD",
+					inputPer1M: 0.5,
+					outputPer1M: 1.0,
+				},
+				capabilities: ["code_generation"],
+				aliases: ["test", "test-model-v1"],
+				status: "active",
+			};
+
+			registry.register(newModel);
+
+			const retrieved = registry.get("test-provider", "test-model");
+			expect(retrieved).toBeDefined();
+			expect(retrieved!.name).toBe("Test Model");
+		});
+
+		it("should unregister a model", () => {
+			registry.unregister("openai", "gpt-4o");
+
+			const retrieved = registry.get("openai", "gpt-4o");
+			expect(retrieved).toBeUndefined();
+		});
+
+		it("should track events", () => {
+			const events: any[] = [];
+			registry.onEvent((e) => events.push(e));
+
+			const newModel: ModelInfo = {
+				id: "event-test",
+				providerId: "test",
+				name: "Event Test",
+				contextWindow: 50000,
+				maxOutputTokens: 2048,
+				pricing: { currency: "USD", inputPer1M: 1, outputPer1M: 2 },
+				capabilities: [],
+				aliases: [],
+				status: "active",
+			};
+
+			registry.register(newModel);
+
+			expect(events).toContainEqual({
+				type: "model.registered",
+				providerId: "test",
+				modelId: "event-test",
+			});
+		});
+	});
+
+	describe("get and getByAlias", () => {
+		it("should get model by provider and ID", () => {
+			const model = registry.get("anthropic", "claude-sonnet-4");
+			expect(model).toBeDefined();
+			expect(model!.name).toBe("Claude Sonnet 4");
+		});
+
+		it("should get model by alias", () => {
+			const model = registry.getByAlias("sonnet");
+			expect(model).toBeDefined();
+			expect(model!.id).toBe("claude-sonnet-4");
+		});
+
+		it("should return undefined for unknown alias", () => {
+			const model = registry.getByAlias("unknown-model");
+			expect(model).toBeUndefined();
+		});
+	});
+
+	describe("list and listActive", () => {
+		it("should list all models", () => {
+			const models = registry.list();
+			expect(models.length).toBe(DEFAULT_MODELS.length);
+		});
+
+		it("should list models by provider", () => {
+			const models = registry.list("openai");
+			expect(models.every((m) => m.providerId === "openai")).toBe(true);
+		});
+
+		it("should list only active models", () => {
+			const active = registry.listActive();
+			expect(active.every((m) => m.status === "active")).toBe(true);
+		});
+	});
+
+	describe("find", () => {
+		it("should find by provider", () => {
+			const models = registry.find({ providerId: "openai" });
+			expect(models.every((m) => m.providerId === "openai")).toBe(true);
+		});
+
+		it("should find by minimum context window", () => {
+			const models = registry.find({ minContextWindow: 200000 });
+			expect(models.every((m) => m.contextWindow >= 200000)).toBe(true);
+		});
+
+		it("should find by max cost", () => {
+			const models = registry.find({ maxCostPer1M: 5 });
+			expect(models.every((m) => getCostPer1M(m.pricing) <= 5)).toBe(true);
+		});
+
+		it("should find by capabilities", () => {
+			const models = registry.find({ capabilities: ["vision"] });
+			expect(models.every((m) => m.capabilities.includes("vision"))).toBe(true);
+		});
+
+		it("should find by status", () => {
+			const models = registry.find({ status: "active" });
+			expect(models.every((m) => m.status === "active")).toBe(true);
+		});
+
+		it("should sort by cost ascending", () => {
+			const models = registry.find({});
+			for (let i = 1; i < models.length; i++) {
+				expect(getCostPer1M(models[i - 1].pricing)).toBeLessThanOrEqual(
+					getCostPer1M(models[i].pricing),
+				);
+			}
+		});
+
+		it("should emit query event", () => {
+			const events: any[] = [];
+			registry.onEvent((e) => events.push(e));
+
+			registry.find({ providerId: "openai" });
+
+			expect(events).toContainEqual(
+				expect.objectContaining({
+					type: "model.queried",
+					filters: { providerId: "openai" },
+				}),
+			);
+		});
+	});
+
+	describe("updateStatus", () => {
+		it("should update model status", () => {
+			registry.updateStatus("openai", "gpt-4o", "deprecated");
+
+			const model = registry.get("openai", "gpt-4o");
+			expect(model!.status).toBe("deprecated");
+			expect(model!.deprecatedAt).toBeDefined();
+		});
+
+		it("should emit status change event", () => {
+			const events: any[] = [];
+			registry.onEvent((e) => events.push(e));
+
+			registry.updateStatus("openai", "gpt-4o", "disabled");
+
+			expect(events).toContainEqual(
+				expect.objectContaining({
+					type: "model.status_changed",
+					providerId: "openai",
+					modelId: "gpt-4o",
+					status: "disabled",
+				}),
+			);
+		});
+	});
+});
+
+describe("Cost Functions", () => {
+	describe("calculateCost", () => {
+		it("should calculate basic cost", () => {
+			const model: ModelInfo = {
+				id: "test",
+				providerId: "test",
+				name: "Test",
+				contextWindow: 100000,
+				maxOutputTokens: 4096,
+				pricing: { currency: "USD", inputPer1M: 1, outputPer1M: 2 },
+				capabilities: [],
+				aliases: [],
+				status: "active",
+			};
+
+			// 1M input, 500k output
+			const cost = calculateCost(model, 1_000_000, 500_000);
+			expect(cost).toBe(2); // $1 + $1
+		});
+
+		it("should apply batch discount", () => {
+			const model: ModelInfo = {
+				id: "test",
+				providerId: "test",
+				name: "Test",
+				contextWindow: 100000,
+				maxOutputTokens: 4096,
+				pricing: {
+					currency: "USD",
+					inputPer1M: 1,
+					outputPer1M: 2,
+					batchInputPer1M: 0.1,
+				},
+				capabilities: [],
+				aliases: [],
+				status: "active",
+			};
+
+			const withBatch = calculateCost(model, 1_000_000, 0, { useBatch: true });
+			const withoutBatch = calculateCost(model, 1_000_000, 0);
+
+			expect(withBatch).toBeLessThan(withoutBatch);
+		});
+	});
+
+	describe("getCostPer1M", () => {
+		it("should sum input and output costs", () => {
+			const cost = getCostPer1M({
+				currency: "USD",
+				inputPer1M: 1,
+				outputPer1M: 2,
+			});
+			expect(cost).toBe(3);
+		});
+	});
+
+	describe("calculateSavings", () => {
+		it("should calculate absolute and percentage savings", () => {
+			const current: ModelInfo = {
+				id: "current",
+				providerId: "test",
+				name: "Current",
+				contextWindow: 100000,
+				maxOutputTokens: 4096,
+				pricing: { currency: "USD", inputPer1M: 1, outputPer1M: 2 },
+				capabilities: [],
+				aliases: [],
+				status: "active",
+			};
+
+			const cheaper: ModelInfo = {
+				id: "cheaper",
+				providerId: "test",
+				name: "Cheaper",
+				contextWindow: 100000,
+				maxOutputTokens: 4096,
+				pricing: { currency: "USD", inputPer1M: 0.5, outputPer1M: 1 },
+				capabilities: [],
+				aliases: [],
+				status: "active",
+			};
+
+			const savings = calculateSavings(current, cheaper, 1_000_000, 500_000);
+			// current: 1M@$1 + 0.5M@$2 = $2
+			// cheaper: 1M@$0.5 + 0.5M@$1 = $1
+			// savings: $2 - $1 = $1
+			expect(savings.absolute).toBe(1);
+			expect(savings.percentage).toBe(50); // 50%
+		});
+	});
+});
+
+describe("Query Functions", () => {
+	let models: ModelInfo[];
+
+	beforeEach(() => {
+		const registry = createModelRegistry(true);
+		models = registry.list();
+	});
+
+	describe("getCheapestModel", () => {
+		it("should return the cheapest model", () => {
+			const cheapest = getCheapestModel(models);
+			expect(cheapest).toBeDefined();
+		});
+	});
+
+	describe("getLargestContextModel", () => {
+		it("should return the model with largest context", () => {
+			const largest = getLargestContextModel(models);
+			expect(largest).toBeDefined();
+			const maxWindow = Math.max(...models.map((m) => m.contextWindow));
+			expect(largest!.contextWindow).toBe(maxWindow);
+		});
+	});
+
+	describe("filterByCostEfficiency", () => {
+		it("should sort by cost per context", () => {
+			const sorted = filterByCostEfficiency(models);
+			expect(sorted.length).toBe(models.length);
+		});
+	});
+});

package/packages/model-registry/tsconfig.json

@@ -0,0 +1,22 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "types": [
+      "node"
+    ],
+    "lib": [
+      "ES2022",
+      "DOM"
+    ]
+  },
+  "include": [
+    "src/**/*"
+  ],
+  "exclude": [
+    "node_modules",
+    "dist",
+    "test"
+  ]
+}

package/packages/observability/README.md

@@ -0,0 +1,241 @@
+# @pi/observability
+
+Comprehensive observability system with structured logging, distributed tracing, metrics collection, health monitoring, and alerting.
+
+## Features
+
+- **Structured Logging** - JSON and pretty-printed logs with correlation IDs
+- **Distributed Tracing** - OpenTelemetry-compatible tracing with spans and events
+- **Metrics Collection** - Prometheus-compatible metrics (counters, gauges, histograms)
+- **Health Monitoring** - Liveness and readiness probes for Kubernetes
+- **Alerting Engine** - Rule-based alerts with multiple action types
+
+## Installation
+
+```bash
+npm install @pi/observability
+```
+
+## Quick Start
+
+### Structured Logging
+
+```typescript
+import { createLogger } from "@pi/observability";
+
+const logger = createLogger({
+  level: "info",
+  format: "json",
+  output: "stdout"
+});
+
+// Basic logging
+logger.info("Application started");
+logger.error("Request failed", { requestId: "123" });
+
+// With correlation
+const jobLogger = logger.withJob("job-123");
+jobLogger.info("Processing task");
+```
+
+### Distributed Tracing
+
+```typescript
+import { createTracer } from "@pi/observability";
+
+const tracer = createTracer({
+  serviceName: "my-service",
+  exporter: "console",
+  sampleRate: 1.0
+});
+
+// Trace a function
+const result = await tracer.trace("processOrder", async (span) => {
+  span.setAttribute("orderId", order.id);
+  
+  await tracer.trace("validateOrder", async () => {
+    // validation logic
+  });
+  
+  return result;
+});
+```
+
+### Metrics
+
+```typescript
+import { createMetrics, createBuiltInMetrics } from "@pi/observability";
+
+const metrics = createMetrics({ serviceName: "my-service" });
+createBuiltInMetrics(metrics);
+
+// Counter
+metrics.incrementCounter("requests_total", { endpoint: "/api" });
+
+// Gauge
+metrics.setGauge("active_connections", 42);
+
+// Histogram
+metrics.observeHistogram("request_duration_seconds", 0.125, { method: "GET" });
+
+// Export metrics
+const prometheusOutput = metrics.exportPrometheus();
+```
+
+### Health Checks
+
+```typescript
+import { createHealthMonitor, createMemoryHealthCheck } from "@pi/observability";
+
+const health = createHealthMonitor("my-service", "1.0.0");
+
+// Register checks
+health.registerHealthCheck("memory", createMemoryHealthCheck("memory", 90));
+
+// Kubernetes probes
+app.get("/health/liveness", async (req, res) => {
+  const result = await health.checkLiveness();
+  res.status(result.status === "healthy" ? 200 : 503).json(result);
+});
+
+app.get("/health/readiness", async (req, res) => {
+  const result = await health.checkReadiness();
+  res.status(result.status === "healthy" ? 200 : 503).json(result);
+});
+```
+
+### Alerting
+
+```typescript
+import { createAlertEngine } from "@pi/observability";
+
+const alertEngine = createAlertEngine();
+
+alertEngine.registerRule({
+  name: "high_error_rate",
+  condition: {
+    type: "threshold",
+    metric: "error_rate",
+    operator: ">",
+    value: 0.05
+  },
+  severity: "critical",
+  cooldown: 300000, // 5 minutes
+  actions: [
+    { type: "log", level: "error" },
+    { type: "notify", channel: "slack", message: "High error rate detected" }
+  ]
+});
+
+// Evaluate alerts
+const alerts = await alertEngine.evaluate(metrics);
+```
+
+## API Reference
+
+### Logger
+
+```typescript
+const logger = createLogger({
+  level: "debug" | "info" | "warn" | "error" | "fatal",
+  format: "json" | "pretty",
+  output: "stdout" | "stderr" | "file",
+  filePath?: string,
+  defaultMeta?: Record<string, unknown>
+});
+
+logger.debug(message, meta?);
+logger.info(message, meta?);
+logger.warn(message, meta?);
+logger.error(message, error?, meta?);
+logger.fatal(message, error?, meta?);
+
+logger.child(defaults);  // Create child logger
+logger.withJob(jobId);   // Attach job ID
+logger.withTask(taskId); // Attach task ID
+logger.withCorrelationId(id); // Set correlation ID
+```
+
+### Tracer
+
+```typescript
+const tracer = createTracer({
+  serviceName: string,
+  exporter: "console" | "otlp" | "jaeger",
+  endpoint?: string,
+  sampleRate?: number
+});
+
+tracer.startSpan(name, options?);
+tracer.trace(name, fn);
+tracer.trace(name, options, fn);
+tracer.traceSync(name, fn);
+```
+
+### Metrics
+
+```typescript
+const metrics = createMetrics({
+  serviceName: string,
+  prefix?: string,
+  defaultLabels?: Labels
+});
+
+metrics.registerCounter(name, help, labels?);
+metrics.incrementCounter(name, labels?);
+metrics.registerGauge(name, help, labels?);
+metrics.setGauge(name, value, labels?);
+metrics.registerHistogram(name, help, buckets?, labels?);
+metrics.observeHistogram(name, value, labels?);
+
+metrics.asyncTimer(name, fn, labels?);
+metrics.timer(name, fn, labels?);
+
+metrics.exportPrometheus();
+metrics.exportJSON();
+```
+
+### Health Monitor
+
+```typescript
+const health = createHealthMonitor(serviceName, version);
+
+health.registerHealthCheck(name, check, critical?);
+health.checkHealth(name);
+health.checkHealthAll();
+health.checkLiveness();
+health.checkReadiness();
+```
+
+### Alert Engine
+
+```typescript
+const alertEngine = createAlertEngine(logger?);
+
+alertEngine.registerRule(rule);
+alertEngine.unregisterRule(name);
+alertEngine.evaluate(metrics);
+alertEngine.getActiveAlerts();
+alertEngine.getAlertHistory(limit?);
+```
+
+## Metrics Format
+
+Prometheus format example:
+
+```
+# HELP my_service_requests_total Total requests
+# TYPE my_service_requests_total counter
+my_service_requests_total{endpoint="/api"} 1234
+
+# HELP my_service_request_duration_seconds Request duration
+# TYPE my_service_request_duration_seconds histogram
+my_service_request_duration_seconds_bucket{le="0.1"} 1000
+my_service_request_duration_seconds_bucket{le="0.5"} 1200
+my_service_request_duration_seconds_sum 150.5
+my_service_request_duration_seconds_count 1234
+```
+
+## License
+
+MIT

package/packages/observability/dist/alerts.d.ts

@@ -0,0 +1,99 @@
+/**
+ * Observability Package - Alert Engine
+ *
+ * Alert rule evaluation and alerting.
+ */
+import type { Alert, AlertRule } from "./types.js";
+import { Logger } from "./logger.js";
+export declare class AlertEngine {
+    private readonly rules;
+    private readonly activeAlerts;
+    private readonly alertHistory;
+    private readonly logger;
+    constructor(logger?: Logger, _maxHistory?: number);
+    /**
+     * Register an alert rule
+     */
+    registerRule(rule: AlertRule): void;
+    /**
+     * Unregister an alert rule
+     */
+    unregisterRule(name: string): void;
+    /**
+     * Get a rule by name
+     */
+    getRule(name: string): AlertRule | undefined;
+    /**
+     * Enable or disable a rule
+     */
+    setRuleEnabled(name: string, enabled: boolean): void;
+    /**
+     * Evaluate all rules
+     */
+    evaluate(metrics: Record<string, number>): Promise<Alert[]>;
+    /**
+     * Evaluate a single condition
+     */
+    private evaluateCondition;
+    /**
+     * Create an alert from a rule
+     */
+    private createAlert;
+    /**
+     * Format alert message
+     */
+    private formatAlertMessage;
+    /**
+     * Trigger alert actions
+     */
+    private triggerActions;
+    /**
+     * Execute a single action
+     */
+    private executeAction;
+    /**
+     * Log an alert
+     */
+    private logAlert;
+    /**
+     * Send a notification
+     */
+    private sendNotification;
+    /**
+     * Send a webhook
+     */
+    private sendWebhook;
+    /**
+     * Execute a command
+     */
+    private executeCommand;
+    /**
+     * Get active alerts
+     */
+    getActiveAlerts(): Alert[];
+    /**
+     * Get alert history
+     */
+    getAlertHistory(limit?: number): Alert[];
+    /**
+     * Clear alert history
+     */
+    clearHistory(): void;
+    /**
+     * List all rules
+     */
+    listRules(): AlertRule[];
+    /**
+     * Get rule count
+     */
+    getRuleCount(): {
+        total: number;
+        enabled: number;
+        disabled: number;
+    };
+}
+/**
+ * Create an alert engine
+ */
+export declare function createAlertEngine(logger?: Logger): AlertEngine;
+//# sourceMappingURL=alerts.d.ts.map

package/packages/observability/dist/alerts.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"alerts.d.ts","sourceRoot":"","sources":["../src/alerts.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,KAAK,EAA+B,SAAS,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAIrC,qBAAa,WAAW;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAqC;IAC3D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAiC;IAC9D,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;gBAEpB,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,SAAO;IAI/C;;OAEG;IACH,YAAY,CAAC,IAAI,EAAE,SAAS,GAAG,IAAI;IAKnC;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAKlC;;OAEG;IACH,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,SAAS;IAI5C;;OAEG;IACH,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,IAAI;IAOpD;;OAEG;IACG,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAwCjE;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA2CzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAUnB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAa1B;;OAEG;YACW,cAAc;IAS5B;;OAEG;YACW,aAAa;IAuB3B;;OAEG;IACH,OAAO,CAAC,QAAQ;IAwBhB;;OAEG;YACW,gBAAgB;IAW9B;;OAEG;YACW,WAAW;IA4BzB;;OAEG;YACW,cAAc;IAO5B;;OAEG;IACH,eAAe,IAAI,KAAK,EAAE;IAI1B;;OAEG;IACH,eAAe,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE;IAQxC;;OAEG;IACH,YAAY,IAAI,IAAI;IAIpB;;OAEG;IACH,SAAS,IAAI,SAAS,EAAE;IAIxB;;OAEG;IACH,YAAY,IAAI;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAA;KAAE;CAkBpE;AAID;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAE9D"}