pi-mega-compact 0.20.78 → 0.20.80

package/extensions/mega-events/analytics-handler.ts

@@ -0,0 +1,236 @@
+/**
+ * analytics-handler.ts — PMA-2 verified ingestion adapter.
+ *
+ * Registers event handlers that capture request-lifecycle facts from verified
+ * pi event seams (proven in PMA-0) into the isolated analytics.db (built in
+ * PMA-1). Mirrors the perf-handler.ts pattern: closure over `runtime`, non-fatal
+ * try/catch on every handler, flag-gated.
+ *
+ * Event-to-fact mapping (all seams verified in docs/specs/pma-0-discovery-findings.md):
+ *   turn_start               → request_started (observedAt, provider/model)
+ *   before_provider_request  → provider_selected (same correlationId)
+ *   message_update[text_start] → TTFT stamp (attached to terminal fact)
+ *   turn_end                 → request_completed/request_failed (tokens, latency, ttft, status)
+ *
+ * Everything is best-effort: AppendResult carries accepted/duplicate/failed
+ * without throwing into the host. When the flag is OFF, no handlers are
+ * registered and no DB is opened.
+ *
+ * PREVENT-PI-004: no network. PREVENT-PI-003: no system-role messages.
+ */
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { type MegaRuntime } from "../mega-runtime.js";
+import { type MegaConfig } from "../mega-config.js";
+import {
+	createAnalyticsStore,
+	closeAllAnalyticsDbs,
+	type AnalyticsStore,
+	type RequestEventFact,
+	type RequestEventKind,
+	type QualityNote,
+} from "../../src/store/analytics/index.js";
+
+// ── Per-stateDir store cache (mirrors mega-turn-store.ts storeFor) ─────
+
+const stores = new Map<string, AnalyticsStore>();
+
+function storeFor(stateDir: string): AnalyticsStore {
+	let s = stores.get(stateDir);
+	if (!s) {
+		s = createAnalyticsStore({ stateDir });
+		stores.set(stateDir, s);
+	}
+	return s;
+}
+
+/** Close all cached analytics stores (test teardown / shutdown). */
+export function closeAnalyticsStores(): void {
+	closeAllAnalyticsDbs();
+	stores.clear();
+}
+
+// ── Per-runtime PMA-2 timing state ────────────────────────────────────
+// Kept off MegaRuntime (a module-private WeakMap) so runtime.ts stays a
+// pure field-declaration shell under the soft-limit headroom gate. The
+// adapter is the sole reader/writer of this transient per-turn state.
+
+interface PmaTiming {
+	providerStart: number;
+	ttft: number;
+	correlationId: string | null;
+}
+
+const pmaTimings = new WeakMap<object, PmaTiming>();
+
+function getPma(rt: object): PmaTiming {
+	let s = pmaTimings.get(rt);
+	if (!s) {
+		s = { providerStart: 0, ttft: 0, correlationId: null };
+		pmaTimings.set(rt, s);
+	}
+	return s;
+}
+
+/** @internal Test-only accessor — production code uses getPma() in-module. */
+export function __pmaTimingForTest(rt: object): PmaTiming {
+	return getPma(rt);
+}
+
+// ── Usage block narrowing (same as perf-handler) ──────────────────────
+
+interface UsageBlock {
+	input: number;
+	output: number;
+	cacheRead: number;
+	cacheWrite: number;
+}
+
+function usageOf(msg: { role?: string; usage?: UsageBlock }): UsageBlock | null {
+	if (msg.role === "assistant" && msg.usage) return msg.usage;
+	return null;
+}
+
+// ── Helpers ───────────────────────────────────────────────────────────
+
+/** Build a QualityNote with provider/model availability. */
+function qualityFor(provider: string | undefined, model: string | undefined): QualityNote {
+	if (!provider || !model) return { unavailable: true, note: "provider/model not captured" };
+	return {};
+}
+
+// ── Registration ──────────────────────────────────────────────────────
+
+/** Register the analytics event handlers. Flag-gated: OFF = complete no-op. */
+export function registerAnalyticsHandler(
+	pi: ExtensionAPI,
+	runtime: MegaRuntime,
+	config: MegaConfig,
+): void {
+	if (!config.providerModelAnalytics) return; // flag OFF — no DB, no handlers
+
+	// ── turn_start: stamp the request start + emit request_started ──────
+
+	pi.on("turn_start", async (event) => {
+		try {
+			const now = Date.now();
+			const corrId = `req_${runtime.rt.sessionId}_${event.turnIndex}`;
+			const pma = getPma(runtime);
+			pma.correlationId = corrId;
+			pma.providerStart = 0; // reset; stamped at before_provider_request
+			pma.ttft = 0;
+
+			const provider = runtime.currentModel?.provider;
+			const model = runtime.currentModel?.modelId;
+			const fact: RequestEventFact = {
+				id: `${corrId}_started`,
+				correlationId: corrId,
+				sessionId: runtime.rt.sessionId,
+				turnId: String(event.turnIndex),
+				eventKind: "request_started",
+				observedAt: now,
+				provider,
+				model,
+				source: "host_adapter",
+				quality: qualityFor(provider, model),
+			};
+			storeFor(runtime.currentStateDir).asWriter().appendRequestEvent(fact);
+		} catch {
+			/* non-fatal: analytics never breaks the agent loop */
+		}
+	});
+
+	// ── before_provider_request: stamp provider start + emit provider_selected
+
+	pi.on("before_provider_request", async () => {
+		try {
+			const pma = getPma(runtime);
+			pma.providerStart = Date.now();
+			const corrId = pma.correlationId;
+			if (!corrId) return;
+			const provider = runtime.currentModel?.provider;
+			const model = runtime.currentModel?.modelId;
+			const fact: RequestEventFact = {
+				id: `${corrId}_selected`,
+				correlationId: corrId,
+				sessionId: runtime.rt.sessionId,
+				eventKind: "provider_selected",
+				observedAt: pma.providerStart,
+				provider,
+				model,
+				source: "host_adapter",
+				quality: qualityFor(provider, model),
+			};
+			storeFor(runtime.currentStateDir).asWriter().appendRequestEvent(fact);
+		} catch {
+			/* non-fatal */
+		}
+	});
+
+	// ── message_update: capture TTFT on first text token ────────────────
+	// Net-new seam — no existing handler registers message_update.
+	// Proven available in PMA-0 (agent-loop.js:201-227).
+
+	pi.on("message_update", async (event) => {
+		try {
+			const pma = getPma(runtime);
+			if (pma.ttft > 0) return; // already captured this turn
+			if (pma.providerStart <= 0) return; // no provider start stamp
+			const chunkType = (event as { assistantMessageEvent?: { type?: string } })
+				.assistantMessageEvent?.type;
+			if (chunkType !== "text_start" && chunkType !== "text_delta") return;
+			pma.ttft = Date.now() - pma.providerStart;
+		} catch {
+			/* non-fatal */
+		}
+	});
+
+	// ── turn_end: emit terminal fact with tokens, latency, ttft, status ─
+
+	pi.on("turn_end", async (event) => {
+		try {
+			const pma = getPma(runtime);
+			const corrId = pma.correlationId;
+			if (!corrId) return;
+
+			const now = Date.now();
+			const durationMs = runtime.perfTurnStart > 0 ? now - runtime.perfTurnStart : undefined;
+			const u = usageOf(event.message as { role?: string; usage?: UsageBlock });
+			const stopReason = (event.message as { stopReason?: string }).stopReason;
+			const provider = runtime.currentModel?.provider;
+			const model = runtime.currentModel?.modelId;
+
+			const isFailed = stopReason === "error" || stopReason === "aborted";
+			const eventKind: RequestEventKind = isFailed ? "request_failed" : "request_completed";
+
+			const fact: RequestEventFact = {
+				id: `${corrId}_${eventKind}`,
+				correlationId: corrId,
+				sessionId: runtime.rt.sessionId,
+				turnId: String(event.turnIndex),
+				eventKind,
+				observedAt: now,
+				provider,
+				model,
+				status: stopReason ?? undefined,
+				inputTokens: u?.input,
+				outputTokens: u?.output,
+				cacheReadTokens: u?.cacheRead,
+				cacheWriteTokens: u?.cacheWrite,
+				durationMs,
+				ttftMs: pma.ttft > 0 ? pma.ttft : undefined,
+				source: "host_adapter",
+				quality: {
+					...qualityFor(provider, model),
+					...(pma.ttft > 0 ? {} : { note: "TTFT not captured (no first-token event)" }),
+				},
+			};
+			storeFor(runtime.currentStateDir).asWriter().appendRequestEvent(fact);
+
+			// Reset per-turn state.
+			pma.correlationId = null;
+			pma.ttft = 0;
+		} catch {
+			/* non-fatal */
+		}
+	});
+}

package/extensions/mega-events/register.ts

@@ -13,6 +13,7 @@ import { registerAgentHandlers } from "./agent-handlers.js";
 import { registerContextHandler } from "./context-handler.js";
 import { registerCompactHandlers } from "./compact-handlers.js";
 import { registerPerfHandler } from "./perf-handler.js";
+import { registerAnalyticsHandler } from "./analytics-handler.js";
 import { registerVectorCortexRender } from "../mega-context/vector-cortex.js";
 
 /**
@@ -49,4 +50,6 @@ export function registerEventHandlers(
 	registerContextHandler(pi, runtime, config);
 	registerCompactHandlers(pi, runtime, config);
 	registerPerfHandler(pi, runtime);
+	// PMA-2: analytics ingestion — flag-gated (OFF = no-op, no DB opened).
+	registerAnalyticsHandler(pi, runtime, config);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-mega-compact",
-  "version": "0.20.78",
+  "version": "0.20.80",
   "description": "Layered, local, vector-backed context compressor for pi — supersede/collapse/cluster compaction with deduped inline recall.",
   "type": "module",
   "license": "BSD-3-Clause",

package/src/queryExpansion.ts

@@ -200,6 +200,7 @@ function loadDb(stateDir: string): { db: import("node:sqlite").DatabaseSync } {
   const { DatabaseSync } = require("node:sqlite") as typeof import("node:sqlite");
   const { join } = require("node:path") as typeof import("node:path");
   const db = new DatabaseSync(join(stateDir, "mega-compact.db"));
+  db.exec("PRAGMA busy_timeout = 5000"); // tolerate concurrent-opener WAL contention (see store/sqlite/utils.ts)
   db.exec("PRAGMA journal_mode=WAL");
   _dbCache.set(stateDir, { db });
   return { db };

package/src/store/analytics/connection.ts

@@ -0,0 +1,128 @@
+/**
+ * connection.ts — PMA-1 connection manager for analytics.db.
+ *
+ * Own module-level connection cache, deliberately SEPARATE from the main
+ * memory store (openStore) and the turns store (openTurnDb) so an analytics-DB
+ * failure can never touch the authoritative memory DB or the turns DB (PMA §3.1).
+ *
+ * Mirrors src/store/turns/connection.ts. PREVENT-PI-004: node:sqlite in-process,
+ * no network. PREVENT-002: parameterized path binding.
+ */
+import { existsSync, mkdirSync, statSync } from "node:fs";
+import { join } from "node:path";
+import type { DatabaseSync } from "node:sqlite";
+import { DatabaseSync as Db } from "node:sqlite";
+import { initAnalyticsSchema } from "./schema.js";
+
+export const ANALYTICS_DB_FILE = "analytics.db";
+
+/** Resolve the analytics DB path, honoring the env override. */
+export function analyticsDbPath(stateDir: string): string {
+	const env = process.env.MEGACOMPACT_ANALYTICS_DB_PATH;
+	if (env && env.trim() !== "") return env.trim();
+	return join(stateDir, ANALYTICS_DB_FILE);
+}
+
+// Module-private cache — separate from openStore's cache and turns' cache.
+const cache = new Map<string, DatabaseSync>();
+
+/** File size helper (0 for in-memory). */
+function dbFileSize(dbPath: string): number {
+	try {
+		return statSync(dbPath).size;
+	} catch {
+		return 0;
+	}
+}
+
+/**
+ * Open (or reuse from cache) the analytics.db for `stateDir`.
+ * Sets PRAGMAs, initializes the schema, and caches the connection.
+ * In-memory mode (`:memory:`) is supported for tests.
+ */
+export function openAnalyticsDb(
+	stateDir: string,
+	options?: { dbPath?: string; inMemory?: boolean },
+): DatabaseSync {
+	const inMemory = options?.inMemory === true;
+	const resolvedPath = inMemory ? ":memory:" : (options?.dbPath ?? analyticsDbPath(stateDir));
+	const cacheKey = inMemory ? `:memory:${stateDir}` : resolvedPath;
+
+	const cached = cache.get(cacheKey);
+	if (cached) {
+		try {
+			cached.exec("SELECT 1");
+			return cached;
+		} catch {
+			// Stale handle — evict and re-open.
+			cache.delete(cacheKey);
+		}
+	}
+
+	// Ensure the state dir exists for file-backed mode.
+	if (!inMemory && !existsSync(stateDir)) {
+		mkdirSync(stateDir, { recursive: true });
+	}
+
+	const db = inMemory ? new Db(":memory:") : new Db(resolvedPath);
+	db.exec("PRAGMA busy_timeout = 5000");
+	db.exec("PRAGMA journal_mode = WAL");
+	db.exec("PRAGMA foreign_keys = ON");
+
+	initAnalyticsSchema(db);
+	cache.set(cacheKey, db);
+	return db;
+}
+
+/** Close and evict the analytics DB for `stateDir` (idempotent). */
+export function closeAnalyticsDb(
+	stateDir: string,
+	options?: { dbPath?: string; inMemory?: boolean },
+): void {
+	const inMemory = options?.inMemory === true;
+	const resolvedPath = inMemory ? ":memory:" : (options?.dbPath ?? analyticsDbPath(stateDir));
+	const cacheKey = inMemory ? `:memory:${stateDir}` : resolvedPath;
+
+	const db = cache.get(cacheKey);
+	if (!db) return;
+	try {
+		db.close();
+	} catch {
+		/* already closed */
+	}
+	cache.delete(cacheKey);
+}
+
+/** Close ALL cached analytics DBs (test teardown / graceful shutdown). */
+export function closeAllAnalyticsDbs(): void {
+	for (const db of cache.values()) {
+		try {
+			db.close();
+		} catch {
+			/* already closed */
+		}
+	}
+	cache.clear();
+}
+
+/** Report the file size of analytics.db for a state dir (0 if absent/in-memory). */
+export function analyticsDbSize(stateDir: string): number {
+	return dbFileSize(analyticsDbPath(stateDir));
+}
+
+/**
+ * SAVEPOINT-based transaction wrapper (nests safely under an outer transaction).
+ * Uses a distinct savepoint name from the turns store (`turns_tx`) and the main
+ * store (`mc_tx`) to avoid collisions.
+ */
+export function withTx(db: DatabaseSync, fn: () => void): void {
+	db.exec("SAVEPOINT analytics_tx");
+	try {
+		fn();
+		db.exec("RELEASE analytics_tx");
+	} catch (e) {
+		db.exec("ROLLBACK TO analytics_tx");
+		db.exec("RELEASE analytics_tx");
+		throw e;
+	}
+}

package/src/store/analytics/contract-compliance.test/_helpers.ts

@@ -0,0 +1,47 @@
+/** Helpers for the analytics compliance suite. */
+import type { AnalyticsStoreOptions, AnalyticsStore, RequestEventFact, MeasurementFact, IdentityObservation } from "../types.js";
+
+export type StoreFactory = (options: AnalyticsStoreOptions) => AnalyticsStore;
+
+let counter = 0;
+
+/** Create a valid RequestEventFact with sensible defaults + overrides. */
+export function makeRequestEvent(overrides: Partial<RequestEventFact> = {}): RequestEventFact {
+	return {
+		id: `evt_${++counter}`,
+		correlationId: "corr_test",
+		sessionId: "sess_test",
+		eventKind: "request_started",
+		observedAt: Date.now(),
+		provider: "anthropic",
+		model: "claude-sonnet-4",
+		source: "test",
+		quality: {},
+		...overrides,
+	};
+}
+
+/** Create a valid MeasurementFact. */
+export function makeMeasurement(overrides: Partial<MeasurementFact> = {}): MeasurementFact {
+	return {
+		observedAt: Date.now(),
+		sampleKind: "cache_hit_pct",
+		value: 72.5,
+		unit: "percent",
+		source: "test",
+		quality: {},
+		...overrides,
+	};
+}
+
+/** Create a valid IdentityObservation. */
+export function makeIdentity(overrides: Partial<IdentityObservation> = {}): IdentityObservation {
+	return {
+		observedAt: Date.now(),
+		provider: "anthropic",
+		model: "claude-sonnet-4",
+		source: "test",
+		metadata: { contextWindow: 200000 },
+		...overrides,
+	};
+}

package/src/store/analytics/index.ts

@@ -0,0 +1,61 @@
+/**
+ * index.ts — Barrel for src/store/analytics/ (PMA-1).
+ *
+ * Hosts import types + the factory. SQL schemas + internals stay private.
+ *
+ * PREVENT-PI-004: no network. PREVENT-002: parameterized SQL only.
+ */
+
+export type {
+	RequestEventKind,
+	FactSource,
+	QualityNote,
+	RequestEventFact,
+	MeasurementFact,
+	IdentityObservation,
+	AppendResult,
+	AnalyticsStatus,
+	PruneReport,
+	IntegrityReport,
+	MaintenanceReport,
+	BackupReport,
+	RetentionPolicy,
+	AnalyticsReader,
+	AnalyticsWriter,
+	AnalyticsAdmin,
+	AnalyticsStore,
+	AnalyticsStoreOptions,
+	AnalyticsStoreFactory,
+} from "./types.js";
+
+export { SqliteAnalyticsStore } from "./sqlite-store.js";
+export { InMemoryAnalyticsStore } from "./memory-store.js";
+
+export {
+	openAnalyticsDb,
+	closeAnalyticsDb,
+	closeAllAnalyticsDbs,
+	analyticsDbPath,
+	analyticsDbSize,
+	ANALYTICS_DB_FILE,
+	withTx,
+} from "./connection.js";
+
+export { initAnalyticsSchema, getAnalyticsSchemaVersion } from "./schema.js";
+
+import { SqliteAnalyticsStore } from "./sqlite-store.js";
+import { InMemoryAnalyticsStore } from "./memory-store.js";
+import type { AnalyticsStore, AnalyticsStoreOptions } from "./types.js";
+
+/**
+ * Factory: create an AnalyticsStore from options.
+ *
+ * - Default: SqliteAnalyticsStore backed by analytics.db in stateDir
+ * - inMemory: InMemoryAnalyticsStore (no file I/O)
+ */
+export function createAnalyticsStore(options: AnalyticsStoreOptions): AnalyticsStore {
+	if (options.inMemory) {
+		return new InMemoryAnalyticsStore(options);
+	}
+	return new SqliteAnalyticsStore(options);
+}

package/src/store/analytics/memory-store.ts

@@ -0,0 +1,161 @@
+/**
+ * memory-store.ts — InMemoryAnalyticsStore: Map-backed dual backend.
+ *
+ * Mirrors the turns store pattern (src/store/turns/memory-store.ts).
+ * Satisfies the same contract so the compliance suite runs against both
+ * backends unchanged.
+ */
+import type {
+	AnalyticsStore,
+	AnalyticsReader,
+	AnalyticsWriter,
+	AnalyticsAdmin,
+	AnalyticsStoreOptions,
+	RequestEventFact,
+	MeasurementFact,
+	IdentityObservation,
+	AppendResult,
+	AnalyticsStatus,
+	RetentionPolicy,
+	PruneReport,
+	IntegrityReport,
+	MaintenanceReport,
+	BackupReport,
+} from "./types.js";
+import { getAnalyticsSchemaVersion } from "./schema.js";
+
+interface MeasurementRow extends MeasurementFact {
+	id: number;
+}
+interface IdentityRow extends IdentityObservation {
+	id: number;
+}
+
+export class InMemoryAnalyticsStore implements AnalyticsStore {
+	private events: Map<string, RequestEventFact> = new Map();
+	private measurements: MeasurementRow[] = [];
+	private identities: IdentityRow[] = [];
+	private nextId = 1;
+
+	constructor(_options?: AnalyticsStoreOptions) {
+		// Options unused — the in-memory store doesn't open a DB.
+	}
+
+	private allocId(): number {
+		return this.nextId++;
+	}
+
+	// ── AnalyticsReader ───────────────────────────────────────────────
+
+	status(): AnalyticsStatus {
+		const lastEvent = this.events.size > 0
+			? Math.max(...[...this.events.values()].map((e) => e.observedAt))
+			: 0;
+		const lastMeas = this.measurements.length > 0
+			? Math.max(...this.measurements.map((m) => m.observedAt))
+			: 0;
+		const lastId = this.identities.length > 0
+			? Math.max(...this.identities.map((i) => i.observedAt))
+			: 0;
+		return {
+			enabled: true,
+			schemaVersion: getAnalyticsSchemaVersion(),
+			requestEventCount: this.events.size,
+			measurementCount: this.measurements.length,
+			identityCount: this.identities.length,
+			freshThrough: Math.max(lastEvent, lastMeas, lastId) || null,
+		};
+	}
+
+	// ── AnalyticsWriter ───────────────────────────────────────────────
+
+	appendRequestEvent(fact: RequestEventFact): AppendResult {
+		if (this.events.has(fact.id)) {
+			return { status: "duplicate", id: fact.id };
+		}
+		this.events.set(fact.id, { ...fact });
+		return { status: "accepted", id: fact.id };
+	}
+
+	appendMeasurement(sample: MeasurementFact): AppendResult {
+		const id = this.allocId();
+		this.measurements.push({ ...sample, id });
+		return { status: "accepted", id };
+	}
+
+	appendIdentity(observation: IdentityObservation): AppendResult {
+		const id = this.allocId();
+		this.identities.push({ ...observation, id });
+		return { status: "accepted", id };
+	}
+
+	// ── AnalyticsAdmin ────────────────────────────────────────────────
+
+	prune(policy: RetentionPolicy): PruneReport {
+		const now = Date.now();
+		const eventCutoff = now - policy.maxEventAgeMs;
+		const measCutoff = now - policy.maxMeasurementAgeMs;
+		const beforeE = this.events.size;
+		const beforeM = this.measurements.length;
+		for (const [k, e] of this.events) {
+			if (e.observedAt < eventCutoff) this.events.delete(k);
+		}
+		this.measurements = this.measurements.filter((m) => m.observedAt >= measCutoff);
+		return {
+			eventsRemoved: beforeE - this.events.size,
+			measurementsRemoved: beforeM - this.measurements.length,
+			identitiesRemoved: 0,
+			freedBytes: 0,
+		};
+	}
+
+	vacuum(): MaintenanceReport {
+		return { ok: true, detail: "no-op for in-memory store" };
+	}
+
+	integrityCheck(): IntegrityReport {
+		return { ok: true, detail: "ok" };
+	}
+
+	backup(): BackupReport {
+		return { ok: false, detail: "Cannot backup an in-memory database" };
+	}
+
+	clear(): void {
+		this.events.clear();
+		this.measurements = [];
+		this.identities = [];
+		this.nextId = 1;
+	}
+
+	close(): void {
+		this.clear();
+	}
+
+	// ── Capability views ──────────────────────────────────────────────
+
+	asReader(): AnalyticsReader {
+		return {
+			status: () => this.status(),
+		};
+	}
+
+	asWriter(): AnalyticsWriter {
+		return {
+			appendRequestEvent: (f) => this.appendRequestEvent(f),
+			appendMeasurement: (s) => this.appendMeasurement(s),
+			appendIdentity: (o) => this.appendIdentity(o),
+		};
+	}
+
+	asAdmin(): AnalyticsAdmin {
+		return {
+			prune: (p) => this.prune(p),
+			vacuum: () => this.vacuum(),
+			integrityCheck: () => this.integrityCheck(),
+			backup: () => this.backup(),
+			clear: () => this.clear(),
+			close: () => this.close(),
+		};
+	}
+}