sidekick-shared 0.24.4 → 0.25.0

package/dist/observedSessionCollector.js

@@ -1,34 +1,200 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ObservedSessionCollector = void 0;
 exports.observedSessionSourceFromProvider = observedSessionSourceFromProvider;
+exports.fileFingerprintParts = fileFingerprintParts;
 exports.fileFingerprint = fileFingerprint;
+const fs = __importStar(require("node:fs"));
 const node_fs_1 = require("node:fs");
 const observedSessionV1_1 = require("./types/observedSessionV1");
+const sessionActivityDetector_1 = require("./parsers/sessionActivityDetector");
+const DEFAULT_MAX_CONCURRENT_READS = 4;
+const MAX_CONCURRENT_READS = 16;
+const DEFAULT_DEBOUNCE_MS = 100;
+const DEFAULT_POLL_INTERVAL_MS = 30_000;
 /**
- * One-shot, host-scheduled observed-session collection with independent
- * provider/session isolation and bounded retry state.
+ * Fail-soft observed-session collection with fingerprint-keyed parse caching,
+ * cooperative reads, and optional provider-level subscriptions.
  */
 class ObservedSessionCollector {
     options;
     failures = new Map();
+    cache = new Map();
+    subscriptions = new Set();
     initialBackoffMs;
     maxBackoffMs;
     clock;
+    maxConcurrentReads;
+    yieldBetweenReads;
     constructor(options) {
         this.options = options;
         this.initialBackoffMs = Math.max(1, options.initialBackoffMs ?? 30_000);
         this.maxBackoffMs = Math.max(this.initialBackoffMs, options.maxBackoffMs ?? 5 * 60_000);
         this.clock = options.clock ?? { now: () => Date.now() };
+        this.maxConcurrentReads = Math.min(MAX_CONCURRENT_READS, Math.max(1, Math.floor(options.maxConcurrentReads ?? DEFAULT_MAX_CONCURRENT_READS)));
+        this.yieldBetweenReads = async () => {
+            if (options.yieldBetweenReads) {
+                await options.yieldBetweenReads();
+                return;
+            }
+            await new Promise((resolve) => setImmediate(resolve));
+        };
     }
     async collect() {
         const nested = await Promise.all(this.options.sources.map((source) => this.collectSource(source)));
         return nested.flat();
     }
+    /**
+     * Subscribe to debounced provider changes. The listener receives fingerprints
+     * only; call collect() to consume changed sessions through the parse cache.
+     */
+    subscribe(listener, options = {}) {
+        const debounceMs = Math.max(0, options.debounceMs ?? DEFAULT_DEBOUNCE_MS);
+        const pollIntervalMs = Math.max(0, options.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS);
+        const known = new Map();
+        for (const item of options.knownFingerprints ?? []) {
+            known.set(this.key(item.providerId, item.sessionId), {
+                fingerprint: item.fingerprint,
+                parts: item.fingerprintParts ?? null,
+            });
+        }
+        for (const [key, entry] of this.cache) {
+            known.set(key, { fingerprint: entry.fingerprint, parts: entry.fingerprintParts });
+        }
+        let disposed = false;
+        let debounceTimer = null;
+        let reconciling = false;
+        let pending = false;
+        const sourceDisposables = [];
+        const reconcile = async () => {
+            if (disposed)
+                return;
+            if (reconciling) {
+                pending = true;
+                return;
+            }
+            reconciling = true;
+            try {
+                do {
+                    pending = false;
+                    for (const source of this.options.sources) {
+                        if (disposed)
+                            break;
+                        const batch = await this.reconcileSource(source, known);
+                        // Re-check after the await: disposal during the reconcile must
+                        // not deliver the pre-disposal batch to the listener.
+                        if (disposed)
+                            break;
+                        if (batch && batch.changes.length > 0) {
+                            try {
+                                listener(batch);
+                            }
+                            catch {
+                                // Subscriber failures cannot stop reconciliation.
+                            }
+                        }
+                    }
+                } while (pending && !disposed);
+            }
+            finally {
+                reconciling = false;
+            }
+        };
+        const schedule = () => {
+            if (disposed)
+                return;
+            if (debounceTimer)
+                clearTimeout(debounceTimer);
+            debounceTimer = setTimeout(() => {
+                debounceTimer = null;
+                void reconcile();
+            }, debounceMs);
+            debounceTimer.unref?.();
+        };
+        let needsCollectorPoll = false;
+        for (const source of this.options.sources) {
+            if (source.subscribe) {
+                try {
+                    sourceDisposables.push(source.subscribe(schedule, { pollIntervalMs }));
+                }
+                catch {
+                    // The collector-level poll below remains the correctness fallback.
+                    needsCollectorPoll = true;
+                }
+            }
+            else {
+                needsCollectorPoll = true;
+            }
+        }
+        let pollTimer = null;
+        if (needsCollectorPoll && pollIntervalMs > 0) {
+            pollTimer = setInterval(schedule, pollIntervalMs);
+            pollTimer.unref?.();
+        }
+        schedule();
+        const disposable = {
+            dispose: () => {
+                if (disposed)
+                    return;
+                disposed = true;
+                if (debounceTimer)
+                    clearTimeout(debounceTimer);
+                if (pollTimer)
+                    clearInterval(pollTimer);
+                for (const item of sourceDisposables) {
+                    try {
+                        item.dispose();
+                    }
+                    catch {
+                        // Disposal is best effort.
+                    }
+                }
+                this.subscriptions.delete(disposable);
+            },
+        };
+        this.subscriptions.add(disposable);
+        return disposable;
+    }
     reset() {
         this.failures.clear();
+        this.cache.clear();
     }
     dispose() {
+        for (const subscription of [...this.subscriptions])
+            subscription.dispose();
         for (const source of this.options.sources) {
             try {
                 source.dispose?.();
@@ -54,49 +220,170 @@ class ObservedSessionCollector {
             return [];
         }
         const activeKeys = new Set(references.map((ref) => this.key(source.providerId, ref.sessionId)));
-        for (const key of this.failures.keys()) {
-            if (key.startsWith(`${source.providerId}\0`) &&
-                !key.endsWith('$discovery') &&
-                !activeKeys.has(key)) {
-                this.failures.delete(key);
-            }
-        }
-        const rows = await Promise.all(references.map((reference) => this.collectSession(source, reference)));
+        this.evictMissing(source.providerId, activeKeys);
+        const rows = await this.mapWithConcurrency(references, async (reference) => {
+            const result = await this.collectSession(source, reference);
+            await this.yieldBetweenReads();
+            return result;
+        });
         return rows.filter((row) => row !== null);
     }
     async collectSession(source, reference) {
         const key = this.key(source.providerId, reference.sessionId);
         const fingerprint = await this.fingerprint(source, reference);
-        const prior = this.failures.get(key);
-        if (prior && prior.fingerprint === fingerprint && this.clock.now() < prior.retryAt)
+        const priorFailure = this.failures.get(key);
+        if (priorFailure &&
+            priorFailure.fingerprint === fingerprint.value &&
+            this.clock.now() < priorFailure.retryAt) {
             return null;
+        }
+        const observedAt = new Date(this.clock.now()).toISOString();
+        const cached = fingerprint.value === null ? undefined : this.cache.get(key);
+        if (cached && cached.fingerprint === fingerprint.value) {
+            let value = cached.value;
+            try {
+                if (source.refreshCached) {
+                    value = await source.refreshCached(reference, cached.value, observedAt);
+                }
+            }
+            catch {
+                value = cached.value;
+            }
+            cached.value = value;
+            cached.fingerprintParts = fingerprint.parts;
+            const observation = this.makeObservation(source, reference, value, fingerprint, true, observedAt, cached.contentObservedAt);
+            this.emitObservation(observation);
+            return observation;
+        }
         let value;
         try {
             value = await source.read(reference);
         }
         catch (error) {
-            this.fail(key, 'session-read-failed', source.providerId, reference.sessionId, fingerprint, error);
+            this.fail(key, 'session-read-failed', source.providerId, reference.sessionId, fingerprint.value, error);
             return null;
         }
         this.recover(key, 'session-recovered', source.providerId, reference.sessionId);
-        const observation = { providerId: source.providerId, sessionId: reference.sessionId, value };
+        const contentObservedAt = contentTimestamp(value) ?? observedAt;
+        if (fingerprint.value !== null) {
+            this.cache.set(key, {
+                value,
+                fingerprint: fingerprint.value,
+                fingerprintParts: fingerprint.parts,
+                contentObservedAt,
+            });
+        }
+        const observation = this.makeObservation(source, reference, value, fingerprint, false, observedAt, contentObservedAt);
+        this.emitObservation(observation);
+        return observation;
+    }
+    makeObservation(source, reference, value, fingerprint, cacheHit, observedAt, contentObservedAt) {
+        return {
+            providerId: source.providerId,
+            sessionId: reference.sessionId,
+            value,
+            fingerprint: fingerprint.value,
+            fingerprintParts: fingerprint.parts,
+            cacheHit,
+            observedAt,
+            contentObservedAt,
+        };
+    }
+    emitObservation(observation) {
         try {
             this.options.onObservation?.(observation);
         }
         catch {
             // Host callbacks cannot turn a healthy provider read into a collection failure.
         }
-        return observation;
+    }
+    async reconcileSource(source, known) {
+        let references;
+        try {
+            references = await source.discover();
+        }
+        catch {
+            return null;
+        }
+        const changes = [];
+        const seen = new Set();
+        for (const reference of references) {
+            const key = this.key(source.providerId, reference.sessionId);
+            seen.add(key);
+            const current = await this.fingerprint(source, reference);
+            if (current.value === null)
+                continue;
+            const previous = known.get(key);
+            if (!previous || previous.fingerprint !== current.value) {
+                changes.push({
+                    type: previous ? 'changed' : 'added',
+                    reference,
+                    previousFingerprint: previous?.fingerprint ?? null,
+                    previousFingerprintParts: previous?.parts ?? null,
+                    fingerprint: current.value,
+                    fingerprintParts: current.parts,
+                });
+            }
+            known.set(key, { fingerprint: current.value, parts: current.parts });
+        }
+        for (const [key, previous] of [...known]) {
+            if (!key.startsWith(`${source.providerId}\0`) || seen.has(key))
+                continue;
+            const sessionId = key.slice(source.providerId.length + 1);
+            changes.push({
+                type: 'removed',
+                reference: { sessionId },
+                previousFingerprint: previous.fingerprint,
+                previousFingerprintParts: previous.parts,
+                fingerprint: null,
+                fingerprintParts: null,
+            });
+            known.delete(key);
+        }
+        return {
+            providerId: source.providerId,
+            changes,
+            observedAt: new Date(this.clock.now()).toISOString(),
+        };
     }
     async fingerprint(source, reference) {
+        let value = reference.fingerprintHint ?? null;
         try {
-            return this.options.fingerprint
-                ? await this.options.fingerprint(source, reference)
-                : (reference.fingerprintHint ?? null);
+            if (this.options.fingerprint)
+                value = await this.options.fingerprint(source, reference);
         }
         catch {
-            return reference.fingerprintHint ?? null;
+            // Retain the reference hint.
         }
+        const parts = reference.fingerprintParts ?? parseFingerprintParts(value);
+        return { value, parts };
+    }
+    evictMissing(providerId, activeKeys) {
+        for (const key of this.failures.keys()) {
+            if (key.startsWith(`${providerId}\0`) &&
+                !key.endsWith('$discovery') &&
+                !activeKeys.has(key)) {
+                this.failures.delete(key);
+            }
+        }
+        for (const key of this.cache.keys()) {
+            if (key.startsWith(`${providerId}\0`) && !activeKeys.has(key))
+                this.cache.delete(key);
+        }
+    }
+    async mapWithConcurrency(values, mapper) {
+        const results = new Array(values.length);
+        let cursor = 0;
+        const workers = Array.from({ length: Math.min(this.maxConcurrentReads, values.length) }, async () => {
+            while (true) {
+                const index = cursor++;
+                if (index >= values.length)
+                    return;
+                results[index] = await mapper(values[index]);
+            }
+        });
+        await Promise.all(workers);
+        return results;
     }
     fail(key, kind, providerId, sessionId, fingerprint, error) {
         const previous = this.failures.get(key);
@@ -106,14 +393,17 @@ class ObservedSessionCollector {
         const delay = Math.min(this.maxBackoffMs, this.initialBackoffMs * 2 ** (attempt - 1));
         const retryAt = this.clock.now() + delay;
         const reportedSignature = previous?.reportedSignature ?? '';
-        this.failures.set(key, {
-            attempt,
-            retryAt,
-            fingerprint,
-            reportedSignature: signature,
-        });
+        this.failures.set(key, { attempt, retryAt, fingerprint, reportedSignature: signature });
         if (signature !== reportedSignature || fingerprintChanged) {
-            this.emitDiagnostic({ kind, providerId, sessionId, attempt, retryAt });
+            this.emitDiagnostic({
+                kind,
+                severity: 'error',
+                phase: kind === 'provider-discovery-failed' ? 'discover' : 'read',
+                providerId,
+                sessionId,
+                attempt,
+                retryAt,
+            });
         }
     }
     recover(key, kind, providerId, sessionId) {
@@ -121,7 +411,14 @@ class ObservedSessionCollector {
         if (!prior)
             return;
         this.failures.delete(key);
-        this.emitDiagnostic({ kind, providerId, sessionId, attempt: prior.attempt });
+        this.emitDiagnostic({
+            kind,
+            severity: 'info',
+            phase: 'recover',
+            providerId,
+            sessionId,
+            attempt: prior.attempt,
+        });
     }
     emitDiagnostic(diagnostic) {
         try {
@@ -140,38 +437,151 @@ exports.ObservedSessionCollector = ObservedSessionCollector;
  * Adapt a provider using path-only discovery and independent adapter reads, so
  * one malformed session cannot reject discovery of its healthy siblings.
  */
-function observedSessionSourceFromProvider(provider, cwd) {
-    const adapter = (0, observedSessionV1_1.createProviderSessionAdapterV1)(provider);
+function observedSessionSourceFromProvider(provider, cwd, options = {}) {
+    const adapter = options.observationOnly
+        ? (0, observedSessionV1_1.createProviderSessionAdapterV1)(provider, { observationOnly: true })
+        : (0, observedSessionV1_1.createProviderSessionAdapterV1)(provider);
+    const referenceFromPath = (sessionPath, sessionId = provider.getSessionId(sessionPath), suppliedParts) => {
+        const metadata = suppliedParts ? null : provider.getSessionMetadata?.(sessionPath);
+        const parts = suppliedParts ??
+            fileFingerprintParts(sessionPath) ??
+            (metadata ? { sizeBytes: 0, mtimeMs: metadata.mtime.getTime() } : null);
+        return {
+            sessionId,
+            sourceKey: sessionPath,
+            fingerprintHint: parts ? fingerprintString(parts) : undefined,
+            fingerprintParts: parts ?? undefined,
+        };
+    };
+    const discover = async () => {
+        if (provider.listSessionFilesAsync) {
+            try {
+                const files = await provider.listSessionFilesAsync(cwd);
+                return files.map((info) => {
+                    const mtimeMs = info.mtime.getTime();
+                    const suppliedParts = info.sizeBytes !== undefined && Number.isFinite(mtimeMs)
+                        ? { sizeBytes: info.sizeBytes, mtimeMs }
+                        : undefined;
+                    return referenceFromPath(info.path, info.sessionId ?? provider.getSessionId(info.path), suppliedParts);
+                });
+            }
+            catch {
+                // Retain compatibility with third-party providers whose optional async
+                // enumerator is temporarily unavailable.
+            }
+        }
+        return provider.findAllSessions(cwd).map((sessionPath) => {
+            const metadata = provider.getSessionMetadata?.(sessionPath);
+            const parts = fileFingerprintParts(sessionPath) ??
+                (metadata ? { sizeBytes: 0, mtimeMs: metadata.mtime.getTime() } : null);
+            return {
+                sessionId: provider.getSessionId(sessionPath),
+                sourceKey: sessionPath,
+                fingerprintHint: parts ? fingerprintString(parts) : undefined,
+                fingerprintParts: parts ?? undefined,
+            };
+        });
+    };
     return {
         providerId: provider.id,
         capabilities: adapter.capabilities,
-        discover() {
-            return provider.findAllSessions(cwd).map((sessionPath) => ({
-                sessionId: provider.getSessionId(sessionPath),
-                sourceKey: sessionPath,
-                fingerprintHint: provider.getSessionMetadata?.(sessionPath)?.mtime.getTime().toString() ??
-                    fileFingerprint(sessionPath) ??
-                    undefined,
-            }));
-        },
-        read(reference) {
+        discover,
+        async read(reference) {
             if (!reference.sourceKey)
                 throw new Error('session source unavailable');
             return adapter.read(reference.sourceKey, cwd);
         },
+        refreshCached(reference, cached, observedAt) {
+            const mtimeMs = reference.fingerprintParts?.mtimeMs ?? Date.parse(cached.observedAt);
+            const activity = (0, sessionActivityDetector_1.refreshSessionActivityState)(cached.activity.value, mtimeMs);
+            return {
+                ...cached,
+                activity: { ...cached.activity, value: activity },
+                observedAt,
+            };
+        },
+        subscribe(listener, subscribeOptions = {}) {
+            return subscribeToProviderRoot(provider, listener, subscribeOptions.pollIntervalMs ?? 0);
+        },
         dispose: () => adapter.dispose(),
     };
 }
-/** Size/mtime fingerprint suitable for retry bypass; returns null for synthetic DB locators. */
-function fileFingerprint(sourcePath) {
+/** Structured size/mtime fingerprint; returns null for inaccessible or synthetic paths. */
+function fileFingerprintParts(sourcePath) {
     try {
         const stat = (0, node_fs_1.statSync)(sourcePath);
-        return `${stat.size}:${stat.mtimeMs}`;
+        return { sizeBytes: stat.size, mtimeMs: stat.mtimeMs };
     }
     catch {
         return null;
     }
 }
+/** Size/mtime fingerprint suitable for retry bypass; returns null for synthetic DB locators. */
+function fileFingerprint(sourcePath) {
+    const parts = fileFingerprintParts(sourcePath);
+    return parts ? fingerprintString(parts) : null;
+}
+function fingerprintString(parts) {
+    return `${parts.sizeBytes}:${parts.mtimeMs}`;
+}
+function parseFingerprintParts(value) {
+    if (!value)
+        return null;
+    const match = value.match(/^([0-9]+):([0-9]+(?:\.[0-9]+)?)$/);
+    if (!match)
+        return null;
+    const sizeBytes = Number(match[1]);
+    const mtimeMs = Number(match[2]);
+    return Number.isFinite(sizeBytes) && Number.isFinite(mtimeMs) ? { sizeBytes, mtimeMs } : null;
+}
+function contentTimestamp(value) {
+    if (!value || typeof value !== 'object')
+        return null;
+    const candidate = value.contentObservedAt;
+    return typeof candidate === 'string' ? candidate : null;
+}
+function subscribeToProviderRoot(provider, listener, pollIntervalMs) {
+    const watchers = new Set();
+    let pollTimer = null;
+    let roots;
+    try {
+        roots = provider.getWatchRoots?.() ?? [provider.getProjectsBaseDir()];
+    }
+    catch {
+        roots = [];
+    }
+    for (const root of new Set(roots)) {
+        try {
+            const watcher = fs.watch(root, { persistent: false, recursive: true }, listener);
+            watchers.add(watcher);
+            watcher.on('error', () => {
+                watchers.delete(watcher);
+                try {
+                    watcher.close();
+                }
+                catch {
+                    // Catch-up polling remains active.
+                }
+            });
+        }
+        catch {
+            // Polling remains the documented fallback.
+        }
+    }
+    if (pollIntervalMs > 0) {
+        pollTimer = setInterval(listener, pollIntervalMs);
+        pollTimer.unref?.();
+    }
+    return {
+        dispose: () => {
+            for (const watcher of watchers)
+                watcher.close();
+            watchers.clear();
+            if (pollTimer)
+                clearInterval(pollTimer);
+        },
+    };
+}
 function failureSignature(error) {
     if (error instanceof Error)
         return `${error.name}\0${error.message}`;

package/dist/parsers/codexHistory.d.ts

@@ -0,0 +1,32 @@
+/**
+ * Reader for Codex's append-only `~/.codex/history.jsonl` (Node-only).
+ *
+ * Each line is `{"session_id","ts","text"}` — one entry per user prompt. The
+ * file grows without bound, so this reader takes a bounded tail instead of the
+ * whole file, drops the leading partial line, and parses tolerantly. Pair an
+ * entry with {@link findCodexRolloutFile} (providers/codex) to jump from a
+ * history row to its full rollout transcript.
+ *
+ * @module codexHistory
+ */
+export interface CodexHistoryEntry {
+    sessionId: string;
+    /** Timestamp exactly as Codex recorded it (epoch seconds). */
+    ts: number;
+    /** Normalized epoch milliseconds, ready for Date construction. */
+    tsMs: number;
+    text: string;
+}
+export interface ReadCodexHistoryOptions {
+    /** Codex home containing `history.jsonl` (default: `$CODEX_HOME` or `~/.codex`). */
+    codexHome?: string;
+    /** Maximum entries returned, newest first (default 100). */
+    limit?: number;
+    /** Bytes read from the end. Defaults to max(512 KiB, limit × 4096). */
+    maxTailBytes?: number;
+}
+/**
+ * Returns the newest history entries, newest first. A missing or unreadable
+ * file yields `[]`; malformed lines are skipped. Never throws.
+ */
+export declare function readCodexHistory(options?: ReadCodexHistoryOptions): CodexHistoryEntry[];