pi-famulus 0.1.7 → 0.1.8

package/src/monitor-batching.ts

@@ -13,19 +13,71 @@ export interface LineBatcherOptions {
   flushMs?: number; // batching window, default 200
   maxLineChars?: number; // per-line cap, default 500
   maxBatchChars?: number; // per-batch cap, default 3000
-  onFlush: (text: string) => void;
+  onFlush: (text: string, info: { truncated: boolean; omittedChars: number }) => void;
   clock?: Clock;
 }
 
+function sourceOfSlice(text: string, source: number, start: number, end: number): number {
+  if (text.length === 0 || end <= start) return 0;
+  return Math.round(source * ((end - start) / text.length));
+}
+
+/** Fixed-memory head/tail preview. Counts omitted source units, not generated ellipses. */
+export class BoundedPreview {
+  private head = "";
+  private tail = "";
+  private sourceHead = 0;
+  private sourceTail = 0;
+  total = 0;
+  private sourceTotal = 0;
+  constructor(readonly limit: number) {
+    if (!Number.isSafeInteger(limit) || limit < 1) throw new RangeError("preview limit must be positive");
+  }
+  append(text: string, sourceChars = text.length): void {
+    const headLimit = Math.ceil((this.limit - 1) / 2);
+    if (this.total + text.length <= this.limit) {
+      this.head += text;
+      this.sourceHead += sourceChars;
+    } else {
+      const old = this.head;
+      const oldSourceHead = this.sourceHead;
+      if (this.total <= this.limit) {
+        const combined = old + text;
+        const combinedSource = oldSourceHead + sourceChars;
+        this.head = combined.slice(0, headLimit);
+        this.sourceHead = sourceOfSlice(combined, combinedSource, 0, this.head.length);
+      }
+      const tailLimit = this.limit - 1 - headLimit;
+      if (tailLimit > 0) {
+        const from = (this.tail || old).concat(text);
+        const fromSource = (this.tail ? this.sourceTail : oldSourceHead) + sourceChars;
+        this.tail = from.slice(-tailLimit);
+        this.sourceTail = sourceOfSlice(from, fromSource, from.length - this.tail.length, from.length);
+      } else {
+        this.tail = "";
+        this.sourceTail = 0;
+      }
+    }
+    this.total += text.length;
+    this.sourceTotal += sourceChars;
+  }
+  get omittedChars(): number {
+    if (this.total <= this.limit) return 0;
+    return Math.max(0, this.sourceTotal - this.sourceHead - this.sourceTail);
+  }
+  get text(): string { return this.total > this.limit ? `${this.head}…${this.tail}` : this.head; }
+}
+
 export class LineBatcher {
   private readonly flushMs: number;
   private readonly maxLineChars: number;
   private readonly maxBatchChars: number;
-  private readonly onFlush: (text: string) => void;
+  private readonly onFlush: LineBatcherOptions["onFlush"];
   private readonly clock: Clock;
-
-  private partial = ""; // trailing bytes not yet terminated by \n
-  private lines: string[] = []; // complete lines awaiting emission
+  private partial: BoundedPreview;
+  private lines: BoundedPreview;
+  private lineCount = 0;
+  private omittedLineChars = 0;
   private timer: ClockTimer | null = null;
   private disposed = false;
 
@@ -35,49 +87,52 @@ export class LineBatcher {
     this.maxBatchChars = opts.maxBatchChars ?? 3000;
     this.onFlush = opts.onFlush;
     this.clock = opts.clock ?? realClock;
+    this.partial = new BoundedPreview(this.maxLineChars);
+    this.lines = new BoundedPreview(this.maxBatchChars);
   }
 
-  /** Feed a raw output chunk (may contain no/newlines or multiple lines). */
+  /** Bound retained data during ingestion, not just after the timer fires. */
   push(chunk: string): void {
     if (this.disposed || chunk.length === 0) return;
-    const combined = this.partial + chunk;
-    const parts = combined.split("\n");
-    this.partial = parts.pop() ?? "";
-    for (const line of parts) {
-      this.lines.push(this.capLine(line));
+    const parts = chunk.split("\n");
+    for (let i = 0; i < parts.length; i++) {
+      this.partial.append(parts[i]);
+      if (i < parts.length - 1) this.finishLine();
     }
-    if (this.lines.length > 0 || this.partial.length > 0) this.schedule();
+    this.schedule();
   }
 
-  /**
-   * Immediately emit the current buffer when non-empty, including any
-   * unterminated trailing line. Used when a monitored process exits.
-   */
   flush(): void {
     if (this.disposed) return;
     this.clearTimer();
     this.emit();
   }
 
-  /** Clear the pending timer and drop buffered content. */
   dispose(): void {
     this.disposed = true;
     this.clearTimer();
-    this.lines = [];
-    this.partial = "";
+    this.reset();
   }
 
-  private capLine(line: string): string {
-    if (line.length <= this.maxLineChars) return line;
-    return line.slice(0, this.maxLineChars);
+  private finishLine(): void {
+    if (this.lineCount++ > 0) this.lines.append("\n", 1);
+    const displayed = this.partial.text;
+    this.lines.append(displayed, this.partial.total - this.partial.omittedChars);
+    this.omittedLineChars += this.partial.omittedChars;
+    this.partial = new BoundedPreview(this.maxLineChars);
+  }
+
+  private reset(): void {
+    this.partial = new BoundedPreview(this.maxLineChars);
+    this.lines = new BoundedPreview(this.maxBatchChars);
+    this.lineCount = 0;
+    this.omittedLineChars = 0;
   }
 
   private schedule(): void {
     if (this.timer !== null) return;
     this.timer = this.clock.setTimeout(() => {
       this.timer = null;
-      // Window end drains everything, including an unterminated trailing
-      // line (no data loss); a fresh window starts on the next push.
       this.emit();
     }, this.flushMs);
     this.clock.unref?.(this.timer);
@@ -91,18 +146,11 @@ export class LineBatcher {
   }
 
   private emit(): void {
-    const lines = this.lines;
-    const partial = this.partial.length > 0 ? this.capLine(this.partial) : "";
-    if (lines.length === 0 && partial.length === 0) return;
-    this.lines = [];
-    this.partial = "";
-
-    let text = partial.length > 0 ? [...lines, partial].join("\n") : lines.join("\n");
-    if (text.length > this.maxBatchChars) {
-      // Keep the tail: for monitoring, the most recent lines matter most.
-      text = `…${text.slice(-(this.maxBatchChars - 1))}`;
-    }
-    if (text.length > 0) this.onFlush(text);
+    if (this.partial.total > 0) this.finishLine();
+    const text = this.lines.text;
+    const omittedChars = this.omittedLineChars + this.lines.omittedChars;
+    this.reset();
+    if (text.length > 0) this.onFlush(text, { truncated: omittedChars > 0, omittedChars });
   }
 }
 

package/src/monitor-output.ts

@@ -0,0 +1,159 @@
+import { LineBatcher } from "./monitor-batching";
+import { realClock, type Clock } from "./clock";
+import type { MonitorOutputRecord } from "./manager-client";
+import type { MonitorOutputInfo } from "./wake";
+
+export const DEFAULT_DEDUP_WINDOW_MS = 60_000;
+export const MAX_DEDUP_KEYS = 2048;
+export type FingerprintMode = "exact" | "ignore-timestamps";
+export interface MonitorOutputBatch { text: string; output: MonitorOutputInfo }
+interface Identity { phase: "buffered" | "queued" | "emitted"; lastEmitted?: number; preview: string }
+
+/** Merge original-byte range envelopes, not a promise of contiguous identical events. */
+export function mergeOutputInfo(a: MonitorOutputInfo, b: MonitorOutputInfo): MonitorOutputInfo {
+  const min = (x?: number, y?: number) => x === undefined ? y : y === undefined ? x : Math.min(x, y);
+  const max = (x?: number, y?: number) => x === undefined ? y : y === undefined ? x : Math.max(x, y);
+  return {
+    firstObservedAt: min(a.firstObservedAt, b.firstObservedAt),
+    lastObservedAt: max(a.lastObservedAt, b.lastObservedAt),
+    cursorStart: min(a.cursorStart, b.cursorStart), cursorEnd: max(a.cursorEnd, b.cursorEnd),
+    truncated: a.truncated || b.truncated,
+    suppressedCount: (a.suppressedCount ?? 0) + (b.suppressedCount ?? 0),
+    omittedChars: (a.omittedChars ?? 0) + (b.omittedChars ?? 0),
+  };
+}
+
+/**
+ * Rust owns framing and hashing; this module owns the notification window.
+ * Pending identities are suppressed until onDelivered starts their fixed window.
+ * No timers send repeat-only summaries. Original bytes stay in the manager log.
+ */
+export class MonitorOutputBuffer {
+  private readonly clock: Clock;
+  private readonly batcher: LineBatcher;
+  private readonly identities = new Map<string, Identity>();
+  private batchKeys = new Set<string>();
+  private batchOutput: MonitorOutputInfo = {};
+  private suppressed: MonitorOutputInfo = {};
+  private disposed = false;
+  private lastNow = 0;
+
+  constructor(private readonly options: {
+    clock?: Clock;
+    windowMs?: number;
+    mode?: FingerprintMode;
+    onFlush: (batch: MonitorOutputBatch) => boolean;
+    onSuppressed?: () => void;
+  }) {
+    this.clock = options.clock ?? realClock;
+    this.batcher = new LineBatcher({ clock: this.clock, onFlush: (text, info) => {
+      this.batchOutput = mergeOutputInfo(this.batchOutput, info);
+      this.emit(text);
+    } });
+  }
+
+  /** Stable callback identity: NotifyCenter retains it once per pending monitor. */
+  readonly onDelivered = (visiblePreview?: string): void => {
+    const now = this.now();
+    const visibleLines = visiblePreview === undefined ? undefined : new Set(visiblePreview ? visiblePreview.split("\n") : []);
+    const queuedPreviews = new Map<string, number>();
+    if (visibleLines) {
+      for (const identity of this.identities.values()) {
+        if (identity.phase === "queued") {
+          queuedPreviews.set(identity.preview, (queuedPreviews.get(identity.preview) ?? 0) + 1);
+        }
+      }
+    }
+    for (const [key, identity] of this.identities) {
+      if (identity.phase !== "queued") continue;
+      // Both local batching and busy coalescing can omit an entire identity.
+      // Never pretend the model saw it: truncated representatives fail open.
+      // Distinct fingerprints can share a truncated preview; fail those open too.
+      const ambiguous = (queuedPreviews.get(identity.preview) ?? 0) > 1;
+      if (visibleLines && (ambiguous || !visibleLines.has(identity.preview))) {
+        if (identity.lastEmitted === undefined) this.identities.delete(key);
+        else identity.phase = "emitted";
+      } else {
+        identity.phase = "emitted";
+        identity.lastEmitted = now;
+      }
+    }
+  };
+
+  push(record: MonitorOutputRecord): void {
+    if (this.disposed || !record.preview) return;
+    const now = this.now();
+    const info: MonitorOutputInfo = {
+      firstObservedAt: record.observed_at, lastObservedAt: record.observed_at,
+      cursorStart: record.cursor_start, cursorEnd: record.cursor_end, truncated: record.truncated,
+    };
+    const hash = record.kind === "line"
+      ? this.options.mode === "ignore-timestamps" ? record.timestamp_fingerprint : record.fingerprint
+      : undefined;
+    const window = this.options.windowMs ?? DEFAULT_DEDUP_WINDOW_MS;
+    const key = window > 0 ? hash : undefined;
+    if (key) {
+      const previous = this.identities.get(key);
+      if (previous && (previous.phase !== "emitted" || now - previous.lastEmitted! < window)) {
+        this.suppressed = mergeOutputInfo(this.suppressed, { ...info, suppressedCount: 1 });
+        this.options.onSuppressed?.();
+        return;
+      }
+      // Capacity eviction fails open: extra output is preferable to false suppression.
+      if (!previous && this.identities.size >= MAX_DEDUP_KEYS) {
+        const oldest = this.identities.keys().next().value;
+        if (oldest !== undefined) {
+          this.identities.delete(oldest);
+          this.batchKeys.delete(oldest);
+        }
+      }
+      this.identities.set(key, { phase: "buffered", lastEmitted: previous?.lastEmitted, preview: record.preview });
+      this.batchKeys.add(key);
+    }
+    this.batchOutput = mergeOutputInfo(mergeOutputInfo(this.batchOutput, this.suppressed), info);
+    this.suppressed = {};
+    this.batcher.push(`${record.preview}\n`);
+  }
+
+  flush(): void { this.batcher.flush(); }
+
+  /** Attach unreported repeats to the terminal notice without another event/wake. */
+  takeSummary(): MonitorOutputInfo {
+    const summary = this.suppressed;
+    this.suppressed = {};
+    return summary;
+  }
+
+  dispose(): void {
+    this.disposed = true;
+    this.batcher.dispose();
+    this.identities.clear();
+    this.batchKeys.clear();
+  }
+
+  private now(): number {
+    this.lastNow = Math.max(this.lastNow, this.clock.now());
+    return this.lastNow;
+  }
+
+  private emit(text: string): void {
+    const keys = this.batchKeys;
+    const output = this.batchOutput;
+    this.batchKeys = new Set();
+    this.batchOutput = {};
+    for (const key of keys) {
+      const identity = this.identities.get(key);
+      if (identity?.phase === "buffered") identity.phase = "queued";
+    }
+    if (!this.options.onFlush({ text, output })) {
+      // A rate-limited batch wasn't delivered: don't start a suppression window.
+      for (const key of keys) {
+        const identity = this.identities.get(key);
+        if (identity?.phase !== "queued") continue;
+        if (identity.lastEmitted === undefined) this.identities.delete(key);
+        else identity.phase = "emitted";
+      }
+      this.suppressed = mergeOutputInfo(this.suppressed, { ...output, suppressedCount: output.suppressedCount ?? 0 });
+    }
+  }
+}