pi-pair-programmer 0.1.0

package/src/logger.ts

@@ -0,0 +1,396 @@
+import { randomUUID } from "node:crypto";
+import { chmod, mkdir, open, readdir, unlink } from "node:fs/promises";
+import { homedir } from "node:os";
+import path from "node:path";
+import { pino } from "pino";
+
+export type PairLogEvent =
+  | "session.start"
+  | "session.stop"
+  | "extension.shutdown"
+  | "extension.toggle"
+  | "extension.persistence_failed"
+  | "review.scheduled"
+  | "review.started"
+  | "review.finished"
+  | "review.skipped"
+  | "model.finished"
+  | "finding.filtered"
+  | "finding.admission"
+  | "finding.verdict"
+  | "delivery.sent"
+  | "delivery.wake";
+
+export interface PairLogFields {
+  sessionId?: string;
+  jobId?: string;
+  reviewerId?: string;
+  modelId?: string;
+  stage?: "review" | "attribution" | "dedup";
+  outcome?: keyof typeof outcomes;
+  reasonCode?: keyof typeof reasonCodes;
+  count?: number;
+  durationMs?: number;
+  inputTokens?: number;
+  outputTokens?: number;
+  cacheReadTokens?: number;
+  cacheWriteTokens?: number;
+  totalTokens?: number;
+  costUsd?: number;
+}
+
+export interface PairLogger {
+  log(event: PairLogEvent, fields?: PairLogFields): void;
+  close(): Promise<void>;
+}
+
+const events: Record<PairLogEvent, true> = {
+  "session.start": true,
+  "session.stop": true,
+  "extension.shutdown": true,
+  "extension.toggle": true,
+  "extension.persistence_failed": true,
+  "review.scheduled": true,
+  "review.started": true,
+  "review.finished": true,
+  "review.skipped": true,
+  "model.finished": true,
+  "finding.filtered": true,
+  "finding.admission": true,
+  "finding.verdict": true,
+  "delivery.sent": true,
+  "delivery.wake": true,
+};
+const numericFields: Record<string, true> = {
+  count: true,
+  durationMs: true,
+  inputTokens: true,
+  outputTokens: true,
+  cacheReadTokens: true,
+  cacheWriteTokens: true,
+  totalTokens: true,
+  costUsd: true,
+};
+const identifierFields: Record<string, true> = {
+  sessionId: true,
+  jobId: true,
+  reviewerId: true,
+  modelId: true,
+};
+const stages: Record<string, true> = {
+  review: true,
+  attribution: true,
+  dedup: true,
+};
+const outcomes = {
+  success: true,
+  failed: true,
+  cancelled: true,
+  timeout: true,
+  obsolete: true,
+  added: true,
+  unchanged: true,
+  accept: true,
+  reject: true,
+  invalid: true,
+  on: true,
+  off: true,
+};
+const reasonCodes = {
+  session_change: true,
+  shutdown: true,
+  disabled: true,
+  cleared: true,
+  superseded: true,
+  stale: true,
+  missing_model: true,
+  unchanged: true,
+  duplicate_config: true,
+  scheduled: true,
+  completed: true,
+  no_findings: true,
+  inherited: true,
+  attribution_fallback: true,
+  dedup_fallback: true,
+  persistence_failed: true,
+  wake_pending: true,
+  busy: true,
+  no_findings_ready: true,
+  already_presented: true,
+  ready: true,
+  outstanding: true,
+  passive: true,
+  baseline_unavailable: true,
+  configuration_failed: true,
+  ui_unavailable: true,
+};
+const maxFileBytes = 1024 * 1024;
+const maxBufferedBytes = 64 * 1024;
+const retainedFiles = 5;
+const filePattern = /^pair-\d+-(\d+)-[\da-f-]+\.jsonl$/u;
+const activeFilesKey = Symbol.for("pi-pair-programmer.active-log-files.v1");
+const retainedFilesInProcess = Reflect.get(globalThis, activeFilesKey) as
+  Set<string> | undefined;
+const activeFiles = retainedFilesInProcess ?? new Set<string>();
+
+function safeFields(fields: PairLogFields): Record<string, string | number> {
+  const safe: Record<string, string | number> = {};
+  for (const [key, value] of Object.entries(fields)) {
+    const numeric =
+      Object.hasOwn(numericFields, key) &&
+      typeof value === "number" &&
+      Number.isFinite(value) &&
+      value >= 0;
+    const identifier =
+      Object.hasOwn(identifierFields, key) &&
+      typeof value === "string" &&
+      /^[\da-f-]{1,64}$/iu.test(value);
+    const code =
+      typeof value === "string" &&
+      ((key === "stage" && Object.hasOwn(stages, value)) ||
+        (key === "outcome" && Object.hasOwn(outcomes, value)) ||
+        (key === "reasonCode" && Object.hasOwn(reasonCodes, value)));
+    if (numeric || identifier || code)
+      Object.defineProperty(safe, key, { value, enumerable: true });
+  }
+  return safe;
+}
+
+type LogLevel = "debug" | "info" | "warn" | "error";
+const warningReasons: Record<string, true> = {
+  attribution_fallback: true,
+  dedup_fallback: true,
+  missing_model: true,
+  baseline_unavailable: true,
+  ui_unavailable: true,
+};
+
+function severity(
+  event: PairLogEvent,
+  fields: Record<string, string | number>,
+): LogLevel {
+  const outcome = fields["outcome"];
+  const reason = fields["reasonCode"];
+  if (
+    event === "extension.persistence_failed" ||
+    reason === "persistence_failed" ||
+    reason === "configuration_failed"
+  ) {
+    return "error";
+  }
+  if (
+    outcome === "timeout" ||
+    outcome === "invalid" ||
+    Object.hasOwn(warningReasons, reason ?? "")
+  ) {
+    return "warn";
+  }
+  if (outcome === "failed") {
+    return "error";
+  }
+  return outcome === "cancelled" ||
+    outcome === "obsolete" ||
+    event === "review.skipped" ||
+    event === "finding.filtered" ||
+    (event === "delivery.wake" && reason !== "ready")
+    ? "debug"
+    : "info";
+}
+
+function guardedLogger(
+  emit: (event: PairLogEvent, fields: Record<string, string | number>) => void,
+  finish: () => Promise<void>,
+): PairLogger {
+  let closed = false;
+  let closing: Promise<void> | undefined;
+  return {
+    log(event, fields = {}) {
+      try {
+        if (!closed && Object.hasOwn(events, event)) {
+          emit(event, safeFields(fields));
+        }
+      } catch {}
+    },
+    close() {
+      closed = true;
+      closing ??= (async () => {
+        try {
+          await finish();
+        } catch {}
+      })();
+      return closing;
+    },
+  };
+}
+
+function nativeLogger(host: unknown): PairLogger | undefined {
+  if (typeof host !== "object" || host === null || !("logger" in host)) {
+    return undefined;
+  }
+  const logger: unknown = host.logger;
+  if (
+    typeof logger !== "object" ||
+    logger === null ||
+    !("info" in logger) ||
+    typeof logger.info !== "function"
+  ) {
+    return undefined;
+  }
+  return guardedLogger(
+    (event, fields) => {
+      const method: unknown = Reflect.get(logger, severity(event, fields));
+      if (typeof method !== "function") return;
+      const emit = method as (
+        message: string,
+        context: Record<string, string | number>,
+      ) => void;
+      emit.call(logger, `pair-programmer.${event}`, fields);
+    },
+    () => Promise.resolve(),
+  );
+}
+
+function processRetired(pid: number): boolean {
+  try {
+    process.kill(pid, 0);
+    return false;
+  } catch (error) {
+    return error instanceof Error && "code" in error && error.code === "ESRCH";
+  }
+}
+
+async function pruneRetiredFiles(directory: string): Promise<void> {
+  const files = await readdir(directory);
+  const retired: string[] = [];
+  for (const name of files) {
+    const match = filePattern.exec(name);
+    if (match === null) {
+      continue;
+    }
+    if (Number(match[1]) === process.pid) {
+      if (!activeFiles.has(path.join(directory, name))) retired.push(name);
+      continue;
+    }
+    if (processRetired(Number(match[1]))) retired.push(name);
+  }
+  retired.sort((a, b) => a.localeCompare(b));
+  for (const name of retired.slice(
+    0,
+    Math.max(0, retired.length - retainedFiles),
+  )) {
+    try {
+      await unlink(path.join(directory, name));
+    } catch {}
+  }
+}
+
+async function fileLogger(directory: string): Promise<PairLogger> {
+  await mkdir(directory, { recursive: true, mode: 0o700 });
+
+  await chmod(directory, 0o700);
+  await pruneRetiredFiles(directory);
+  const prefix = path.join(
+    directory,
+    `pair-${String(Date.now())}-${String(process.pid)}-${randomUUID()}`,
+  );
+  let generation = 0;
+  let filename = `${prefix}-${String(generation)}.jsonl`;
+  activeFiles.add(filename);
+  let destination: Awaited<ReturnType<typeof open>>;
+  try {
+    destination = await open(filename, "wx", 0o600);
+  } catch (error) {
+    activeFiles.delete(filename);
+    throw error;
+  }
+  const files = [filename];
+  const state = { failed: false };
+  let bytes = 0;
+  let bufferedBytes = 0;
+  let pending = Promise.resolve();
+  const logger = pino(
+    { base: null, level: "debug" },
+    {
+      write(line: string) {
+        const length = Buffer.byteLength(line);
+        if (state.failed || bufferedBytes + length > maxBufferedBytes) {
+          return;
+        }
+        bufferedBytes += length;
+        const previous = pending;
+        pending = (async () => {
+          await previous;
+          try {
+            if (state.failed) {
+              return;
+            }
+            if (bytes + length > maxFileBytes) {
+              await destination.close();
+              generation++;
+              filename = `${prefix}-${String(generation)}.jsonl`;
+              activeFiles.add(filename);
+
+              destination = await open(filename, "wx", 0o600);
+              bytes = 0;
+              files.push(filename);
+              if (files.length > retainedFiles) {
+                for (const retired of files.splice(0, 1)) {
+                  activeFiles.delete(retired);
+
+                  await unlink(retired);
+                }
+              }
+            }
+            await destination.writeFile(line);
+            bytes += length;
+          } catch {
+            state.failed = true;
+          } finally {
+            bufferedBytes -= length;
+          }
+        })();
+      },
+    },
+  );
+  return guardedLogger(
+    (event, fields) => {
+      logger[severity(event, fields)](fields, `pair-programmer.${event}`);
+    },
+    async () => {
+      await pending;
+      try {
+        await destination.close();
+      } finally {
+        for (const file of files) activeFiles.delete(file);
+        activeFiles.delete(filename);
+        await pruneRetiredFiles(directory);
+      }
+    },
+  );
+}
+
+export async function createPairLogger(
+  host: unknown,
+  options: { directory?: string } = {},
+): Promise<PairLogger> {
+  Reflect.set(globalThis, activeFilesKey, activeFiles);
+  try {
+    const native = nativeLogger(host);
+    if (native !== undefined) {
+      return native;
+    }
+    const stateDirectory =
+      process.env["PI_CODING_AGENT_DIR"] ??
+      path.join(homedir(), ".pi", "agent");
+    return await fileLogger(
+      options.directory ?? path.join(stateDirectory, "logs", "pair-programmer"),
+    );
+  } catch {
+    return guardedLogger(
+      () => {
+        return;
+      },
+      () => Promise.resolve(),
+    );
+  }
+}

package/src/model-usage.ts

@@ -0,0 +1,285 @@
+import { APIError, APITimeoutError, APIUserAbortError } from "@typesafe-ai/sdk";
+
+export interface ModelUsage {
+  inputTokens?: number;
+  outputTokens?: number;
+  cacheReadTokens?: number;
+  cacheWriteTokens?: number;
+  totalTokens?: number;
+  costUsd?: number;
+}
+
+export interface ModelCallObservation {
+  stage: "review" | "attribution" | "dedup";
+  requestedModel: string;
+  model?: string;
+  provider?: string;
+  outcome: "success" | "failed" | "cancelled" | "timeout";
+  durationMs: number;
+  usage?: ModelUsage;
+}
+
+export type ModelCallObserver = (observation: ModelCallObservation) => void;
+type Outcome = ModelCallObservation["outcome"];
+type Metadata = Pick<ModelCallObservation, "model" | "provider" | "usage">;
+
+function record(value: unknown): Record<string, unknown> | undefined {
+  return typeof value === "object" && value !== null && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : undefined;
+}
+
+function numeric(value: unknown): value is number {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function metadata(value: unknown, judge = false): Metadata {
+  const message = record(value);
+  const result: Metadata = {};
+  const model =
+    !judge && typeof message?.["responseModel"] === "string"
+      ? message["responseModel"]
+      : message?.["model"];
+  if (typeof model === "string") result.model = model;
+  if (typeof message?.["provider"] === "string") {
+    result.provider = message["provider"];
+  }
+  const raw = record(message?.["usage"]);
+  if (raw === undefined) return result;
+  const usage: ModelUsage = {};
+  const fields: [keyof ModelUsage, unknown][] = judge
+    ? [
+        ["inputTokens", raw["input_tokens"]],
+        ["outputTokens", raw["output_tokens"]],
+      ]
+    : [
+        ["inputTokens", raw["input"]],
+        ["outputTokens", raw["output"]],
+        ["cacheReadTokens", raw["cacheRead"]],
+        ["cacheWriteTokens", raw["cacheWrite"]],
+        ["totalTokens", raw["totalTokens"]],
+        ["costUsd", record(raw["cost"])?.["total"]],
+      ];
+  for (const [key, value] of fields) {
+    if (numeric(value) && (key === "costUsd" || Number.isSafeInteger(value))) {
+      Object.defineProperty(usage, key, { value, enumerable: true });
+    }
+  }
+  if (Object.keys(usage).length > 0) result.usage = usage;
+  return result;
+}
+
+function notify(
+  observer: ModelCallObserver | undefined,
+  event: ModelCallObservation,
+): void {
+  try {
+    observer?.(event);
+  } catch {}
+}
+
+export async function observeJudgment<T>(
+  stage: "attribution" | "dedup",
+  signal: AbortSignal,
+  observer: ModelCallObserver | undefined,
+  invoke: () => PromiseLike<T>,
+): Promise<T> {
+  const start = performance.now();
+  let details: Metadata = {};
+  let outcome: Outcome = "success";
+  try {
+    const response = await invoke();
+    details = metadata(response, true);
+    return response;
+  } catch (error) {
+    if (signal.aborted || error instanceof APIUserAbortError) {
+      outcome = "cancelled";
+    } else {
+      outcome = error instanceof APITimeoutError ? "timeout" : "failed";
+    }
+    if (error instanceof APIError) details = metadata(error.body, true);
+    throw error;
+  } finally {
+    notify(observer, {
+      stage,
+      requestedModel: "jev-latest",
+      ...details,
+      outcome,
+      durationMs: performance.now() - start,
+    });
+  }
+}
+
+export class ReviewEventStream {
+  private readonly model: string;
+  private readonly observer: ModelCallObserver | undefined;
+  private readonly outcome: () => Exclude<Outcome, "success">;
+  private readonly started = performance.now();
+  private readonly seen = new Set<string>();
+  private buffer = "";
+  private invalid = false;
+  private sequence = 0;
+  private pending: { started: number; details: Metadata } | undefined;
+  private answer: string | undefined;
+  private finished = false;
+
+  constructor(
+    model: string,
+    observer: ModelCallObserver | undefined,
+    outcome: () => Exclude<Outcome, "success">,
+  ) {
+    this.model = model;
+    this.observer = observer;
+    this.outcome = outcome;
+  }
+
+  push(chunk: string): void {
+    if (this.finished) return;
+    this.buffer += chunk;
+    let newline = this.buffer.indexOf("\n");
+    while (newline >= 0) {
+      const line = this.buffer.slice(0, newline).trim();
+      this.buffer = this.buffer.slice(newline + 1);
+      this.checkRecordSize(line);
+      this.accept(line);
+      newline = this.buffer.indexOf("\n");
+    }
+    this.checkRecordSize(this.buffer);
+  }
+
+  private checkRecordSize(record: string): void {
+    if (Buffer.byteLength(record) <= 1024 * 1024) {
+      return;
+    }
+
+    this.buffer = "";
+    this.invalid = true;
+    throw new Error("Reviewer output exceeded limit");
+  }
+
+  finish(): void {
+    if (this.finished) return;
+    if (this.buffer.trim().length > 0) this.accept(this.buffer.trim());
+    this.buffer = "";
+    this.finished = true;
+    if (this.pending === undefined) {
+      return;
+    }
+
+    delete this.pending.details.usage;
+    this.emit(this.pending.details, this.outcome());
+  }
+
+  text(): string {
+    this.finish();
+    if (this.invalid)
+      throw new Error("Reviewer returned an invalid JSON event stream");
+    if (this.answer === undefined)
+      throw new Error("Reviewer returned no successful assistant answer");
+    return this.answer;
+  }
+
+  private accept(line: string): void {
+    if (line.length === 0) return;
+    try {
+      const event = record(JSON.parse(line) as unknown);
+      if (typeof event?.["type"] !== "string") {
+        this.invalid = true;
+        return;
+      }
+      this.event(event);
+    } catch {
+      this.invalid = true;
+    }
+  }
+
+  private event(event: Record<string, unknown>): void {
+    const message = record(event["message"]);
+    switch (event["type"]) {
+      case "message_start":
+        if (message?.["role"] === "assistant") {
+          this.sequence += 1;
+          this.answer = undefined;
+          this.pending = {
+            started: performance.now(),
+            details: metadata(message),
+          };
+        }
+        break;
+      case "message_update": {
+        this.answer = undefined;
+        this.pending ??= { started: performance.now(), details: {} };
+        const details = metadata(message ?? event);
+        delete details.usage;
+        this.pending.details = { ...this.pending.details, ...details };
+        break;
+      }
+      case "message_end":
+        if (message === undefined) this.invalid = true;
+        else if (message["role"] === "assistant")
+          this.endMessage(event, message);
+        break;
+      default:
+        break;
+    }
+  }
+
+  private endMessage(
+    event: Record<string, unknown>,
+    message: Record<string, unknown>,
+  ): void {
+    const identity =
+      typeof event["messageId"] === "string"
+        ? event["messageId"]
+        : `${String(this.sequence)}:${JSON.stringify(message)}`;
+    if (this.seen.has(identity)) return;
+    this.seen.add(identity);
+    const reason = message["stopReason"];
+    const successful =
+      reason === "stop" || reason === "length" || reason === "toolUse";
+    let outcome: Outcome = successful ? "success" : "failed";
+    if (reason === "aborted") {
+      outcome = this.outcome() === "timeout" ? "timeout" : "cancelled";
+    }
+    this.emit({ ...this.pending?.details, ...metadata(message) }, outcome);
+    this.answer = undefined;
+    if (!successful) return;
+    const content = message["content"];
+    if (!Array.isArray(content)) {
+      this.invalid = true;
+      return;
+    }
+    this.answer = content
+      .flatMap((block: unknown) => {
+        const part = record(block);
+        return part?.["type"] === "text" && typeof part["text"] === "string"
+          ? [part["text"]]
+          : [];
+      })
+      .join("\n");
+    if (Buffer.byteLength(this.answer) <= 256_000) {
+      return;
+    }
+
+    this.answer = undefined;
+    this.invalid = true;
+  }
+
+  private emit(details: Metadata, outcome: Outcome): void {
+    if (
+      outcome !== "success" &&
+      details.usage !== undefined &&
+      Object.values(details.usage).every((value) => value <= 0)
+    ) {
+      delete details.usage;
+    }
+    notify(this.observer, {
+      stage: "review",
+      requestedModel: this.model,
+      ...details,
+      outcome,
+      durationMs: performance.now() - (this.pending?.started ?? this.started),
+    });
+    this.pending = undefined;
+  }
+}