pi-ui-extend 0.1.35 → 0.1.37

package/external/pi-tools-suite/src/dcp/debug-log.ts

@@ -0,0 +1,235 @@
+import * as fs from "node:fs/promises"
+import * as os from "node:os"
+import * as path from "node:path"
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent"
+import type { DcpConfig } from "./config.js"
+import type { DcpState } from "./state.js"
+
+const TRUE_ENV_RE = /^(1|true|yes|on)$/i
+const FALSE_ENV_RE = /^(0|false|no|off)$/i
+const MAX_IDS = 16
+const DEFAULT_DEBUG_LOG_MAX_BYTES = 5 * 1024 * 1024 // 5 MB
+const DEFAULT_DEBUG_LOG_MAX_BACKUPS = 3
+const MIN_DEBUG_LOG_MAX_BACKUPS = 1
+
+function truthyEnv(value: string | undefined): boolean | undefined {
+  if (value === undefined) return undefined
+  const trimmed = value.trim()
+  if (TRUE_ENV_RE.test(trimmed)) return true
+  if (FALSE_ENV_RE.test(trimmed)) return false
+  return undefined
+}
+
+export function dcpDebugEnabled(config: DcpConfig): boolean {
+  return truthyEnv(process.env.PI_DCP_DEBUG)
+    ?? truthyEnv(process.env.PI_TOOLS_SUITE_DCP_DEBUG)
+    ?? config.debug
+}
+
+function defaultLogPath(): string {
+  const agentDir = process.env.PI_AGENT_DIR || path.join(os.homedir(), ".pi", "agent")
+  return path.join(agentDir, "dcp-debug.jsonl")
+}
+
+function dcpDebugLogPath(): string {
+  const explicit = process.env.PI_DCP_DEBUG_LOG?.trim()
+  return explicit || defaultLogPath()
+}
+
+function positiveIntEnv(value: string | undefined): number | undefined {
+  if (value === undefined) return undefined
+  const parsed = Number.parseInt(value.trim(), 10)
+  return Number.isFinite(parsed) && parsed > 0 ? parsed : undefined
+}
+
+/** Maximum size the active debug log is allowed to reach before it is rotated. */
+export function dcpDebugLogMaxBytes(config: DcpConfig): number {
+  return positiveIntEnv(process.env.PI_DCP_DEBUG_MAX_BYTES)
+    ?? config.debugLog?.maxBytes
+    ?? DEFAULT_DEBUG_LOG_MAX_BYTES
+}
+
+/** Number of rotated backups to keep (e.g. `.1`, `.2`, `.3`). */
+export function dcpDebugLogMaxBackups(config: DcpConfig): number {
+  const value = positiveIntEnv(process.env.PI_DCP_DEBUG_MAX_BACKUPS)
+    ?? config.debugLog?.maxBackups
+    ?? DEFAULT_DEBUG_LOG_MAX_BACKUPS
+  return Math.max(MIN_DEBUG_LOG_MAX_BACKUPS, Math.floor(value))
+}
+
+function compactIds(ids: string[]): { count: number; head: string[]; tail: string[] } {
+  return {
+    count: ids.length,
+    head: ids.slice(0, MAX_IDS),
+    tail: ids.length > MAX_IDS ? ids.slice(-MAX_IDS) : [],
+  }
+}
+
+function safeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error)
+}
+
+function sessionInfo(ctx: ExtensionContext | undefined): Record<string, unknown> {
+  if (!ctx) return {}
+  const info: Record<string, unknown> = {}
+  try {
+    const header = (ctx as any).sessionManager?.getHeader?.()
+    if (header?.id) info.sessionId = header.id
+    if (header?.cwd) info.cwd = header.cwd
+  } catch (error) {
+    info.sessionInfoError = safeError(error)
+  }
+  try {
+    const name = (ctx as any).sessionManager?.getSessionName?.()
+    if (name) info.sessionName = name
+  } catch {
+    // Optional diagnostic only.
+  }
+  return info
+}
+
+export function summarizeDcpState(state: DcpState): Record<string, unknown> {
+  const rawIds = [...new Set([
+    ...state.messageIdSnapshot.keys(),
+    ...state.messageMetaSnapshot.keys(),
+  ])]
+  const activeBlocks = state.compressionBlocks
+    .filter((block) => block.active)
+    .sort((a, b) => a.id - b.id)
+    .map((block) => ({
+      id: `b${block.id}`,
+      topic: block.topic,
+      mode: block.mode,
+      startMessageId: block.startMessageId,
+      endMessageId: block.endMessageId,
+      anchorMessageId: block.anchorMessageId,
+      coveredBlockIds: block.coveredBlockIds ?? [],
+      summaryTokens: block.summaryTokenEstimate,
+    }))
+  const inactiveBlocks = state.compressionBlocks
+    .filter((block) => !block.active)
+    .sort((a, b) => a.id - b.id)
+    .slice(-MAX_IDS)
+    .map((block) => ({
+      id: `b${block.id}`,
+      topic: block.topic,
+      reason: block.deactivatedReason,
+      coveredBlockIds: block.coveredBlockIds ?? [],
+    }))
+
+  return {
+    rawIds: compactIds(rawIds),
+    activeBlocks,
+    blockCounts: {
+      active: activeBlocks.length,
+      inactive: state.compressionBlocks.length - activeBlocks.length,
+      total: state.compressionBlocks.length,
+      nextBlockId: state.nextBlockId,
+    },
+    inactiveBlocksTail: inactiveBlocks,
+    prunedTools: state.prunedToolIds.size,
+    nudgeAnchors: state.nudgeAnchors.map((anchor) => ({
+      id: anchor.id,
+      type: anchor.type,
+      anchorStableId: anchor.anchorStableId,
+      anchorTimestamp: anchor.anchorTimestamp,
+    })),
+  }
+}
+
+// Serializes all debug-log writes so rotation and appends never race.
+let logWriteChain: Promise<void> = Promise.resolve()
+const ensuredLogDirs = new Set<string>()
+
+async function ensureLogDir(logPath: string): Promise<void> {
+  const dir = path.dirname(logPath)
+  if (ensuredLogDirs.has(dir)) return
+  await fs.mkdir(dir, { recursive: true })
+  ensuredLogDirs.add(dir)
+}
+
+/**
+ * When the active log has reached `maxBytes`, rotate numbered backups:
+ * drop `.N`, shift `.(N-1)`→`.N`, …, `.1`→`.2`, and rename the active file to
+ * `.1` so the next append starts a fresh file. Best-effort: fs errors are
+ * swallowed because debug logging must never affect the session.
+ */
+async function rotateDebugLogIfNeeded(
+  logPath: string,
+  maxBytes: number,
+  maxBackups: number,
+): Promise<void> {
+  let stat: Awaited<ReturnType<typeof fs.stat>>
+  try {
+    stat = await fs.stat(logPath)
+  } catch {
+    return // active file does not exist yet; nothing to rotate
+  }
+  if (stat.size < maxBytes) return
+
+  const dir = path.dirname(logPath)
+  const base = path.basename(logPath)
+
+  // Drop the oldest backup (`base.N`) so the chain can shift up by one.
+  await fs.rm(path.join(dir, `${base}.${maxBackups}`), { force: true })
+  // Shift existing backups `base.i` → `base.(i+1)` from highest to lowest.
+  for (let i = maxBackups - 1; i >= 1; i--) {
+    await fs
+      .rename(path.join(dir, `${base}.${i}`), path.join(dir, `${base}.${i + 1}`))
+      .catch(() => {
+        // A missing intermediate backup is expected; ignore.
+      })
+  }
+  // Rotate the active file into `base.1`.
+  await fs.rename(logPath, path.join(dir, `${base}.1`)).catch(() => {
+    // If we cannot move the active file, truncate it so growth is bounded.
+    void fs.truncate(logPath, 0).catch(() => {})
+  })
+}
+
+async function appendDebugLogRecord(
+  logPath: string,
+  record: Record<string, unknown>,
+  maxBytes: number,
+  maxBackups: number,
+): Promise<void> {
+  await ensureLogDir(logPath)
+  await rotateDebugLogIfNeeded(logPath, maxBytes, maxBackups)
+  await fs.appendFile(logPath, `${JSON.stringify(record)}\n`, "utf8")
+}
+
+export function writeDcpDebugLog(
+  config: DcpConfig,
+  event: string,
+  details: Record<string, unknown> = {},
+  ctx?: ExtensionContext,
+): void {
+  if (!dcpDebugEnabled(config)) return
+
+  const record = {
+    ts: new Date().toISOString(),
+    event,
+    ...sessionInfo(ctx),
+    ...details,
+  }
+
+  const logPath = dcpDebugLogPath()
+  const maxBytes = dcpDebugLogMaxBytes(config)
+  const maxBackups = dcpDebugLogMaxBackups(config)
+
+  // Serialize writes so concurrent records append in order and rotation is safe.
+  logWriteChain = logWriteChain
+    .then(() => appendDebugLogRecord(logPath, record, maxBytes, maxBackups))
+    .catch(() => {
+      // Debug logging must never affect the session or tool outcome.
+    })
+}
+
+/**
+ * Resolves when all queued debug-log writes (and rotations) have settled. Useful
+ * for flushing before shutdown and for deterministic test assertions.
+ */
+export function dcpDebugLogDrain(): Promise<void> {
+  return logWriteChain
+}
+

package/external/pi-tools-suite/src/dcp/index.ts

@@ -9,10 +9,12 @@ import {
 	resetState,
 	createInputFingerprint,
 	restoreState,
+	inheritCompressionBlocks,
 } from "./state.js"
 import {
 	cleanupStaleDcpStateFiles,
 	loadDcpState,
+	loadDcpStateFromSessionFile,
 	resetDcpPersistenceDedup,
 	saveDcpState,
 } from "./state-persistence.js"
@@ -32,12 +34,21 @@ import {
 	detectMessageCompressionCandidates,
 	appendConcreteNudgeGuidance,
 	applyAnchoredNudges,
+	clearDcpNudgeAnchors,
 	nudgeTypeLabel,
 	upsertNudgeAnchor,
 	getActiveSummaryTokenEstimate,
 	resolveContextThresholds,
 	estimateTokens,
 } from "./pruner.js"
+import {
+	stripStaleDcpMetadataFromAssistantMessage,
+	stripStaleDcpMetadataFromMessage,
+} from "./pruner-metadata.js"
+import {
+	buildMessageIdControlText,
+} from "./pruner-message-ids.js"
+import { summarizeDcpState, writeDcpDebugLog } from "./debug-log.js"
 import type { DcpNudgeType } from "./pruner-types.js"
 import { registerCompressTool } from "./compress-tool.js"
 import { DCP_STATS_MESSAGE_TYPE, registerCommands } from "./commands.js"
@@ -88,13 +99,87 @@ function isUserVisibleOnlyMessage(message: any): boolean {
 	return message.details?.userVisibleOnly === true
 }
 
-const DCP_CONTROL_PLANE_CUSTOM_TYPES = new Set(["dcp-state", "dcp-nudge"])
+// Control-plane custom message types filtered out of the transcript.
+// `dcp-message-ids` is retained only for backward-compat with logs written by
+// the removed inline control-message path.
+const DCP_CONTROL_PLANE_CUSTOM_TYPES = new Set(["dcp-state", "dcp-nudge", "dcp-message-ids"])
 const SUMMARY_BUFFER_MAX_CONTEXT_BONUS = 0.05
 
 function isDcpControlPlaneMessage(message: any): boolean {
 	return message?.role === "custom" && DCP_CONTROL_PLANE_CUSTOM_TYPES.has(message.customType)
 }
 
+const DCP_PROVIDER_CONTROL_HEADER = "DCP message ID control data (do not quote or output):"
+
+function appendTextToContent(content: unknown, text: string): unknown {
+	if (typeof content === "string") return `${content}\n\n${text}`
+	if (Array.isArray(content)) {
+		const textType = content.some((part: any) => part?.type === "input_text") ? "input_text" : "text"
+		return [...content, { type: textType, text }]
+	}
+	return text
+}
+
+function appendDcpControlToMessages(messages: unknown, text: string): unknown {
+	if (!Array.isArray(messages)) return messages
+	const existingIndex = messages.findIndex((message: any) =>
+		message?.role === "system" || message?.role === "developer"
+	)
+	const block = `${DCP_PROVIDER_CONTROL_HEADER}\n${text}`
+	if (existingIndex >= 0) {
+		return messages.map((message: any, index) => index === existingIndex
+			? { ...message, content: appendTextToContent(message.content, block) }
+			: message)
+	}
+	return [{ role: "system", content: block }, ...messages]
+}
+
+function appendDcpControlToAnthropicSystem(system: unknown, text: string): unknown {
+	const block = `${DCP_PROVIDER_CONTROL_HEADER}\n${text}`
+	if (typeof system === "string") return `${system}\n\n${block}`
+	if (Array.isArray(system)) return [...system, { type: "text", text: block }]
+	if (system === undefined || system === null) return [{ type: "text", text: block }]
+	return system
+}
+
+function appendDcpControlToGoogleSystemInstruction(systemInstruction: unknown, text: string): unknown {
+	const block = `${DCP_PROVIDER_CONTROL_HEADER}\n${text}`
+	if (typeof systemInstruction === "string") return `${systemInstruction}\n\n${block}`
+	if (systemInstruction === undefined || systemInstruction === null) return block
+	return systemInstruction
+}
+
+function appendDcpControlToProviderPayload(payload: unknown, text: string): unknown {
+	if (Array.isArray(payload)) return appendDcpControlToMessages(payload, text)
+	if (!payload || typeof payload !== "object") return payload
+	const record = payload as Record<string, unknown>
+
+	if ("system" in record) {
+		return { ...record, system: appendDcpControlToAnthropicSystem(record.system, text) }
+	}
+
+	if (Array.isArray(record.input)) {
+		return { ...record, input: appendDcpControlToMessages(record.input, text) }
+	}
+
+	if (Array.isArray(record.messages)) {
+		return { ...record, messages: appendDcpControlToMessages(record.messages, text) }
+	}
+
+	if (record.config && typeof record.config === "object") {
+		const config = record.config as Record<string, unknown>
+		return {
+			...record,
+			config: {
+				...config,
+				systemInstruction: appendDcpControlToGoogleSystemInstruction(config.systemInstruction, text),
+			},
+		}
+	}
+
+	return payload
+}
+
 // ---------------------------------------------------------------------------
 // Module export
 // ---------------------------------------------------------------------------
@@ -150,7 +235,7 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 	registerCommands(pi, state, config)
 
 	// ── 5. session_start: restore state from session entries ──────────────────
-	pi.on("session_start", async (_event, ctx) => {
+	pi.on("session_start", async (event, ctx) => {
 		// Reset to a clean slate first.
 		resetState(state)
 
@@ -169,11 +254,37 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 		})
 		restoreState(state, await loadDcpState(ctx))
 
+		// fork/resume/new sessions inherit the source conversation but get a fresh
+		// sidecar; inherit the previous session's compression blocks so they are
+		// not silently lost (which previously forced re-compressing all history).
+		if (state.compressionBlocks.length === 0 && event.previousSessionFile) {
+			try {
+				const inherited = await loadDcpStateFromSessionFile(event.previousSessionFile)
+				const added = inheritCompressionBlocks(state, inherited)
+				if (added > 0) {
+					writeDcpDebugLog(configForContext(ctx), "session_start.inherited_blocks", {
+						reason: event.reason,
+						previousSessionFile: event.previousSessionFile,
+						added,
+						totalBlocks: state.compressionBlocks.length,
+					}, ctx)
+					// Persist inherited state into this session's own sidecar so a later
+					// reload restores it directly.
+					await saveDcpState(ctx, state)
+				}
+			} catch {
+				// Inheritance is best-effort; never block session startup.
+			}
+		}
+
 		// Headless by design: no extension status/footer/widgets are rendered.
 	})
 
 	// ── 6. session_shutdown: save state ───────────────────────────────────────
 	pi.on("session_shutdown", async (_event, ctx) => {
+		// Force-flush: bypass the dedup hash so the final snapshot is always
+		// written, guaranteeing the next session_start can restore it.
+		resetDcpPersistenceDedup()
 		await saveDcpState(ctx, state)
 	})
 
@@ -191,6 +302,15 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 		}
 	})
 
+	// ── 7b. message_end: never persist provider-echoed DCP control markers ─────
+	pi.on("message_end", async (event, ctx) => {
+		const effectiveConfig = configForContext(ctx)
+		if (!effectiveConfig.enabled || event.message?.role !== "assistant") return undefined
+
+		const sanitized = stripStaleDcpMetadataFromAssistantMessage(event.message)
+		return { message: sanitized }
+	})
+
 	// ── 8. tool_call: record input args for dedup / purge fingerprinting ───────
 	pi.on("tool_call", async (event, _ctx) => {
 		if (!state.toolCalls.has(event.toolCallId)) {
@@ -247,10 +367,32 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 	// ── 10. context: apply pruning and inject nudges ──────────────────────────
 	pi.on("context", async (event, ctx) => {
 		const effectiveConfig = configForContext(ctx)
-		const contextMessages = event.messages.filter((message: any) =>
-			!isUserVisibleOnlyMessage(message) && !isDcpControlPlaneMessage(message)
-		)
+		const contextMessages = event.messages
+			.filter((message: any) => !isUserVisibleOnlyMessage(message) && !isDcpControlPlaneMessage(message))
+			.map((message: any) => stripStaleDcpMetadataFromMessage(message))
+		const finishContext = (reason: string, messages: any[], details: Record<string, unknown> = {}) => {
+			writeDcpDebugLog(effectiveConfig, "context.result", {
+				reason,
+				inputMessages: event.messages.length,
+				filteredMessages: contextMessages.length,
+				outputMessages: messages.length,
+				messageIdControl: "provider-payload",
+				state: summarizeDcpState(state),
+				...details,
+			}, ctx)
+			return { messages }
+		}
+
+		writeDcpDebugLog(effectiveConfig, "context.start", {
+			inputMessages: event.messages.length,
+			filteredMessages: contextMessages.length,
+			filteredDcpControlPlaneMessages: event.messages.length - contextMessages.length,
+		}, ctx)
 		if (!effectiveConfig.enabled) {
+			writeDcpDebugLog(effectiveConfig, "context.disabled", {
+				inputMessages: event.messages.length,
+				filteredMessages: contextMessages.length,
+			}, ctx)
 			return { messages: contextMessages }
 		}
 		annotateMessagesWithBranchEntryIds(contextMessages, ctx)
@@ -270,15 +412,9 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 					: undefined
 
 			if (contextPercent === undefined) {
-				if (state.manualMode) {
-					state.nudgeAnchors = state.nudgeAnchors.filter((anchor) =>
-						anchor.type === "context-strong" || anchor.type === "context-soft",
-					)
-				}
-				applyAnchoredNudges(prunedMessages, state, (anchor) =>
-					appendConcreteNudgeGuidance(baseNudgeText(anchor.type), candidate, messageCandidates, state),
-				)
-				return { messages: prunedMessages }
+				const clearedAnchors = clearDcpNudgeAnchors(state)
+				if (clearedAnchors > 0) await saveDcpState(ctx, state)
+				return finishContext("unknown-context-percent", prunedMessages, { clearedAnchors })
 			}
 
 			const ctxModel = (ctx as any).model
@@ -293,6 +429,18 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 				thresholds.maxContextPercent += Math.min(summaryBonus, SUMMARY_BUFFER_MAX_CONTEXT_BONUS)
 			}
 
+			const contextLimitReached = contextPercent > thresholds.maxContextPercent
+			const routineNudgesAllowed = contextPercent > thresholds.minContextPercent
+			if (!contextLimitReached && !routineNudgesAllowed) {
+				const clearedAnchors = clearDcpNudgeAnchors(state)
+				if (clearedAnchors > 0) await saveDcpState(ctx, state)
+				return finishContext("below-threshold", prunedMessages, {
+					contextPercent,
+					thresholds,
+					clearedAnchors,
+				})
+			}
+
 			let toolCallsSinceLastUser = 0
 			for (let i = prunedMessages.length - 1; i >= 0; i--) {
 				const msg = prunedMessages[i] as any
@@ -312,18 +460,28 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 				state.manualMode &&
 				(nudgeType !== "context-strong" && nudgeType !== "context-soft")
 
-			candidate = detectCompressionCandidate(
-				prunedMessages,
-				state,
-				effectiveConfig,
-				contextPercent,
-			)
-			messageCandidates = detectMessageCompressionCandidates(
-				prunedMessages,
-				state,
-				effectiveConfig,
-				contextPercent,
-			)
+			if (!manualEmergencyOnly) {
+				candidate = detectCompressionCandidate(
+					prunedMessages,
+					state,
+					effectiveConfig,
+					contextPercent,
+				)
+				messageCandidates = detectMessageCompressionCandidates(
+					prunedMessages,
+					state,
+					effectiveConfig,
+					contextPercent,
+				)
+				writeDcpDebugLog(effectiveConfig, "context.candidates", {
+					contextPercent,
+					thresholds,
+					nudgeType,
+					candidate,
+					messageCandidates,
+					state: summarizeDcpState(state),
+				}, ctx)
+			}
 
 			if (nudgeType && !manualEmergencyOnly) {
 				const nudgeText = appendConcreteNudgeGuidance(
@@ -386,7 +544,26 @@ export default async function dcpModule(pi: ExtensionAPI): Promise<void> {
 			appendConcreteNudgeGuidance(baseNudgeText(anchor.type), candidate, messageCandidates, state),
 		)
 
-		return { messages: prunedMessages }
+		return finishContext("complete", prunedMessages, {
+			candidate,
+			messageCandidates,
+		})
+	})
+
+	// ── 10b. before_provider_request: inject DCP IDs outside transcript ────────
+	pi.on("before_provider_request", async (event, ctx) => {
+		const effectiveConfig = configForContext(ctx)
+		if (!effectiveConfig.enabled) return undefined
+
+		const controlText = buildMessageIdControlText(state)
+		if (!controlText) return undefined
+
+		const payload = appendDcpControlToProviderPayload(event.payload, controlText)
+		writeDcpDebugLog(effectiveConfig, "provider_payload.message_ids", {
+			injected: payload !== event.payload,
+			state: summarizeDcpState(state),
+		}, ctx)
+		return payload === event.payload ? undefined : payload
 	})
 
 	// ── 11. agent_end: persist state after each agent run ────────────────────