pi-ui-extend 1.0.40 → 1.0.42

package/external/pi-tools-suite/src/dcp/compression-preview.ts

@@ -0,0 +1,122 @@
+import type { DcpConfig } from "./config.js"
+import { canonicalMessageHash, copyRawMutationHash } from "./conversation-index.js"
+import { cloneDcpTransactionState } from "./state-transaction.js"
+import type { CompressionBlock, ConversationIndexEntry, DcpState } from "./state.js"
+import { stripStaleDcpMetadataFromMessage, estimateMessageTokens } from "./pruner-metadata.js"
+import { injectMessageIds, stableMessageKeys } from "./pruner-message-ids.js"
+import { applyCompressionBlocks } from "./pruner-compression-blocks.js"
+
+/** The latest provider projection is runtime-only and intentionally never persisted. */
+const projectionSnapshots = new WeakMap<ConversationIndexEntry[], any[]>()
+
+const DCP_PROVENANCE_KEYS = ["_dcpOrigin", "_dcpStableId", "_dcpBlockId"] as const
+
+function copyDcpProvenance(source: any, target: any): any {
+  if (!source || !target || typeof source !== "object" || typeof target !== "object") return target
+  for (const key of DCP_PROVENANCE_KEYS) {
+    const descriptor = Object.getOwnPropertyDescriptor(source, key)
+    if (descriptor) Object.defineProperty(target, key, descriptor)
+  }
+  copyRawMutationHash(source, target)
+  return target
+}
+
+/** Clone message objects while retaining the non-enumerable DCP provenance used by exact spans. */
+export function cloneProjectionMessages(messages: readonly any[]): any[] {
+  return messages.map((message) => {
+    const clone = message && typeof message === "object" ? { ...message } : message
+    if (!clone || typeof clone !== "object") return clone
+    if (message.content && typeof message.content === "object") clone.content = structuredClone(message.content)
+    return copyDcpProvenance(message, clone)
+  })
+}
+
+/** Store a detached copy of the exact provider projection that produced this index. */
+export function recordCompressionProjection(
+  entries: ConversationIndexEntry[],
+  messages: readonly any[],
+): void {
+  projectionSnapshots.set(entries, cloneProjectionMessages(messages))
+}
+
+function verifiedSnapshot(state: DcpState): any[] {
+  const snapshot = projectionSnapshots.get(state.conversationIndexSnapshot)
+  if (!snapshot || snapshot.length !== state.conversationIndexSnapshot.length) {
+    throw new Error(
+      "Manual compression requires a current verifiable provider projection; refresh the DCP context and retry.",
+    )
+  }
+  const stableKeys = stableMessageKeys(snapshot)
+  for (let index = 0; index < snapshot.length; index++) {
+    const entry = state.conversationIndexSnapshot[index]
+    if (!entry || entry.stableId !== stableKeys[index] || entry.contentHash !== canonicalMessageHash(snapshot[index])) {
+      throw new Error(
+        "Manual compression provider projection no longer matches the current DCP snapshot; refresh the context and retry.",
+      )
+    }
+  }
+  return snapshot
+}
+
+/**
+ * Return a detached, verified copy of the exact provider projection used by
+ * manual compression. Generated summaries must use this source rather than
+ * reconstructing history from timestamps or persisted session entries.
+ */
+export function verifiedManualCompressionMessages(state: DcpState): any[] {
+  return cloneProjectionMessages(verifiedSnapshot(state))
+}
+
+function stripProjectionMetadata(message: any): any {
+  const stripped = stripStaleDcpMetadataFromMessage(message)
+  return copyDcpProvenance(message, stripped)
+}
+
+export interface ManualCompressionProjection {
+  projectedBeforeTokens: number
+  projectedAfterTokens: number
+  netGain: number
+}
+
+/**
+ * Materialize only an operation's new blocks against the exact current
+ * provider projection. Existing blocks are already represented there as
+ * synthetic entries, so replaying them would double-count and hide a bad
+ * manual summary behind unrelated automatic pruning.
+ */
+export function previewManualCompressionProjection(
+  state: DcpState,
+  config: DcpConfig,
+  newBlocks: CompressionBlock[],
+): ManualCompressionProjection {
+  const source = verifiedSnapshot(state)
+  const before = cloneProjectionMessages(source)
+  const after = cloneProjectionMessages(source).map(stripProjectionMetadata)
+  const previewState = cloneDcpTransactionState(state)
+  previewState.compressionBlocks = newBlocks.map((block) => structuredClone(block))
+  previewState.accountedCompressionBlockIds = new Set()
+  previewState.compressionTokenSavings = new Map()
+
+  applyCompressionBlocks(after, previewState)
+  const appliedBlockIds = new Set(
+    after.flatMap((message) => Number.isInteger(message?._dcpBlockId) ? [message._dcpBlockId as number] : []),
+  )
+  const missing = newBlocks.filter((block) => !appliedBlockIds.has(block.id))
+  if (missing.length > 0) {
+    throw new Error(
+      `Manual compression could not materialize ${missing.map((block) => `b${block.id}`).join(", ")} in the current provider projection; no state was published.`,
+    )
+  }
+
+  // Recreate carrier overhead on a detached state. This deliberately happens
+  // after applying the wrapper so the comparison includes the full delivered
+  // provider projection rather than just summary text.
+  injectMessageIds(after, previewState, { config })
+  const projectedBeforeTokens = before.reduce((sum, message) => sum + estimateMessageTokens(message), 0)
+  const projectedAfterTokens = after.reduce((sum, message) => sum + estimateMessageTokens(message), 0)
+  return {
+    projectedBeforeTokens,
+    projectedAfterTokens,
+    netGain: projectedBeforeTokens - projectedAfterTokens,
+  }
+}

package/external/pi-tools-suite/src/dcp/compression-progress.ts

@@ -0,0 +1,70 @@
+import type { DcpState } from "./state.js"
+
+/** Routine recovery aims for up to five percentage points, never below min. */
+export function routineRecoveryTokens(tokens: number, window: number, minPercent: number): number {
+  return Math.max(1, Math.ceil(Math.min(window * 0.05, Math.max(0, tokens - window * minPercent))))
+}
+
+export interface CompressionProgressInput {
+  projectedTokens: number
+  contextWindow: number
+  requiredTokens: number
+  kind: "routine" | "emergency"
+  observedTokens?: number
+  targetHeadroomTokens?: number
+}
+
+/** Remaining debt follows new content/usage, not a stale pre-compression sample. */
+export function outstandingCompressionTokens(state: DcpState, input: CompressionProgressInput): number | undefined {
+  const previous = state.compressionProgress
+  if (!previous || previous.contextWindow !== input.contextWindow) return undefined
+  const projectionChange = input.projectedTokens - previous.projectedTokens
+  const observedGrowth = input.observedTokens !== undefined && previous.observedTokens !== undefined
+    ? Math.max(0, input.observedTokens - previous.observedTokens) : 0
+  const growth = observedGrowth > 0 ? Math.max(projectionChange, observedGrowth) : projectionChange
+  const stricterTarget = previous.kind === input.kind && previous.targetHeadroomTokens !== undefined && input.targetHeadroomTokens !== undefined
+    ? Math.max(0, previous.targetHeadroomTokens - input.targetHeadroomTokens) : 0
+  return Math.max(0, previous.remainingTokens + growth + stricterTarget)
+}
+
+/** Follow actual projection growth, not context callback/retry count. */
+export function trackCompressionProgress(
+  state: DcpState,
+  input: CompressionProgressInput,
+): void {
+  const previous = state.compressionProgress
+  const remaining = outstandingCompressionTokens(state, input)
+  state.compressionProgress = {
+    projectedTokens: input.projectedTokens,
+    contextWindow: input.contextWindow,
+    // The unchanged native sample still describes the input before a manual
+    // commit. Do not charge the same already-recovered tokens on every callback.
+    remainingTokens: previous && previous.kind === input.kind && remaining !== undefined && remaining > 0
+      ? remaining : Math.max(1, input.requiredTokens, remaining ?? 0),
+    kind: input.kind,
+    observedTokens: input.observedTokens,
+    targetHeadroomTokens: previous?.kind === input.kind && previous.targetHeadroomTokens !== undefined && input.targetHeadroomTokens !== undefined
+      ? Math.min(previous.targetHeadroomTokens, input.targetHeadroomTokens) : input.targetHeadroomTokens,
+  }
+}
+
+export function resetCompressionProgress(state: DcpState): void {
+  state.consecutiveIgnoredStrongNudges = 0
+  state.consecutiveIgnoredNudges = 0
+  state.compressionProgress = undefined
+}
+
+/** Called on a detached state only after positive full-projection gain is proved. */
+export function settleCompressionProgress(state: DcpState, gain: number, projectedAfterTokens: number): boolean {
+  const goal = state.compressionProgress
+  if (!goal || gain >= goal.remainingTokens) {
+    resetCompressionProgress(state)
+    return true
+  }
+  state.compressionProgress = {
+    ...goal,
+    remainingTokens: goal.remainingTokens - gain,
+    projectedTokens: projectedAfterTokens,
+  }
+  return false
+}

package/external/pi-tools-suite/src/dcp/config.ts

@@ -10,13 +10,14 @@ import { parse as parseJsonc, type ParseError } from "jsonc-parser"
 export interface DcpConfig {
   enabled: boolean
   debug: boolean
+  /** Non-fatal loader diagnostics, e.g. removed settings that were ignored. */
+  issues?: string[]
   debugLog?: {
     maxBytes?: number
     maxBackups?: number
   }
   manualMode: {
     enabled: boolean
-    automaticStrategies: boolean // run dedup/purge even in manual mode
   }
   compress: {
     maxContextPercent: number | string // accepts a 0-1 fraction, absolute tokens, or a percent string like "80%"
@@ -50,41 +51,27 @@ export interface DcpConfig {
       maxSuggestions: number
     }
     /**
-     * Auto-compress fallback: when the model ignores repeated context-strong
-     * nudges above the emergency threshold, DCP creates a compression block
-     * itself (without waiting for the model). Lossy and irreversible within a
+     * Auto-compress fallback: after completed actionable provider opportunities
+     * fail to meet the recovery goal, DCP can create a safe compression block,
+     * including after routine-reminder escalation. Lossy within a
      * session — disabled by default; opt in via config.
      */
     autoCompress: {
+      /** Gates autonomous firing only. Explicit `compress` calls may still use
+       * the summarizer settings below when their per-selection summary is omitted. */
       enabled: boolean
-      /** Number of context-strong nudges emitted (and ignored) before DCP
-       * auto-compresses. The model gets `patience` genuine strong chances. */
+      /** Auto fallback is eligible after more than this many completed,
+       * correlated opportunities with a delivered actionable reminder.
+       * Failed calls and insufficient commits do not reset patience. */
       patience: number
-      /** Models to try, in order, when producing a model-generated summary.
+      /** Models to try, in order, for auto-compress and explicit generated summaries.
        * Empty array → deterministic programmatic digest (no model call). */
       summarizerModel: string[]
-      /** Hard ceiling in ms for a single summarizer model call. */
+      /** Hard ceiling in ms for a summarizer model call on either path. */
       timeoutMs: number
     }
   }
   strategies: {
-    deduplication: {
-      enabled: boolean
-      protectedTools: string[]
-    }
-    purgeErrors: {
-      enabled: boolean
-      turns: number // prune error inputs after N user turns (default: 4)
-      protectedTools: string[]
-    }
-    autoToolPruning: {
-      enabled: boolean
-      maxOutputTokens: number
-      keepRecentTurns: number
-      readLikeTools: string[]
-      readLikeTurns: number
-      protectedTools: string[]
-    }
     emergencyCurrentTurnPruning: {
       /** Enable same-turn candidates and lossy fallback pruning; emergency reminders remain active. */
       enabled: boolean
@@ -104,11 +91,10 @@ export interface DcpConfig {
     }
   }
   protectedFilePatterns: string[]
-  pruneNotification: "off" | "minimal" | "detailed"
   modelOverrides: Record<string, DcpConfigOverride>
 }
 
-export type DcpConfigOverride = DeepPartial<Omit<DcpConfig, "modelOverrides">>
+export type DcpConfigOverride = DeepPartial<Omit<DcpConfig, "modelOverrides" | "issues">>
 
 type DeepPartial<T> = T extends Array<infer U>
   ? Array<DeepPartial<U>>
@@ -123,9 +109,9 @@ type DeepPartial<T> = T extends Array<infer U>
 const DEFAULT_CONFIG: DcpConfig = {
   enabled: true,
   debug: false,
+  issues: [],
   manualMode: {
     enabled: false,
-    automaticStrategies: true,
   },
   compress: {
     maxContextPercent: 0.65,
@@ -162,39 +148,6 @@ const DEFAULT_CONFIG: DcpConfig = {
     },
   },
   strategies: {
-    deduplication: {
-      enabled: true,
-      protectedTools: [],
-    },
-    purgeErrors: {
-      enabled: true,
-      turns: 4,
-      protectedTools: [],
-    },
-    autoToolPruning: {
-      enabled: true,
-      maxOutputTokens: 1200,
-      keepRecentTurns: 1,
-      readLikeTools: [
-        "read",
-        "shell",
-        "bash",
-        "grep",
-        "find",
-        "ls",
-        "web_search",
-        "web_fetch",
-        "repo_architecture",
-        "repo_structure",
-        "repo_ast",
-        "repo_search",
-        "repo_explain",
-        "repo_deps",
-        "ast_grep",
-      ],
-      readLikeTurns: 3,
-      protectedTools: [],
-    },
     emergencyCurrentTurnPruning: {
       enabled: true,
       hardContextPercent: 0.82,
@@ -207,7 +160,6 @@ const DEFAULT_CONFIG: DcpConfig = {
     },
   },
   protectedFilePatterns: [],
-  pruneNotification: "detailed",
   modelOverrides: {},
 }
 
@@ -281,15 +233,75 @@ function readJsoncFile(filePath: string): Record<string, unknown> {
 /**
  * Return the nested DCP config from a shared pi-tools-suite config file.
  */
-function readDcpFromSuiteConfig(filePath: string): Record<string, unknown> {
+function readDcpFromSuiteConfig(filePath: string, issues: string[]): Record<string, unknown> {
   const raw = readJsoncFile(filePath)
   const dcp = raw["dcp"]
   if (dcp === null || typeof dcp !== "object" || Array.isArray(dcp)) return {}
-  return dcp as Record<string, unknown>
+  return stripRemovedDcpKeys(dcp as Record<string, unknown>, issues)
 }
 
-function mergeSuiteDcpConfig(config: DcpConfig, filePath: string): DcpConfig {
-  const raw = readDcpFromSuiteConfig(filePath)
+const REMOVED_DCP_GUIDANCE: Record<string, string> = {
+  pruneNotification: "removed; pruning notifications are no longer configurable",
+  "manualMode.automaticStrategies": "removed; manual mode now has one explicit enabled flag",
+  "strategies.deduplication": "removed; no automatic replacement policy is currently mapped to this setting",
+  "strategies.purgeErrors": "removed; no automatic replacement policy is currently mapped to this setting",
+  "strategies.autoToolPruning": "removed; use explicit /dcp sweep or DCP compression/emergency policies instead",
+}
+
+function reportRemovedKey(issues: string[], prefix: string, key: string): void {
+  const fullKey = prefix ? `${prefix}.${key}` : key
+  const canonical = fullKey.replace(/^modelOverrides\.[^.]+\./, "")
+  const guidance = REMOVED_DCP_GUIDANCE[canonical] ?? "removed and ignored"
+  issues.push(`dcp.${fullKey} is ${guidance}.`)
+}
+
+function stripRemovedDcpKeys(
+  raw: Record<string, unknown>,
+  issues: string[],
+  prefix = "",
+): Record<string, unknown> {
+  const cleaned = structuredClone(raw)
+  if (Object.prototype.hasOwnProperty.call(cleaned, "pruneNotification")) {
+    reportRemovedKey(issues, prefix, "pruneNotification")
+  }
+  delete cleaned.pruneNotification
+
+  const manualMode = cleaned.manualMode
+  if (manualMode && typeof manualMode === "object" && !Array.isArray(manualMode)) {
+    if (Object.prototype.hasOwnProperty.call(manualMode, "automaticStrategies")) {
+      reportRemovedKey(issues, prefix, "manualMode.automaticStrategies")
+    }
+    delete (manualMode as Record<string, unknown>).automaticStrategies
+  }
+
+  const strategies = cleaned.strategies
+  if (strategies && typeof strategies === "object" && !Array.isArray(strategies)) {
+    const record = strategies as Record<string, unknown>
+    for (const key of ["deduplication", "purgeErrors", "autoToolPruning"] as const) {
+      if (Object.prototype.hasOwnProperty.call(record, key)) {
+        reportRemovedKey(issues, prefix, `strategies.${key}`)
+      }
+      delete record[key]
+    }
+  }
+
+  const modelOverrides = cleaned.modelOverrides
+  if (modelOverrides && typeof modelOverrides === "object" && !Array.isArray(modelOverrides)) {
+    for (const [key, override] of Object.entries(modelOverrides as Record<string, unknown>)) {
+      if (!override || typeof override !== "object" || Array.isArray(override)) continue
+      ;(modelOverrides as Record<string, unknown>)[key] = stripRemovedDcpKeys(
+        override as Record<string, unknown>,
+        issues,
+        prefix ? `${prefix}.modelOverrides.${key}` : `modelOverrides.${key}`,
+      )
+    }
+  }
+
+  return cleaned
+}
+
+function mergeSuiteDcpConfig(config: DcpConfig, filePath: string, issues: string[]): DcpConfig {
+  const raw = readDcpFromSuiteConfig(filePath, issues)
   if (Object.keys(raw).length === 0) return config
   return deepMerge(config, raw as Partial<DcpConfig>)
 }
@@ -431,10 +443,16 @@ export interface LoadConfigOptions {
  */
 export function loadConfig(options: LoadConfigOptions = {}): DcpConfig {
   // Layer 1: defaults (deep clone so we never mutate the constant)
-  let config: DcpConfig = deepMerge(DEFAULT_CONFIG, {})
+  let config: DcpConfig = structuredClone(DEFAULT_CONFIG)
+  const issues: string[] = []
 
   const homeDir = options.homeDir ?? os.homedir()
-  config = mergeSuiteDcpConfig(config, path.join(homeDir, ".config", "pi", "pi-tools-suite.jsonc"))
+  config = mergeSuiteDcpConfig(
+    config,
+    path.join(homeDir, ".config", "pi", "pi-tools-suite.jsonc"),
+    issues,
+  )
+  config.issues = issues
 
   return config
 }